Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 37 additions & 2 deletions python-ecosys/requests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,49 @@ bodies with Content-Length or Transfer-Encoding: chunked via streaming

### Limitations

* Certificate validation is not currently supported.
* Certificates are only verified when an ``ssl`` context set to ``CERT_REQUIRED``
is passed (see below).
* A dictionary passed as post data will not do automatic JSON or
multipart-form encoding of post data (this can be done manually).
* Compressed requests/responses are not currently supported.
* File upload is not supported.
* HTTP keep-alive connection reuse is not supported (Connection: close by default).

### Verifying certificates

By default HTTPS requests accept any server certificate. To verify the
certificate chain and the hostname, pass an ``SSLContext`` whose
``verify_mode`` is ``CERT_REQUIRED`` as ``ssl``:

```py
import ssl, requests

ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ctx.verify_mode = ssl.CERT_REQUIRED
ctx.load_verify_locations(cafile="ca.der")
with requests.get("https://example.com", ssl=ctx) as r:
print(r.status_code)
```

* ``ca.der`` holds the root CA certificate the server's chain ends at: a
single DER certificate, or one or more in PEM. The server must send any
intermediate certificates.
* ``ssl.SSLContext`` starts with ``CERT_NONE``, and on mbedTLS ports
``CERT_OPTIONAL`` doesn't verify either, so set ``CERT_REQUIRED``.
* The device's clock must be set, for example with ``ntptime``, or
certificates fail their validity check.
* A certificate that fails verification raises ``ValueError`` (``OSError``
in rare cases), so catch both when retrying.
* With ``ssl`` given, an ``http://`` URL, or a redirect to one, raises
``ValueError`` instead of sending the request unencrypted. Redirects to
``https://`` are verified with the same context.
* Verification needs a port whose ``tls`` module uses mbedTLS. esp8266's
axTLS can't verify certificates.
* Firmware that freezes ``requests`` uses the frozen copy ahead of one
installed in ``/lib``.

### Follow-up work

* TLS certificate verification (see micropython-lib issue #838).
* Verifying certificates by default, which needs a source of root
certificates on the device (see issue #838).
* ``stream=True`` incremental body reads (see issue #777).
2 changes: 1 addition & 1 deletion python-ecosys/requests/manifest.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
metadata(version="1.1.1", pypi="requests")
metadata(version="1.2.0", pypi="requests")

package("requests")
20 changes: 15 additions & 5 deletions python-ecosys/requests/requests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ def request(
auth=None,
timeout=None,
parse_headers=True,
ssl=None,
):
if headers is None:
headers = {}
Expand Down Expand Up @@ -124,6 +125,11 @@ def request(
else:
raise ValueError("Unsupported protocol: " + proto)

if ssl is not None and proto != "https:":
# A request that asked for a verified connection never goes out without
# one, including when a redirect points to http://.
raise ValueError("ssl needs https")

if "?" in host:
host, _path = host.split("?", 1)
path = "?" + _path + path
Expand Down Expand Up @@ -153,8 +159,10 @@ def request(
try:
s.connect(ai[-1])
if proto == "https:":
context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT)
context.verify_mode = tls.CERT_NONE
context = ssl
if context is None:
context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT)
context.verify_mode = tls.CERT_NONE
s = context.wrap_socket(s, server_hostname=host)
s.write(b"%s /%s HTTP/1.1\r\n" % (method, path))

Expand Down Expand Up @@ -245,7 +253,9 @@ def request(
remaining = int(v)
else:
parse_headers(l, resp_d)
except OSError:
except:
# Close the socket whatever went wrong: a certificate that fails
# verification raises ValueError, and running out of memory MemoryError.
s.close()
raise

Expand All @@ -254,9 +264,9 @@ def request(
# Use the host specified in the redirect URL, as it may not be the same as the original URL.
headers.pop("Host", None)
if status in [301, 302, 303]:
return request("GET", redirect, None, None, headers, stream)
return request("GET", redirect, None, None, headers, stream, ssl=ssl)
else:
return request(method, redirect, data, json, headers, stream)
return request(method, redirect, data, json, headers, stream, ssl=ssl)
else:
if chunked:
resp = Response(BodyStream(s, -1))
Expand Down
147 changes: 147 additions & 0 deletions python-ecosys/requests/test_requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,33 @@ def socket(af=AF_INET, type=SOCK_STREAM, proto=IPPROTO_TCP):


sys.modules["socket"] = socket


class SSLContext:
def __init__(self, protocol=None):
self.protocol = protocol
self.verify_mode = None
self.wrapped = []

def wrap_socket(self, sock, server_hostname=None):
self.wrapped.append(server_hostname)
return sock


class tls:
PROTOCOL_TLS_CLIENT = 0
CERT_NONE = 0
CERT_REQUIRED = 2
contexts = []

@staticmethod
def SSLContext(protocol):
context = SSLContext(protocol)
tls.contexts.append(context)
return context


sys.modules["tls"] = tls
# ruff: noqa: E402
import requests

Expand Down Expand Up @@ -296,6 +323,120 @@ def test_raw_readinto_content_length():
socket.socket = lambda *a, **k: Socket()


def test_https_without_ssl_skips_verification():
tls.contexts.clear()

requests.request("GET", "https://example.com")

assert len(tls.contexts) == 1
assert tls.contexts[0].verify_mode == tls.CERT_NONE
assert tls.contexts[0].wrapped == ["example.com"]


def test_https_wraps_with_the_given_ssl_context():
tls.contexts.clear()
context = SSLContext()
context.verify_mode = tls.CERT_REQUIRED

requests.get("https://example.com:8443/x", ssl=context)

assert tls.contexts == []
assert context.wrapped == ["example.com"]
assert context.verify_mode == tls.CERT_REQUIRED


def test_redirect_keeps_the_ssl_context():
server = iter(
[
b"HTTP/1.1 302 Found\r\nLocation: https://other.example.com/next\r\n\r\n",
SERVER_RESPONSE_200_OK,
]
)
socket.socket = lambda *a, **k: Socket(next(server))
context = SSLContext()

requests.get("https://example.com", ssl=context)

assert context.wrapped == ["example.com", "other.example.com"]

socket.socket = lambda *a, **k: Socket()


def test_errors_close_the_socket():
class RaisingContext:
def __init__(self, exc):
self.exc = exc

def wrap_socket(self, s, server_hostname=None):
raise self.exc

# A failed verification raises ValueError; running out of memory, MemoryError;
# a value that isn't a context, AttributeError.
for ssl, exc_type in (
(RaisingContext(ValueError("certificate verify failed")), ValueError),
(RaisingContext(MemoryError()), MemoryError),
(True, AttributeError),
):
sock = Socket()
closed = []
sock.close = lambda: closed.append(True)
socket.socket = lambda *a, **k: sock

try:
requests.get("https://example.com", ssl=ssl)
except exc_type:
pass
else:
assert False, "expected %s" % exc_type

assert closed == [True], exc_type

socket.socket = lambda *a, **k: Socket()


def test_ssl_refuses_an_http_url():
made = []
socket.socket = lambda *a, **k: made.append(1) or Socket()

try:
requests.get("http://example.com", ssl=SSLContext())
except ValueError:
pass
else:
assert False, "expected ValueError"

assert made == []

socket.socket = lambda *a, **k: Socket()


def test_ssl_refuses_a_redirect_to_http():
first = Socket(b"HTTP/1.1 302 Found\r\nLocation: http://example.com/next\r\n\r\n")
closed = []
first.close = lambda: closed.append(True)
made = []

def make(*a, **k):
made.append(1)
return first

socket.socket = make
context = SSLContext()

try:
requests.get("https://example.com", ssl=context)
except ValueError:
pass
else:
assert False, "expected ValueError"

assert context.wrapped == ["example.com"]
assert closed == [True]
assert made == [1]

socket.socket = lambda *a, **k: Socket()


def test_redirect_absolute():
server = iter(
[
Expand Down Expand Up @@ -379,6 +520,12 @@ def test_content_length_lowercase_header():
test_post_data_str_unicode()
test_post_data_bytes()
test_post_chunked_data()
test_https_without_ssl_skips_verification()
test_https_wraps_with_the_given_ssl_context()
test_redirect_keeps_the_ssl_context()
test_errors_close_the_socket()
test_ssl_refuses_an_http_url()
test_ssl_refuses_a_redirect_to_http()
test_overwrite_get_headers()
test_overwrite_post_json_headers()
test_overwrite_post_chunked_data_headers()
Expand Down
Loading