diff --git a/python-ecosys/requests/README.md b/python-ecosys/requests/README.md index 7d888e240..c845b5e6f 100644 --- a/python-ecosys/requests/README.md +++ b/python-ecosys/requests/README.md @@ -10,14 +10,49 @@ bodies with Content-Length or Transfer-Encoding: chunked via streaming ### Limitations -* Certificate validation is not currently supported. +* Certificates are only verified when an ``ssl`` context set to ``CERT_REQUIRED`` + is passed (see below). * A dictionary passed as post data will not do automatic JSON or multipart-form encoding of post data (this can be done manually). * Compressed requests/responses are not currently supported. * File upload is not supported. * HTTP keep-alive connection reuse is not supported (Connection: close by default). +### Verifying certificates + +By default HTTPS requests accept any server certificate. To verify the +certificate chain and the hostname, pass an ``SSLContext`` whose +``verify_mode`` is ``CERT_REQUIRED`` as ``ssl``: + +```py +import ssl, requests + +ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) +ctx.verify_mode = ssl.CERT_REQUIRED +ctx.load_verify_locations(cafile="ca.der") +with requests.get("https://example.com", ssl=ctx) as r: + print(r.status_code) +``` + +* ``ca.der`` holds the root CA certificate the server's chain ends at: a + single DER certificate, or one or more in PEM. The server must send any + intermediate certificates. +* ``ssl.SSLContext`` starts with ``CERT_NONE``, and on mbedTLS ports + ``CERT_OPTIONAL`` doesn't verify either, so set ``CERT_REQUIRED``. +* The device's clock must be set, for example with ``ntptime``, or + certificates fail their validity check. +* A certificate that fails verification raises ``ValueError`` (``OSError`` + in rare cases), so catch both when retrying. +* With ``ssl`` given, an ``http://`` URL, or a redirect to one, raises + ``ValueError`` instead of sending the request unencrypted. Redirects to + ``https://`` are verified with the same context. +* Verification needs a port whose ``tls`` module uses mbedTLS. esp8266's + axTLS can't verify certificates. +* Firmware that freezes ``requests`` uses the frozen copy ahead of one + installed in ``/lib``. + ### Follow-up work -* TLS certificate verification (see micropython-lib issue #838). +* Verifying certificates by default, which needs a source of root + certificates on the device (see issue #838). * ``stream=True`` incremental body reads (see issue #777). diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index cbb311fbd..31535950e 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="1.1.1", pypi="requests") +metadata(version="1.2.0", pypi="requests") package("requests") diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index b0e479ead..3f626517e 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -93,6 +93,7 @@ def request( auth=None, timeout=None, parse_headers=True, + ssl=None, ): if headers is None: headers = {} @@ -124,6 +125,11 @@ def request( else: raise ValueError("Unsupported protocol: " + proto) + if ssl is not None and proto != "https:": + # A request that asked for a verified connection never goes out without + # one, including when a redirect points to http://. + raise ValueError("ssl needs https") + if "?" in host: host, _path = host.split("?", 1) path = "?" + _path + path @@ -153,8 +159,10 @@ def request( try: s.connect(ai[-1]) if proto == "https:": - context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) - context.verify_mode = tls.CERT_NONE + context = ssl + if context is None: + context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + context.verify_mode = tls.CERT_NONE s = context.wrap_socket(s, server_hostname=host) s.write(b"%s /%s HTTP/1.1\r\n" % (method, path)) @@ -245,7 +253,9 @@ def request( remaining = int(v) else: parse_headers(l, resp_d) - except OSError: + except: + # Close the socket whatever went wrong: a certificate that fails + # verification raises ValueError, and running out of memory MemoryError. s.close() raise @@ -254,9 +264,9 @@ def request( # Use the host specified in the redirect URL, as it may not be the same as the original URL. headers.pop("Host", None) if status in [301, 302, 303]: - return request("GET", redirect, None, None, headers, stream) + return request("GET", redirect, None, None, headers, stream, ssl=ssl) else: - return request(method, redirect, data, json, headers, stream) + return request(method, redirect, data, json, headers, stream, ssl=ssl) else: if chunked: resp = Response(BodyStream(s, -1)) diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index c12589c02..5e1879e41 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -42,6 +42,33 @@ def socket(af=AF_INET, type=SOCK_STREAM, proto=IPPROTO_TCP): sys.modules["socket"] = socket + + +class SSLContext: + def __init__(self, protocol=None): + self.protocol = protocol + self.verify_mode = None + self.wrapped = [] + + def wrap_socket(self, sock, server_hostname=None): + self.wrapped.append(server_hostname) + return sock + + +class tls: + PROTOCOL_TLS_CLIENT = 0 + CERT_NONE = 0 + CERT_REQUIRED = 2 + contexts = [] + + @staticmethod + def SSLContext(protocol): + context = SSLContext(protocol) + tls.contexts.append(context) + return context + + +sys.modules["tls"] = tls # ruff: noqa: E402 import requests @@ -296,6 +323,120 @@ def test_raw_readinto_content_length(): socket.socket = lambda *a, **k: Socket() +def test_https_without_ssl_skips_verification(): + tls.contexts.clear() + + requests.request("GET", "https://example.com") + + assert len(tls.contexts) == 1 + assert tls.contexts[0].verify_mode == tls.CERT_NONE + assert tls.contexts[0].wrapped == ["example.com"] + + +def test_https_wraps_with_the_given_ssl_context(): + tls.contexts.clear() + context = SSLContext() + context.verify_mode = tls.CERT_REQUIRED + + requests.get("https://example.com:8443/x", ssl=context) + + assert tls.contexts == [] + assert context.wrapped == ["example.com"] + assert context.verify_mode == tls.CERT_REQUIRED + + +def test_redirect_keeps_the_ssl_context(): + server = iter( + [ + b"HTTP/1.1 302 Found\r\nLocation: https://other.example.com/next\r\n\r\n", + SERVER_RESPONSE_200_OK, + ] + ) + socket.socket = lambda *a, **k: Socket(next(server)) + context = SSLContext() + + requests.get("https://example.com", ssl=context) + + assert context.wrapped == ["example.com", "other.example.com"] + + socket.socket = lambda *a, **k: Socket() + + +def test_errors_close_the_socket(): + class RaisingContext: + def __init__(self, exc): + self.exc = exc + + def wrap_socket(self, s, server_hostname=None): + raise self.exc + + # A failed verification raises ValueError; running out of memory, MemoryError; + # a value that isn't a context, AttributeError. + for ssl, exc_type in ( + (RaisingContext(ValueError("certificate verify failed")), ValueError), + (RaisingContext(MemoryError()), MemoryError), + (True, AttributeError), + ): + sock = Socket() + closed = [] + sock.close = lambda: closed.append(True) + socket.socket = lambda *a, **k: sock + + try: + requests.get("https://example.com", ssl=ssl) + except exc_type: + pass + else: + assert False, "expected %s" % exc_type + + assert closed == [True], exc_type + + socket.socket = lambda *a, **k: Socket() + + +def test_ssl_refuses_an_http_url(): + made = [] + socket.socket = lambda *a, **k: made.append(1) or Socket() + + try: + requests.get("http://example.com", ssl=SSLContext()) + except ValueError: + pass + else: + assert False, "expected ValueError" + + assert made == [] + + socket.socket = lambda *a, **k: Socket() + + +def test_ssl_refuses_a_redirect_to_http(): + first = Socket(b"HTTP/1.1 302 Found\r\nLocation: http://example.com/next\r\n\r\n") + closed = [] + first.close = lambda: closed.append(True) + made = [] + + def make(*a, **k): + made.append(1) + return first + + socket.socket = make + context = SSLContext() + + try: + requests.get("https://example.com", ssl=context) + except ValueError: + pass + else: + assert False, "expected ValueError" + + assert context.wrapped == ["example.com"] + assert closed == [True] + assert made == [1] + + socket.socket = lambda *a, **k: Socket() + + def test_redirect_absolute(): server = iter( [ @@ -379,6 +520,12 @@ def test_content_length_lowercase_header(): test_post_data_str_unicode() test_post_data_bytes() test_post_chunked_data() +test_https_without_ssl_skips_verification() +test_https_wraps_with_the_given_ssl_context() +test_redirect_keeps_the_ssl_context() +test_errors_close_the_socket() +test_ssl_refuses_an_http_url() +test_ssl_refuses_a_redirect_to_http() test_overwrite_get_headers() test_overwrite_post_json_headers() test_overwrite_post_chunked_data_headers()