From aa9326dda173888cd0622f2f606a39d1874c033e Mon Sep 17 00:00:00 2001 From: Joshua Wilson Date: Tue, 17 Nov 2026 14:11:24 +1300 Subject: [PATCH] requests: Add ssl argument to pass a verifying SSLContext. requests builds its own tls.SSLContext with verify_mode CERT_NONE and offers no way to change it, so HTTPS requests accept any certificate. Add an optional ssl argument to request() and the method helpers that takes an SSLContext, as umqtt.simple does. When given, the socket is wrapped with it and the host passed as server_hostname, and redirects to https keep using it. An http URL, or a redirect to one, raises ValueError instead of sending the request unencrypted. When ssl is None, requests are sent as before. Close the socket on any exception, not only OSError: a certificate that fails verification raises ValueError, and running out of memory raises MemoryError. This also stops the socket leaking on other errors, such as a non-redirect status with a Location header. Co-Authored-By: Claude Signed-off-by: Joshua Wilson --- python-ecosys/requests/README.md | 39 +++++- python-ecosys/requests/manifest.py | 2 +- python-ecosys/requests/requests/__init__.py | 20 ++- python-ecosys/requests/test_requests.py | 147 ++++++++++++++++++++ 4 files changed, 200 insertions(+), 8 deletions(-) diff --git a/python-ecosys/requests/README.md b/python-ecosys/requests/README.md index 7d888e240..c845b5e6f 100644 --- a/python-ecosys/requests/README.md +++ b/python-ecosys/requests/README.md @@ -10,14 +10,49 @@ bodies with Content-Length or Transfer-Encoding: chunked via streaming ### Limitations -* Certificate validation is not currently supported. +* Certificates are only verified when an ``ssl`` context set to ``CERT_REQUIRED`` + is passed (see below). * A dictionary passed as post data will not do automatic JSON or multipart-form encoding of post data (this can be done manually). * Compressed requests/responses are not currently supported. * File upload is not supported. * HTTP keep-alive connection reuse is not supported (Connection: close by default). +### Verifying certificates + +By default HTTPS requests accept any server certificate. To verify the +certificate chain and the hostname, pass an ``SSLContext`` whose +``verify_mode`` is ``CERT_REQUIRED`` as ``ssl``: + +```py +import ssl, requests + +ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) +ctx.verify_mode = ssl.CERT_REQUIRED +ctx.load_verify_locations(cafile="ca.der") +with requests.get("https://example.com", ssl=ctx) as r: + print(r.status_code) +``` + +* ``ca.der`` holds the root CA certificate the server's chain ends at: a + single DER certificate, or one or more in PEM. The server must send any + intermediate certificates. +* ``ssl.SSLContext`` starts with ``CERT_NONE``, and on mbedTLS ports + ``CERT_OPTIONAL`` doesn't verify either, so set ``CERT_REQUIRED``. +* The device's clock must be set, for example with ``ntptime``, or + certificates fail their validity check. +* A certificate that fails verification raises ``ValueError`` (``OSError`` + in rare cases), so catch both when retrying. +* With ``ssl`` given, an ``http://`` URL, or a redirect to one, raises + ``ValueError`` instead of sending the request unencrypted. Redirects to + ``https://`` are verified with the same context. +* Verification needs a port whose ``tls`` module uses mbedTLS. esp8266's + axTLS can't verify certificates. +* Firmware that freezes ``requests`` uses the frozen copy ahead of one + installed in ``/lib``. + ### Follow-up work -* TLS certificate verification (see micropython-lib issue #838). +* Verifying certificates by default, which needs a source of root + certificates on the device (see issue #838). * ``stream=True`` incremental body reads (see issue #777). diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index cbb311fbd..31535950e 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="1.1.1", pypi="requests") +metadata(version="1.2.0", pypi="requests") package("requests") diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index b0e479ead..3f626517e 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -93,6 +93,7 @@ def request( auth=None, timeout=None, parse_headers=True, + ssl=None, ): if headers is None: headers = {} @@ -124,6 +125,11 @@ def request( else: raise ValueError("Unsupported protocol: " + proto) + if ssl is not None and proto != "https:": + # A request that asked for a verified connection never goes out without + # one, including when a redirect points to http://. + raise ValueError("ssl needs https") + if "?" in host: host, _path = host.split("?", 1) path = "?" + _path + path @@ -153,8 +159,10 @@ def request( try: s.connect(ai[-1]) if proto == "https:": - context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) - context.verify_mode = tls.CERT_NONE + context = ssl + if context is None: + context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + context.verify_mode = tls.CERT_NONE s = context.wrap_socket(s, server_hostname=host) s.write(b"%s /%s HTTP/1.1\r\n" % (method, path)) @@ -245,7 +253,9 @@ def request( remaining = int(v) else: parse_headers(l, resp_d) - except OSError: + except: + # Close the socket whatever went wrong: a certificate that fails + # verification raises ValueError, and running out of memory MemoryError. s.close() raise @@ -254,9 +264,9 @@ def request( # Use the host specified in the redirect URL, as it may not be the same as the original URL. headers.pop("Host", None) if status in [301, 302, 303]: - return request("GET", redirect, None, None, headers, stream) + return request("GET", redirect, None, None, headers, stream, ssl=ssl) else: - return request(method, redirect, data, json, headers, stream) + return request(method, redirect, data, json, headers, stream, ssl=ssl) else: if chunked: resp = Response(BodyStream(s, -1)) diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index c12589c02..5e1879e41 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -42,6 +42,33 @@ def socket(af=AF_INET, type=SOCK_STREAM, proto=IPPROTO_TCP): sys.modules["socket"] = socket + + +class SSLContext: + def __init__(self, protocol=None): + self.protocol = protocol + self.verify_mode = None + self.wrapped = [] + + def wrap_socket(self, sock, server_hostname=None): + self.wrapped.append(server_hostname) + return sock + + +class tls: + PROTOCOL_TLS_CLIENT = 0 + CERT_NONE = 0 + CERT_REQUIRED = 2 + contexts = [] + + @staticmethod + def SSLContext(protocol): + context = SSLContext(protocol) + tls.contexts.append(context) + return context + + +sys.modules["tls"] = tls # ruff: noqa: E402 import requests @@ -296,6 +323,120 @@ def test_raw_readinto_content_length(): socket.socket = lambda *a, **k: Socket() +def test_https_without_ssl_skips_verification(): + tls.contexts.clear() + + requests.request("GET", "https://example.com") + + assert len(tls.contexts) == 1 + assert tls.contexts[0].verify_mode == tls.CERT_NONE + assert tls.contexts[0].wrapped == ["example.com"] + + +def test_https_wraps_with_the_given_ssl_context(): + tls.contexts.clear() + context = SSLContext() + context.verify_mode = tls.CERT_REQUIRED + + requests.get("https://example.com:8443/x", ssl=context) + + assert tls.contexts == [] + assert context.wrapped == ["example.com"] + assert context.verify_mode == tls.CERT_REQUIRED + + +def test_redirect_keeps_the_ssl_context(): + server = iter( + [ + b"HTTP/1.1 302 Found\r\nLocation: https://other.example.com/next\r\n\r\n", + SERVER_RESPONSE_200_OK, + ] + ) + socket.socket = lambda *a, **k: Socket(next(server)) + context = SSLContext() + + requests.get("https://example.com", ssl=context) + + assert context.wrapped == ["example.com", "other.example.com"] + + socket.socket = lambda *a, **k: Socket() + + +def test_errors_close_the_socket(): + class RaisingContext: + def __init__(self, exc): + self.exc = exc + + def wrap_socket(self, s, server_hostname=None): + raise self.exc + + # A failed verification raises ValueError; running out of memory, MemoryError; + # a value that isn't a context, AttributeError. + for ssl, exc_type in ( + (RaisingContext(ValueError("certificate verify failed")), ValueError), + (RaisingContext(MemoryError()), MemoryError), + (True, AttributeError), + ): + sock = Socket() + closed = [] + sock.close = lambda: closed.append(True) + socket.socket = lambda *a, **k: sock + + try: + requests.get("https://example.com", ssl=ssl) + except exc_type: + pass + else: + assert False, "expected %s" % exc_type + + assert closed == [True], exc_type + + socket.socket = lambda *a, **k: Socket() + + +def test_ssl_refuses_an_http_url(): + made = [] + socket.socket = lambda *a, **k: made.append(1) or Socket() + + try: + requests.get("http://example.com", ssl=SSLContext()) + except ValueError: + pass + else: + assert False, "expected ValueError" + + assert made == [] + + socket.socket = lambda *a, **k: Socket() + + +def test_ssl_refuses_a_redirect_to_http(): + first = Socket(b"HTTP/1.1 302 Found\r\nLocation: http://example.com/next\r\n\r\n") + closed = [] + first.close = lambda: closed.append(True) + made = [] + + def make(*a, **k): + made.append(1) + return first + + socket.socket = make + context = SSLContext() + + try: + requests.get("https://example.com", ssl=context) + except ValueError: + pass + else: + assert False, "expected ValueError" + + assert context.wrapped == ["example.com"] + assert closed == [True] + assert made == [1] + + socket.socket = lambda *a, **k: Socket() + + def test_redirect_absolute(): server = iter( [ @@ -379,6 +520,12 @@ def test_content_length_lowercase_header(): test_post_data_str_unicode() test_post_data_bytes() test_post_chunked_data() +test_https_without_ssl_skips_verification() +test_https_wraps_with_the_given_ssl_context() +test_redirect_keeps_the_ssl_context() +test_errors_close_the_socket() +test_ssl_refuses_an_http_url() +test_ssl_refuses_a_redirect_to_http() test_overwrite_get_headers() test_overwrite_post_json_headers() test_overwrite_post_chunked_data_headers()