From f68f2ae6b8d40572a92942139ca4b32d7a91704b Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 7 Jun 2023 15:06:18 -0500 Subject: [PATCH 1/2] Support urllib3 1.26.x and 2.x This changes the assert_fingerprint hack to more directly tell urllib3 that we'll assert the fingerprint ourselves to add support for pinning root certificates, not only the leaves. --- .../_node/_urllib3_chain_certs.py | 26 ++++++++++++++++--- setup.py | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/elastic_transport/_node/_urllib3_chain_certs.py b/elastic_transport/_node/_urllib3_chain_certs.py index 0d99896..e01f6a9 100644 --- a/elastic_transport/_node/_urllib3_chain_certs.py +++ b/elastic_transport/_node/_urllib3_chain_certs.py @@ -36,7 +36,17 @@ __all__ = ["HTTPSConnectionPool"] +class HTTPSConnection(urllib3.connection.HTTPSConnection): + def connect(self) -> None: + super().connect() + # Hack to prevent a warning within HTTPSConnectionPool._validate_conn() + if self._elastic_assert_fingerprint: + self.is_verified = True + + class HTTPSConnectionPool(urllib3.HTTPSConnectionPool): + ConnectionCls = HTTPSConnection + """HTTPSConnectionPool implementation which supports ``assert_fingerprint`` on certificates within the chain instead of only the leaf cert using private APIs in CPython 3.10+ @@ -60,13 +70,21 @@ def __init__( f", should be one of '{valid_lengths}'" ) - if assert_fingerprint: - # Falsey but not None. This is a hack to skip fingerprinting by urllib3 - # but still set 'is_verified=True' within HTTPSConnectionPool._validate_conn() - kwargs["assert_fingerprint"] = "" + if self._elastic_assert_fingerprint: + # Skip fingerprinting by urllib3 as we'll do it ourselves + kwargs["assert_fingerprint"] = None super().__init__(*args, **kwargs) + def _new_conn(self) -> HTTPSConnection: + """ + Return a fresh :class:`urllib3.connection.HTTPSConnection`. + """ + conn = super()._new_conn() + # Tell our custom connection if we'll assert fingerprint ourselves + conn._elastic_assert_fingerprint = self._elastic_assert_fingerprint + return conn + def _validate_conn(self, conn: urllib3.connection.HTTPSConnection) -> None: """ Called right before a request is made, after the socket is created. diff --git a/setup.py b/setup.py index 6e7a490..f7efc36 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ package_data={"elastic_transport": ["py.typed"]}, packages=packages, install_requires=[ - "urllib3>=1.26.2, <2", + "urllib3>=1.26.2, <3", "certifi", "dataclasses; python_version<'3.7'", ], From 08bd71367ffdce0db7d6cf217753c99b16bfd20d Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 27 Sep 2023 14:45:07 +0400 Subject: [PATCH 2/2] Fix mypy --- elastic_transport/_node/_http_urllib3.py | 11 ++++++++--- elastic_transport/_node/_urllib3_chain_certs.py | 14 +++++++++----- noxfile.py | 7 +++++-- setup.py | 1 + 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/elastic_transport/_node/_http_urllib3.py b/elastic_transport/_node/_http_urllib3.py index 565d9d5..c1de4a1 100644 --- a/elastic_transport/_node/_http_urllib3.py +++ b/elastic_transport/_node/_http_urllib3.py @@ -21,6 +21,11 @@ import warnings from typing import Any, Dict, Optional, Union +try: + from importlib import metadata +except ImportError: + import importlib_metadata as metadata # type: ignore[import,no-redef] + import urllib3 from urllib3.exceptions import ConnectTimeoutError, NewConnectionError, ReadTimeoutError from urllib3.util.retry import Retry @@ -47,7 +52,7 @@ class Urllib3HttpNode(BaseNode): """Default synchronous node class using the ``urllib3`` library via HTTP""" - _CLIENT_META_HTTP_CLIENT = ("ur", client_meta_version(urllib3.__version__)) + _CLIENT_META_HTTP_CLIENT = ("ur", client_meta_version(metadata.version("urllib3"))) def __init__(self, config: NodeConfig): super().__init__(config) @@ -159,13 +164,13 @@ def perform_request( else: body_to_send = None - response = self.pool.urlopen( # type: ignore[no-untyped-call] + response = self.pool.urlopen( method, target, body=body_to_send, retries=Retry(False), headers=request_headers, - **kw, + **kw, # type: ignore[arg-type] ) response_headers = HttpHeaders(response.headers) data = response.data diff --git a/elastic_transport/_node/_urllib3_chain_certs.py b/elastic_transport/_node/_urllib3_chain_certs.py index e01f6a9..30790eb 100644 --- a/elastic_transport/_node/_urllib3_chain_certs.py +++ b/elastic_transport/_node/_urllib3_chain_certs.py @@ -37,6 +37,10 @@ class HTTPSConnection(urllib3.connection.HTTPSConnection): + def __init__(self, *args: Any, **kwargs: Any) -> None: + self._elastic_assert_fingerprint: Optional[str] = None + super().__init__(*args, **kwargs) + def connect(self) -> None: super().connect() # Hack to prevent a warning within HTTPSConnectionPool._validate_conn() @@ -80,16 +84,16 @@ def _new_conn(self) -> HTTPSConnection: """ Return a fresh :class:`urllib3.connection.HTTPSConnection`. """ - conn = super()._new_conn() + conn: HTTPSConnection = super()._new_conn() # type: ignore[assignment] # Tell our custom connection if we'll assert fingerprint ourselves conn._elastic_assert_fingerprint = self._elastic_assert_fingerprint return conn - def _validate_conn(self, conn: urllib3.connection.HTTPSConnection) -> None: + def _validate_conn(self, conn: HTTPSConnection) -> None: # type: ignore[override] """ Called right before a request is made, after the socket is created. """ - super(HTTPSConnectionPool, self)._validate_conn(conn) # type: ignore[misc] + super(HTTPSConnectionPool, self)._validate_conn(conn) if self._elastic_assert_fingerprint: hash_func = _HASHES_BY_LENGTH[len(self._elastic_assert_fingerprint)] @@ -107,7 +111,7 @@ def _validate_conn(self, conn: urllib3.connection.HTTPSConnection) -> None: # See: https://github.com/python/cpython/pull/25467 fingerprints = [ hash_func(cert.public_bytes(_ENCODING_DER)).digest() - for cert in conn.sock._sslobj.get_verified_chain() + for cert in conn.sock._sslobj.get_verified_chain() # type: ignore[union-attr] ] except RERAISE_EXCEPTIONS: # pragma: nocover raise @@ -118,7 +122,7 @@ def _validate_conn(self, conn: urllib3.connection.HTTPSConnection) -> None: # Only add the peercert in front of the chain if it's not there for some reason. # This is to make sure old behavior of 'ssl_assert_fingerprint' still works. - peercert_fingerprint = hash_func(conn.sock.getpeercert(True)).digest() + peercert_fingerprint = hash_func(conn.sock.getpeercert(True)).digest() # type: ignore[union-attr] if peercert_fingerprint not in fingerprints: # pragma: nocover fingerprints.insert(0, peercert_fingerprint) diff --git a/noxfile.py b/noxfile.py index f54a3b1..8ad913a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -43,11 +43,14 @@ def lint(session): "flake8", "black~=23.0", "isort", - "mypy==1.0.1", - "types-urllib3", + "mypy==1.5.1", "types-requests", "types-certifi", ) + # https://github.com/python/typeshed/issues/10786 + session.run( + "python", "-m", "pip", "uninstall", "--yes", "types-urllib3", silent=True + ) session.install(".[develop]") session.run("black", "--check", "--target-version=py36", *SOURCE_FILES) session.run("isort", "--check", *SOURCE_FILES) diff --git a/setup.py b/setup.py index f7efc36..62e1e5a 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ "urllib3>=1.26.2, <3", "certifi", "dataclasses; python_version<'3.7'", + "importlib-metadata; python_version<'3.8'", ], python_requires=">=3.6", extras_require={