Add support for synchronous TLS-in-TLS connections.

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+- Add support for synchronous **TLS-in-TLS** connections. (#732)
 - Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762)
 - Handle HTTP/1.1 half-closed connections gracefully. (#641)
 - Drop Python 3.7 support. (#727)

httpcore/_backends/base.py

@@ -1,6 +1,10 @@
 import ssl
 import time
 import typing
+from types import TracebackType
+
+if typing.TYPE_CHECKING: # pragma: no cover
+ from typing_extensions import Self
 
 SOCKET_OPTION = typing.Union[
  typing.Tuple[int, int, int],
@@ -16,6 +20,17 @@ def read(self, max_bytes: int, timeout: typing.Optional[float] = None) -> bytes:
  def write(self, buffer: bytes, timeout: typing.Optional[float] = None) -> None:
  raise NotImplementedError() # pragma: nocover
 
+ def __enter__(self) -> "Self":
+ return self
+
+ def __exit__(
+ self,
+ exc_type: typing.Optional[typing.Type[BaseException]] = None,
+ exc_value: typing.Optional[BaseException] = None,
+ traceback: typing.Optional[TracebackType] = None,
+ ) -> None:
+ self.close()
+
  def close(self) -> None:
  raise NotImplementedError() # pragma: nocover
 
@@ -65,6 +80,19 @@ async def write(
  ) -> None:
  raise NotImplementedError() # pragma: nocover
 
+ async def __aenter__(
+ self,
+ ) -> "Self":
+ return self
+
+ async def __aexit__(
+ self,
+ exc_type: typing.Optional[typing.Type[BaseException]] = None,
+ exc_value: typing.Optional[BaseException] = None,
+ traceback: typing.Optional[TracebackType] = None,
+ ) -> None:
+ await self.aclose()
+
  async def aclose(self) -> None:
  raise NotImplementedError() # pragma: nocover
 

httpcore/_backends/sync.py

@@ -2,6 +2,8 @@
 import ssl
 import sys
 import typing
+from functools import partial
+from time import perf_counter
 
 from .._exceptions import (
  ConnectError,
@@ -17,6 +19,117 @@
 from .base import SOCKET_OPTION, NetworkBackend, NetworkStream
 
 
+class SyncTLSStream(NetworkStream):
+ """
+ Because the standard `SSLContext.wrap_socket` method does
+ not work for `SSLSocket` objects, we need this class
+ to implement TLS stream using an underlying `SSLObject`
+ instance in order to support TLS on top of TLS.
+ """
+
+ # Defined in RFC 8449
+ TLS_RECORD_SIZE = 16384
+
+ def __init__(
+ self,
+ sock: socket.socket,
+ ssl_context: ssl.SSLContext,
+ server_hostname: typing.Optional[str] = None,
+ timeout: typing.Optional[float] = None,
+ ):
+ self._sock = sock
+ self._incoming = ssl.MemoryBIO()
+ self._outgoing = ssl.MemoryBIO()
+
+ self.ssl_obj = ssl_context.wrap_bio(
+ incoming=self._incoming,
+ outgoing=self._outgoing,
+ server_hostname=server_hostname,
+ )
+
+ self._perform_io(self.ssl_obj.do_handshake, timeout)
+
+ def _perform_io(
+ self,
+ func: typing.Callable[..., typing.Any],
+ timeout: typing.Optional[float],
+ ) -> typing.Any:
+ ret = None
+ timeout = timeout or None # Replaces `0` with `None`
+
+ while True:
+ errno = None
+ try:
+ ret = func()
+ except (ssl.SSLWantReadError, ssl.SSLWantWriteError) as e:
+ errno = e.errno
+
+ if timeout is not None and timeout <= 0: # pragma: no cover
+ raise socket.timeout()
+
+ self._sock.settimeout(timeout)
+ operation_start = perf_counter()
+ self._sock.sendall(self._outgoing.read())
+ # If the timeout is `None`, don't touch it.
+ timeout = timeout and timeout - (perf_counter() - operation_start)
+
+ if errno == ssl.SSL_ERROR_WANT_READ:
+ if timeout is not None and timeout <= 0: # pragma: no cover
+ raise socket.timeout()
+
+ self._sock.settimeout(timeout)
+ operation_start = perf_counter()
+ buf = self._sock.recv(self.TLS_RECORD_SIZE)
+
+ # If the timeout is `None`, don't touch it.
+ timeout = timeout and timeout - (perf_counter() - operation_start)
+
+ if buf:
+ self._incoming.write(buf)
+ else:
+ self._incoming.write_eof() # pragma: no cover
+ if errno is None:
+ return ret
+
+ def read(self, max_bytes: int, timeout: typing.Optional[float] = None) -> bytes:
+ exc_map: ExceptionMapping = {socket.timeout: ReadTimeout, OSError: ReadError}
+ with map_exceptions(exc_map):
+ return typing.cast(
+ bytes, self._perform_io(partial(self.ssl_obj.read, max_bytes), timeout)
+ )
+
+ def write(self, buffer: bytes, timeout: typing.Optional[float] = None) -> None:
+ exc_map: ExceptionMapping = {socket.timeout: WriteTimeout, OSError: WriteError}
+ with map_exceptions(exc_map):
+ while buffer:
+ nsent = self._perform_io(partial(self.ssl_obj.write, buffer), timeout)
+ buffer = buffer[nsent:]
+
+ def close(self) -> None:
+ self._sock.close()
+
+ def start_tls(
+ self,
+ ssl_context: ssl.SSLContext,
+ server_hostname: typing.Optional[str] = None,
+ timeout: typing.Optional[float] = None,
+ ) -> "NetworkStream":
+ raise NotImplementedError() # pragma: no cover
+
+ def get_extra_info(self, info: str) -> typing.Any: # pragma: no cover
+ if info == "ssl_object":
+ return self.ssl_obj
+ if info == "client_addr":
+ return self._sock.getsockname()
+ if info == "server_addr":
+ return self._sock.getpeername()
+ if info == "socket":
+ return self._sock
+ if info == "is_readable":
+ return is_socket_readable(self._sock)
+ return None
+
+
 class SyncStream(NetworkStream):
  def __init__(self, sock: socket.socket) -> None:
  self._sock = sock
@@ -53,10 +166,18 @@ def start_tls(
  }
  with map_exceptions(exc_map):
  try:
- self._sock.settimeout(timeout)
- sock = ssl_context.wrap_socket(
- self._sock, server_hostname=server_hostname
- )
+ if isinstance(self._sock, ssl.SSLSocket):
+ # If the underlying socket has already been upgraded
+ # to the TLS layer (i.e. is an instance of SSLSocket),
+ # we want to use another stream object that supports TLS-in-TLS.
+ return SyncTLSStream(
+ self._sock, ssl_context, server_hostname, timeout
+ )
+ else:
+ self._sock.settimeout(timeout)
+ sock = ssl_context.wrap_socket(
+ self._sock, server_hostname=server_hostname
+ )
  except Exception as exc: # pragma: nocover
  self.close()
  raise exc

requirements.txt

@@ -21,6 +21,8 @@ black==23.3.0
 coverage[toml]==7.2.7
 ruff==0.0.277
 mypy==1.4.1
+trustme==1.0.0
+typing-extensions==4.6.3
 trio-typing==0.8.0
 types-certifi==2021.10.8.3
 pytest==7.4.0

tests/_backends/test_anyio.py

@@ -0,0 +1,145 @@
+import pytest
+
+import httpcore
+
+READ_TIMEOUT = 2
+WRITE_TIMEOUT = 2
+CONNECT_TIMEOUT = 2
+
+
+@pytest.mark.anyio
+async def test_connect_without_tls(tcp_server):
+ backend = httpcore.AnyIOBackend()
+ stream = await backend.connect_tcp(
+ tcp_server.host, tcp_server.port, timeout=CONNECT_TIMEOUT
+ )
+ await stream.aclose()
+
+
+@pytest.mark.anyio
+async def test_write_without_tls(tcp_server):
+ backend = httpcore.AnyIOBackend()
+ stream = await backend.connect_tcp(
+ tcp_server.host, tcp_server.port, timeout=CONNECT_TIMEOUT
+ )
+ async with stream:
+ await stream.write(b"ping", timeout=WRITE_TIMEOUT)
+
+
+@pytest.mark.anyio
+async def test_read_without_tls(tcp_server):
+ backend = httpcore.AnyIOBackend()
+ stream = await backend.connect_tcp(
+ tcp_server.host, tcp_server.port, timeout=CONNECT_TIMEOUT
+ )
+ async with stream:
+ await stream.write(b"ping", timeout=WRITE_TIMEOUT)
+ await stream.read(1024, timeout=READ_TIMEOUT)
+
+
+@pytest.mark.anyio
+async def test_connect_with_tls(tls_server, client_context):
+ backend = httpcore.AnyIOBackend()
+ stream = await backend.connect_tcp(
+ tls_server.host, tls_server.port, timeout=CONNECT_TIMEOUT
+ )
+ async with stream:
+ tls_stream = await stream.start_tls(
+ ssl_context=client_context, timeout=CONNECT_TIMEOUT
+ )
+ await tls_stream.aclose()
+
+
+@pytest.mark.anyio
+async def test_write_with_tls(tls_server, client_context):
+ backend = httpcore.AnyIOBackend()
+ stream = await backend.connect_tcp(
+ tls_server.host, tls_server.port, timeout=CONNECT_TIMEOUT
+ )
+ async with stream:
+ tls_stream = await stream.start_tls(
+ ssl_context=client_context, timeout=CONNECT_TIMEOUT
+ )
+ async with tls_stream:
+ await tls_stream.write(b"ping", timeout=WRITE_TIMEOUT)
+
+
+@pytest.mark.anyio
+async def test_read_with_tls(tls_server, client_context):
+ backend = httpcore.AnyIOBackend()
+ stream = await backend.connect_tcp(
+ tls_server.host, tls_server.port, timeout=CONNECT_TIMEOUT
+ )
+ async with stream:
+ tls_stream = await stream.start_tls(
+ ssl_context=client_context, timeout=CONNECT_TIMEOUT
+ )
+ async with tls_stream:
+ await tls_stream.write(b"ping", timeout=WRITE_TIMEOUT)
+ await tls_stream.read(1024, timeout=READ_TIMEOUT)
+
+
+@pytest.mark.anyio
+async def test_connect_with_tls_in_tls(tls_in_tls_server, client_context):
+ backend = httpcore.AnyIOBackend()
+ stream = await backend.connect_tcp(
+ tls_in_tls_server.host, tls_in_tls_server.port, timeout=CONNECT_TIMEOUT
+ )
+ async with stream:
+ tls_stream = await stream.start_tls(
+ ssl_context=client_context,
+ server_hostname="localhost",
+ timeout=CONNECT_TIMEOUT,
+ )
+ async with tls_stream:
+ tls_in_tls_stream = await tls_stream.start_tls(
+ ssl_context=client_context,
+ server_hostname="localhost",
+ timeout=CONNECT_TIMEOUT,
+ )
+ await tls_in_tls_stream.aclose()
+
+
+@pytest.mark.anyio
+async def test_write_with_tls_in_tls(tls_in_tls_server, client_context):
+ backend = httpcore.AnyIOBackend()
+ stream = await backend.connect_tcp(
+ tls_in_tls_server.host, tls_in_tls_server.port, timeout=CONNECT_TIMEOUT
+ )
+ async with stream:
+ tls_stream = await stream.start_tls(
+ ssl_context=client_context,
+ server_hostname="localhost",
+ timeout=CONNECT_TIMEOUT,
+ )
+ async with tls_stream:
+ tls_in_tls_stream = await tls_stream.start_tls(
+ ssl_context=client_context,
+ server_hostname="localhost",
+ timeout=CONNECT_TIMEOUT,
+ )
+ async with tls_in_tls_stream:
+ await tls_in_tls_stream.write(b"ping", timeout=WRITE_TIMEOUT)
+
+
+@pytest.mark.anyio
+async def test_read_with_tls_in_tls(tls_in_tls_server, client_context):
+ backend = httpcore.AnyIOBackend()
+ stream = await backend.connect_tcp(
+ tls_in_tls_server.host, tls_in_tls_server.port, timeout=CONNECT_TIMEOUT
+ )
+ async with stream:
+ tls_stream = await stream.start_tls(
+ ssl_context=client_context,
+ server_hostname="localhost",
+ timeout=CONNECT_TIMEOUT,
+ )
+ async with tls_stream:
+ tls_in_tls_stream = await tls_stream.start_tls(
+ ssl_context=client_context,
+ server_hostname="localhost",
+ timeout=CONNECT_TIMEOUT,
+ )
+ async with tls_in_tls_stream:
+ await tls_in_tls_stream.write(b"ping", timeout=WRITE_TIMEOUT)
+ await tls_in_tls_stream.read(1024, timeout=READ_TIMEOUT)