feat: Add HTTPX-based HTTP client

README.md

@@ -57,6 +57,15 @@
  uv add "apify-client[brotli]"
  ```
 
+ [Impit](https://github.com/apify/impit) is the default HTTP client and is installed automatically. To use the
+ built-in [HTTPX](https://www.python-httpx.org/) client instead, install its optional extra:
+
+ ```bash
+ pip install "apify-client[httpx]"
+ # or
+ uv add "apify-client[httpx]"
+ ```
+
 - From [conda-forge](https://anaconda.org/conda-forge/apify-client), it can be installed with [conda](https://docs.conda.io/en/latest/):
 
  ```bash
@@ -124,7 +133,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r
 - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)).
 - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)).
 - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)).
-- **Pluggable HTTP layer** — swap the default [Impit](https://github.com/apify/impit)-based HTTP client for `httpx`, `requests`, `aiohttp`, or any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)).
+- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://www.python-httpx.org/) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)).
 - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)).
 - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)).
 

pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
 
 [project.optional-dependencies]
 brotli = ["brotli>=1.0.9"]
+httpx = ["httpx>=0.27.0,<1.0.0"]
 
 [project.urls]
 "Apify Homepage" = "https://apify.com"

src/apify_client/http_clients/__init__.py

@@ -1,10 +1,35 @@
+from apify_client._utils.try_import import install_import_hook as _install_import_hook
+from apify_client._utils.try_import import try_import as _try_import
 from apify_client.http_clients._base import HttpClient, HttpClientAsync, HttpResponse
 from apify_client.http_clients._impit import ImpitHttpClient, ImpitHttpClientAsync
 
-__all__ = [
- 'HttpClient',
- 'HttpClientAsync',
- 'HttpResponse',
- 'ImpitHttpClient',
- 'ImpitHttpClientAsync',
-]
+_install_import_hook(__name__)
+
+# `httpx` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX clients
+# without the extra installed raises a clear ImportError instead of failing at package import time.
+with _try_import(
+ __name__,
+ 'HttpxHttpClient',
+ 'HttpxHttpClientAsync',
+ dependency_name='httpx',
+) as _httpx_import:
+ from apify_client.http_clients._httpx import HttpxHttpClient, HttpxHttpClientAsync
+
+if _httpx_import.available:
+ __all__ = [
+ 'HttpClient',
+ 'HttpClientAsync',
+ 'HttpResponse',
+ 'HttpxHttpClient',
+ 'HttpxHttpClientAsync',
+ 'ImpitHttpClient',
+ 'ImpitHttpClientAsync',
+ ]
+else:
+ __all__ = [
+ 'HttpClient',
+ 'HttpClientAsync',
+ 'HttpResponse',
+ 'ImpitHttpClient',
+ 'ImpitHttpClientAsync',
+ ]

src/apify_client/http_clients/_httpx.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import httpx
+from typing_extensions import override
+
+from apify_client._consts import (
+ DEFAULT_MAX_RETRIES,
+ DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
+ DEFAULT_TIMEOUT_LONG,
+ DEFAULT_TIMEOUT_MAX,
+ DEFAULT_TIMEOUT_MEDIUM,
+ DEFAULT_TIMEOUT_SHORT,
+)
+from apify_client._docs import docs_group
+from apify_client.http_clients._base import HttpClient, HttpClientAsync
+
+if TYPE_CHECKING:
+ from datetime import timedelta
+
+ from apify_client._statistics import ClientStatistics
+ from apify_client.http_compressors._base import HttpCompressor
+
+
+_PERMANENT_ERRORS = (
+ # A request HTTPX rejects before sending it, e.g. one carrying an invalid header value.
+ httpx.LocalProtocolError,
+ # A URL scheme HTTPX refuses to speak, which repeating the request cannot change.
+ httpx.UnsupportedProtocol,
+ # An over-long redirect chain is a routing loop, which repeating the request cannot break.
+ httpx.TooManyRedirects,
+ # Only `Response.raise_for_status()` raises this, and the client never calls it - the shared pipeline decides on
+ # status codes from the response itself.
+ httpx.HTTPStatusError,
+)
+"""HTTPX errors that a retry cannot fix. Everything else in the `httpx.HTTPError` tree counts as transient."""
+
+
+@docs_group('HTTP clients')
+class HttpxHttpClient(HttpClient):
+ """Synchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/).
+
+ This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited
+ (HTTP 429) and server error (HTTP 5xx) responses.
+
+ Requires the `httpx` extra: `pip install "apify-client[httpx]"`.
+ """
+
+ def __init__(
+ self,
+ *,
+ token: str | None = None,
+ timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
+ timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
+ timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
+ timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
+ max_retries: int = DEFAULT_MAX_RETRIES,
+ min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
+ statistics: ClientStatistics | None = None,
+ headers: dict[str, str] | None = None,
+ http_compressor: HttpCompressor | None = None,
+ ) -> None:
+ """Initialize the HTTPX-based synchronous HTTP client.
+
+ Args:
+ token: Apify API token for authentication.
+ timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
+ timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
+ timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
+ timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
+ max_retries: Maximum number of retry attempts for failed requests.
+ min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
+ statistics: Statistics tracker for API calls. Created automatically if not provided.
+ headers: Additional HTTP headers to include in all requests.
+ http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
+ """
+ super().__init__(
+ token=token,
+ timeout_short=timeout_short,
+ timeout_medium=timeout_medium,
+ timeout_long=timeout_long,
+ timeout_max=timeout_max,
+ max_retries=max_retries,
+ min_delay_between_retries=min_delay_between_retries,
+ statistics=statistics,
+ headers=headers,
+ http_compressor=http_compressor,
+ )
+
+ self._httpx_client = httpx.Client(
+ follow_redirects=True,
+ event_hooks={'response': [self._clear_response_cookies]},
+ )
+
+ @override
+ def is_timeout_error(self, exc: Exception) -> bool:
+ return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException)
+
+ @override
+ def is_retryable_transport_error(self, exc: Exception) -> bool:
+ # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in
+ # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures
+ # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the
+ # response status code, not here.
+ return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS)
+
+ @override
+ def close(self) -> None:
+ """Close the underlying HTTPX connection pool."""
+ self._httpx_client.close()
+
+ def _clear_response_cookies(self, _response: httpx.Response) -> None:
+ """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests."""
+ self._httpx_client.cookies.clear()
+
+ @override
+ def send_request(
+ self,
+ *,
+ method: str,
+ url: str,
+ headers: dict[str, str],
+ content: bytes | None,
+ timeout: float | None,
+ stream: bool,
+ ) -> httpx.Response:
+ request = self._httpx_client.build_request(
+ method=method,
+ url=url,
+ headers=headers,
+ content=content,
+ timeout=timeout,
+ )
+ _restore_explicit_cookie_header(request, headers)
+ return self._httpx_client.send(request, stream=stream)
+
+
+@docs_group('HTTP clients')
+class HttpxHttpClientAsync(HttpClientAsync):
+ """Asynchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/).
+
+ This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited
+ (HTTP 429) and server error (HTTP 5xx) responses.
+
+ Requires the `httpx` extra: `pip install "apify-client[httpx]"`.
+ """
+
+ def __init__(
+ self,
+ *,
+ token: str | None = None,
+ timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
+ timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
+ timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
+ timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
+ max_retries: int = DEFAULT_MAX_RETRIES,
+ min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
+ statistics: ClientStatistics | None = None,
+ headers: dict[str, str] | None = None,
+ http_compressor: HttpCompressor | None = None,
+ ) -> None:
+ """Initialize the HTTPX-based asynchronous HTTP client.
+
+ Args:
+ token: Apify API token for authentication.
+ timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
+ timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
+ timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
+ timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
+ max_retries: Maximum number of retry attempts for failed requests.
+ min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
+ statistics: Statistics tracker for API calls. Created automatically if not provided.
+ headers: Additional HTTP headers to include in all requests.
+ http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
+ """
+ super().__init__(
+ token=token,
+ timeout_short=timeout_short,
+ timeout_medium=timeout_medium,
+ timeout_long=timeout_long,
+ timeout_max=timeout_max,
+ max_retries=max_retries,
+ min_delay_between_retries=min_delay_between_retries,
+ statistics=statistics,
+ headers=headers,
+ http_compressor=http_compressor,
+ )
+
+ self._httpx_async_client = httpx.AsyncClient(
+ follow_redirects=True,
+ event_hooks={'response': [self._clear_response_cookies]},
+ )
+
+ @override
+ def is_timeout_error(self, exc: Exception) -> bool:
+ return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException)
+
+ @override
+ def is_retryable_transport_error(self, exc: Exception) -> bool:
+ # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in
+ # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures
+ # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the
+ # response status code, not here.
+ return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS)
+
+ @override
+ async def aclose(self) -> None:
+ """Close the underlying asynchronous HTTPX connection pool."""
+ await self._httpx_async_client.aclose()
+
+ async def _clear_response_cookies(self, _response: httpx.Response) -> None:
+ """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests."""
+ self._httpx_async_client.cookies.clear()
+
+ @override
+ async def send_request(
+ self,
+ *,
+ method: str,
+ url: str,
+ headers: dict[str, str],
+ content: bytes | None,
+ timeout: float | None,
+ stream: bool,
+ ) -> httpx.Response:
+ request = self._httpx_async_client.build_request(
+ method=method,
+ url=url,
+ headers=headers,
+ content=content,
+ timeout=timeout,
+ )
+ _restore_explicit_cookie_header(request, headers)
+ return await self._httpx_async_client.send(request, stream=stream)
+
+
+def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None:
+ """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar."""
+ explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None)
+ if explicit_cookie is None:
+ request.headers.pop('cookie', None)
+ else:
+ request.headers['cookie'] = explicit_cookie