Using httpx with httpcore newer than 0.14.5 results in `The connection pool was closed while XXX HTTP requests/responses were still in-flight.` . Need help figuring it out.

[teodoryantcheff]

Using httpx with httpcore newer than 0.14.5 results in RuntimeError: The connection pool was closed while XXX HTTP requests/responses were still in-flight. Anything newer and it breaks, but I need help figuring out why.

So, I have this code:

async def main():
 async with httpx.AsyncClient(proxies=proxyconfig.proxies) as client:
 .
 .
 .
 tasks = []
 for y, yms in year_makes:
 for make_slug in yms:
 tasks.append(get_models(year=y, make_slug=make_slug, client=client))

 # here tasks has ~2000 API call croutine objects in it

 print(f'making {len(tasks)} calls') 
 responses = await asyncio.gather(*tasks)
 .
 .

async def get_models(year: int | str, make_slug: tuple[str, str], *, client=None) -> list[list]:
 r = await api_call('Models', params={'year': year, 'make': make_slug[0]}, client=client)
 ret = [[year, make_slug[0], make_slug[1], i['model'], i['value']] for i in r.json()['items']]
 return ret

@backoff.on_exception(backoff.expo,
 (httpx.HTTPStatusError, # 429s
 httpx.RemoteProtocolError,
 httpx.ReadTimeout, # fsck the proxy
 httpx.ConnectTimeout),
 on_backoff=backoff_hdlr,
 max_time=240)
async def api_call(url: str, *, params: dict[str, ...] = None, client: httpx.AsyncClient = None) -> httpx.Response:
 params = params if params else {}
 if client:
 r = await client.get(URL_BASE + url, params=DFL_PARAMS | params)
 else:
 r = await httpx.AsyncClient().get(URL_BASE + url, params=DFL_PARAMS | params)
 r.raise_for_status()
 return r

and this combination of httpcore and httpx:

httpcore==0.14.5
httpx==0.23.1

on

(venv) teo@thinkpad:~/src/sgt--ymm-scraper$ python -V 
Python 3.11.0
(venv) teo@thinkpad:~/src/sgt--ymm-scraper$ lsb_release -a 
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu 22.04.1 LTS
Release:	22.04
Codename:	jammy
(venv) teo@thinkpad:~/src/sgt--ymm-scraper$ 

results in smooth sailing -- It all works as expected and life is good and all that.

While using

httpcore==0.16.1
httpx==0.23.1

Suffers a horrible death with:

unhandled exception during asyncio.run() shutdown
task: <Task finished name='Task-1464' coro=<get_models() done, defined at /home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py:84> exception=ValueError('list.remove(x): x not in list')>
Traceback (most recent call last):
 File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 110, in main
 responses = await asyncio.gather(*tasks)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 85, in get_models
 r = await api_call('Models', params={'year': year, 'make': make_slug[0]}, client=client)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/backoff/_async.py", line 151, in retry
 ret = await target(*args, **kwargs)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 65, in api_call
 r = await client.get(URL_BASE + url, params=DFL_PARAMS | params)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1757, in get
 return await self.request(
 ^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
 return await self.send(request, auth=auth, follow_redirects=follow_redirects)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1620, in send
 response = await self._send_handling_auth(
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
 response = await self._send_handling_redirects(
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
 response = await self._send_single_request(request)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1722, in _send_single_request
 response = await transport.handle_async_request(request)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
 resp = await self._pool.handle_async_request(req)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 221, in handle_async_request
 await self._attempt_to_acquire_connection(status)
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 178, in _attempt_to_acquire_connection
 status.set_connection(connection)
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 22, in set_connection
 assert self.connection is None
AssertionError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
 File "/usr/lib/python3.11/asyncio/runners.py", line 190, in run
 return runner.run(main)
 ^^^^^^^^^^^^^^^^
 File "/usr/lib/python3.11/asyncio/runners.py", line 118, in run
 return self._loop.run_until_complete(task)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/usr/lib/python3.11/asyncio/base_events.py", line 650, in run_until_complete
 return future.result()
 ^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 91, in main
 async with httpx.AsyncClient(proxies=proxyconfig.proxies) as client:
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 2006, in __aexit__
 await proxy.__aexit__(exc_type, exc_value, traceback)
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_transports/default.py", line 332, in __aexit__
 await self._pool.__aexit__(exc_type, exc_value, traceback)
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 326, in __aexit__
 await self.aclose()
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 312, in aclose
 raise RuntimeError(
RuntimeError: The connection pool was closed while 170 HTTP requests/responses were still in-flight.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 227, in handle_async_request
 connection = await status.wait_for_connection(timeout=timeout)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 34, in wait_for_connection
 await self._connection_acquired.wait(timeout=timeout)
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_synchronization.py", line 38, in wait
 await self._event.wait()
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 1842, in wait
 if await self._event.wait():
 ^^^^^^^^^^^^^^^^^^^^^^^^
 File "/usr/lib/python3.11/asyncio/locks.py", line 213, in wait
 await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
 File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 85, in get_models
 r = await api_call('Models', params={'year': year, 'make': make_slug[0]}, client=client)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/backoff/_async.py", line 151, in retry
 ret = await target(*args, **kwargs)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 65, in api_call
 r = await client.get(URL_BASE + url, params=DFL_PARAMS | params)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1757, in get
 return await self.request(
 ^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
 return await self.send(request, auth=auth, follow_redirects=follow_redirects)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1620, in send
 response = await self._send_handling_auth(
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
 response = await self._send_handling_redirects(
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
 response = await self._send_single_request(request)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1722, in _send_single_request
 response = await transport.handle_async_request(request)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
 resp = await self._pool.handle_async_request(req)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 232, in handle_async_request
 self._requests.remove(status)
ValueError: list.remove(x): x not in list
unhandled exception during asyncio.run() shutdown
task: <Task finished name='Task-1466' coro=<get_models() done, defined at /home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py:84> exception=ValueError('list.remove(x): x not in list')>
Traceback (most recent call last):
 File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 110, in main
 responses = await asyncio.gather(*tasks)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 85, in get_models
 r = await api_call('Models', params={'year': year, 'make': make_slug[0]}, client=client)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/backoff/_async.py", line 151, in retry
 ret = await target(*args, **kwargs)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 65, in api_call
 r = await client.get(URL_BASE + url, params=DFL_PARAMS | params)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1757, in get
 return await self.request(
 ^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
 return await self.send(request, auth=auth, follow_redirects=follow_redirects)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1620, in send
 response = await self._send_handling_auth(
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
 response = await self._send_handling_redirects(
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
 response = await self._send_single_request(request)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1722, in _send_single_request
 response = await transport.handle_async_request(request)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
 resp = await self._pool.handle_async_request(req)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 221, in handle_async_request
 await self._attempt_to_acquire_connection(status)
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 178, in _attempt_to_acquire_connection
 status.set_connection(connection)
 File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 22, in set_connection
 assert self.connection is None
AssertionError

And this happens always near the end of the request pile. Tested with splitting tasks in chunks of various sizes but it always dies at the last chunk.
Same behavior using any version of httpcore newer than httpcore==0.14.5.

I read @tomchristie 's post hinting at "oddly structured" code, but I honestly cannot see anything wrong with it.

I have looked at:

• httpx 0.21.0 showing pool-connection traceback when 0.20.0 does not. #510
• this
• this
• this

And still have no idea what (if anything) I'm doing wrong.

Please advice on what I'm doing wrong.

Many thanks in advance!

[obendidi]

+1
on python3.8
fails with httpcore=0.16.1 but works with httpcore=0.15.0, both with httpx=0.23

I'm using a similar code to @teodoryantcheff: 1 async client across different tasks used with asyncio.gather

The error I'm getting:

[2022-11-24T18:21:12Z] Traceback (most recent call last):
--
 | [2022-11-24T18:21:12Z] File "scripts/cli.py", line 43, in <module>
 | [2022-11-24T18:21:12Z] main()
 | [2022-11-24T18:21:12Z] File "scripts/cli.py", line 30, in main
 | [2022-11-24T18:21:12Z] FCVCli(
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/cli.py", line 350, in __init__
 | [2022-11-24T18:21:12Z] self._run_subcommand(self.subcommand)
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/cli.py", line 626, in _run_subcommand
 | [2022-11-24T18:21:12Z] fn(**fn_kwargs)
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 696, in fit
 | [2022-11-24T18:21:12Z] self._call_and_handle_interrupt(
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _call_and_handle_interrupt
 | [2022-11-24T18:21:12Z] return trainer_fn(*args, **kwargs)
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 737, in _fit_impl
 | [2022-11-24T18:21:12Z] results = self._run(model, ckpt_path=self.ckpt_path)
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1097, in _run
 | [2022-11-24T18:21:12Z] self._data_connector.prepare_data()
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 114, in prepare_data
 | [2022-11-24T18:21:12Z] self.trainer._call_lightning_datamodule_hook("prepare_data")
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1571, in _call_lightning_datamodule_hook
 | [2022-11-24T18:21:12Z] return fn(*args, **kwargs)
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/fcv/service/lightning/data/dali/data_module.py", line 268, in prepare_data
 | [2022-11-24T18:21:12Z] super().prepare_data(*args, **kwargs)
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/fcv/service/lightning/data/base/data_module.py", line 58, in prepare_data
 | [2022-11-24T18:21:12Z] self._image_keys = loop.run_until_complete(
 | [2022-11-24T18:21:12Z] File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
 | [2022-11-24T18:21:12Z] return future.result()
 | [2022-11-24T18:21:12Z] File "/repo/folder/libs/datasets/fcv/lib/datasets/download.py", line 114, in download_datasets
 | [2022-11-24T18:21:12Z] await client.aclose()
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/httpx/_client.py", line 1974, in aclose
 | [2022-11-24T18:21:12Z] await self._transport.aclose()
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/httpx/_transports/default.py", line 365, in aclose
 | [2022-11-24T18:21:12Z] await self._pool.aclose()
 | [2022-11-24T18:21:12Z] File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/httpcore/_async/connection_pool.py", line 312, in aclose
 | [2022-11-24T18:21:12Z] raise RuntimeError(
 | [2022-11-24T18:21:12Z] RuntimeError: The connection pool was closed while 15 HTTP requests/responses were still in-flight.

[lovelydinosaur]

Closed by #627 and released as 0.16.2