Shipped version of importlib_metadata incompatible with pip_tools

[lbhdc]

🐞 bug report

Affected Rule

The issue is caused by the rule:
rules_python (module and workspace variant)

Is this a regression?

Yes, the I don't recall the exact version of rules_python that this didn't appear in, but the bug was introduced here #781

Description

Original slack thread.

When attempting to upgrade rules_python I kept running into the following error.

AttributeError: 'PathDistribution' object has no attribute 'name'

This error was raised when anything attempts to resolve a pypi dependency. I explored using different parts of the rules_python api, but the only solution has been to bump importlib_metadata.

🔬 Minimal Reproduction

My module file

bazel_dep(name = "rules_python", version = "0.24.0")

pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
pip.parse(
 hub_name = "pip",
 requirements_lock = "//:requirements.txt",
)
use_repo(pip, "pip")

Build file. Both requirements files are empty (got the same result when they werent).

compile_pip_requirements(
 name = "requirements",
 extra_args = [
 "--allow-unsafe",
 "--resolver=backtracking",
 ],
 requirements_in = "requirements.in",
 requirements_txt = "requirements.txt",
)

🔥 Exception or Error

exec ${PAGER:-/usr/bin/less} "$0" || exit 1
Executing tests from //:requirements_test
-----------------------------------------------------------------------------
['/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0/python/pip_install/tools/dependency_resolver/dependency_resolver.py', '--generate-hashes', '--allow-unsafe', '--resolver=backtracking', '--cache-dir', '/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/_tmp/24e3066da9cc133a367098d5abc917e6', '--output-file', '/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/_tmp/24e3066da9cc133a367098d5abc917e6/requirements.txt.out', 'requirements.in']
Checking _main/requirements.txt
Traceback (most recent call last):
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0/python/pip_install/tools/dependency_resolver/dependency_resolver.py", line 188, in 
 cli()
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__click/click/core.py", line 1137, in __call__
 return self.main(*args, **kwargs)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__click/click/core.py", line 1062, in main
 rv = self.invoke(ctx)
 ^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__click/click/core.py", line 1404, in invoke
 return ctx.invoke(self.callback, **ctx.params)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__click/click/core.py", line 763, in invoke
 return __callback(*args, **kwargs)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__click/click/decorators.py", line 26, in new_func
 return f(get_current_context(), *args, **kwargs)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip_tools/piptools/scripts/compile.py", line 421, in cli
 repository = PyPIRepository(pip_args, cache_dir=cache_dir)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip_tools/piptools/repositories/pypi.py", line 75, in __init__
 self._session = self.command._build_session(options)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/cli/req_command.py", line 125, in _build_session
 session = PipSession(
 ^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/network/session.py", line 343, in __init__
 self.headers["User-Agent"] = user_agent()
 ^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/network/session.py", line 175, in user_agent
 setuptools_dist = get_default_environment().get_distribution("setuptools")
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 188, in get_distribution
 return next(matches, None)
 ^^^^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 183, in 
 matches = (
 ^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/metadata/base.py", line 612, in iter_all_distributions
 for dist in self._iter_distributions():
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 175, in _iter_distributions
 yield from finder.find(location)
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 79, in find
 for dist, info_location in self._find_impl(location):
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 64, in _find_impl
 raw_name = get_dist_name(dist)
 ^^^^^^^^^^^^^^^^^^^
 File "/home//.cache/bazel/_bazel_/0a2a8470bd48485f12b59e4a2004868a/execroot/_main/bazel-out/k8-fastbuild/bin/requirements_test.runfiles/rules_python~0.24.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_compat.py", line 52, in get_dist_name
 name = cast(Any, dist).name
 ^^^^^^^^^^^^^^^^^^^^
AttributeError: 'PathDistribution' object has no attribute 'name'

🌍 Your Environment

Operating System:

uname -srm
Linux 6.4.6-200.fc38.x86_64 x86_64

Output of bazel version:

bazel version
Bazelisk version: development
Build label: 6.2.1
Build target: bazel-out/k8-opt/bin/src/main/java/com/google/devtools/build/lib/bazel/BazelServer_deploy.jar
Build time: Fri Jun 2 16:59:58 2023 (1685725198)
Build timestamp: 1685725198
Build timestamp as int: 1685725198

Rules_python version:

0.24.0

Anything else relevant?

I bisected the different pre-built wheels of imputlib_metadata to determine which version has the property being looked for the the above error, and the first version that is compatible is 3.3.0.

This resolves the above error and pip works as expected.

diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl
index efe3bc7..e462fae 100644
--- a/python/pip_install/repositories.bzl
+++ b/python/pip_install/repositories.bzl
@@ -77,8 +77,8 @@ _RULE_DEPS = [
 ),
 (
 "pypi__importlib_metadata",
- "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
- "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
+ "https://files.pythonhosted.org/packages/7a/85/ac225e35048e050a6351b6f1251cdb2b6060092f2c6840aff1d6319941b1/importlib_metadata-3.3.0-py3-none-any.whl",
+ "bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450",
 ),
 (
 "pypi__zipp",

[lbhdc]

Tried out 0.25.0, and I am still seeing this bug. The above patch doesn't work on 0.25.0 as it seems that file has changed.

ERROR: /home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~pip~pip_rsa/BUILD.bazel:2:6: @rules_python~0.25.0~pip~pip_rsa//:pkg depends on @rules_python~0.25.0~pip~pip_311_rsa//:pkg in repository @rules_python~0.25.0~pip~pip_311_rsa which failed to fetch. no such package '@rules_python~0.25.0~pip~pip_311_rsa//': whl_library rules_python~0.25.0~pip~pip_311_rsa failed: (ERROR: Exception:
Traceback (most recent call last):
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/cli/base_command.py", line 160, in exc_logging_wrapper
 status = run_func(*args)
 ^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/cli/req_command.py", line 247, in wrapper
 return func(self, options, args)
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/commands/wheel.py", line 108, in run
 session = self.get_default_session(options)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/cli/req_command.py", line 98, in get_default_session
 self._session = self.enter_context(self._build_session(options))
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/cli/req_command.py", line 125, in _build_session
 session = PipSession(
 ^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/network/session.py", line 343, in __init__
 self.headers["User-Agent"] = user_agent()
 ^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/network/session.py", line 175, in user_agent
 setuptools_dist = get_default_environment().get_distribution("setuptools")
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 188, in get_distribution
 return next(matches, None)
 ^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 183, in <genexpr>
 matches = (
 ^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/base.py", line 612, in iter_all_distributions
 for dist in self._iter_distributions():
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 175, in _iter_distributions
 yield from finder.find(location)
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 79, in find
 for dist, info_location in self._find_impl(location):
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 64, in _find_impl
 raw_name = get_dist_name(dist)
 ^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_compat.py", line 52, in get_dist_name
 name = cast(Any, dist).name
 ^^^^^^^^^^^^^^^^^^^^
AttributeError: 'PathDistribution' object has no attribute 'name'
Traceback (most recent call last):
 File "<frozen runpy>", line 198, in _run_module_as_main
 File "<frozen runpy>", line 88, in _run_code
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/__main__.py", line 31, in <module>
 sys.exit(_main())
 ^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/cli/main.py", line 70, in main
 return command.main(cmd_args)
 ^^^^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/cli/base_command.py", line 101, in main
 return self._main(args)
 ^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/cli/base_command.py", line 216, in _main
 self.handle_pip_version_check(options)
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/cli/req_command.py", line 179, in handle_pip_version_check
 session = self._build_session(
 ^^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/cli/req_command.py", line 125, in _build_session
 session = PipSession(
 ^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/network/session.py", line 343, in __init__
 self.headers["User-Agent"] = user_agent()
 ^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/network/session.py", line 175, in user_agent
 setuptools_dist = get_default_environment().get_distribution("setuptools")
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 188, in get_distribution
 return next(matches, None)
 ^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 183, in <genexpr>
 matches = (
 ^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/base.py", line 612, in iter_all_distributions
 for dist in self._iter_distributions():
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 175, in _iter_distributions
 yield from finder.find(location)
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 79, in find
 for dist, info_location in self._find_impl(location):
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_envs.py", line 64, in _find_impl
 raw_name = get_dist_name(dist)
 ^^^^^^^^^^^^^^^^^^^
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~internal_deps~pypi__pip/pip/_internal/metadata/importlib/_compat.py", line 52, in get_dist_name
 name = cast(Any, dist).name
 ^^^^^^^^^^^^^^^^^^^^
AttributeError: 'PathDistribution' object has no attribute 'name'
Traceback (most recent call last):
 File "<frozen runpy>", line 198, in _run_module_as_main
 File "<frozen runpy>", line 88, in _run_code
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0/python/pip_install/tools/wheel_installer/wheel_installer.py", line 200, in <module>
 main()
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0/python/pip_install/tools/wheel_installer/wheel_installer.py", line 180, in main
 subprocess.run(pip_args, check=True, env=env)
 File "/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~python~python_3_11_x86_64-unknown-linux-gnu/lib/python3.11/subprocess.py", line 571, in run
 raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['/home/<user>/.cache/bazel/_bazel_<user>/0a2a8470bd48485f12b59e4a2004868a/external/rules_python~0.25.0~python~python_3_11_x86_64-unknown-linux-gnu/bin/python3', '-m', 'pip', '--isolated', 'wheel', '--no-deps', '-r', '/tmp/tmp4hq6mu7i']' returned non-zero exit status 1.
) error code: '1'

[lbhdc]

Updated patch for 0.25.0

diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl
index 4b209b3..69d6c83 100644
--- a/python/pip_install/repositories.bzl
+++ b/python/pip_install/repositories.bzl
@@ -38,8 +38,8 @@ _RULE_DEPS = [
 ),
 (
 "pypi__importlib_metadata",
- "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
- "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
+ "https://files.pythonhosted.org/packages/7a/85/ac225e35048e050a6351b6f1251cdb2b6060092f2c6840aff1d6319941b1/importlib_metadata-3.3.0-py3-none-any.whl",
+ "bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450",
 ),
 (
 "pypi__installer",