Skip to content
12 changes: 12 additions & 0 deletions docs/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0

_DOCS = {
"extensions": "//docs:extensions-docs",
"packaging": "//docs:packaging-docs",
"pip": "//docs:pip-docs",
"pip_repository": "//docs:pip-repository",
Expand DownExpand Up@@ -101,6 +102,17 @@ stardoc(
deps = [":defs"],
)

stardoc(
name = "extensions-docs",
out = "extensions.md_",
input = "//python:extensions.bzl",
target_compatible_with = _NOT_WINDOWS,
deps = [
":pip_install_bzl",
"@bazel_skylib//lib:versions",
],
)

stardoc(
name = "pip-docs",
out = "pip.md_",
Expand Down
40 changes: 40 additions & 0 deletions docs/extensions.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions examples/bzlmod/BUILD.bazel
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
load("@pip//:requirements.bzl", "requirement")
load("@pip//yamllint:bin.bzl", yamllint_bin = "bin")
load("@python3_9//:defs.bzl", py_test_with_transition = "py_test")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
Expand DownExpand Up@@ -43,3 +44,11 @@ py_test_with_transition(
main = "test.py",
deps = [":lib"],
)

alias(
name = "yamllint",
# This is using the struct defined by the spoke repo 'yamllint' and it is
# re-exported by the hub repo named 'pip'. This allows bzlmod and
# non-bzlmod users to access the entry point targets.
actual = yamllint_bin.yamllint,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really follow this part -- why does it have to be put into a bzl file and loaded like this? This is pointing to a target, right? So why can't it just be "@pip//yamllint:bin"?

(I'm guessing the answer has something to do with bzlmod, repo mapping, and repo visibility, but I'd like to understand more).

)
3 changes: 3 additions & 0 deletions examples/bzlmod/MODULE.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,9 @@ register_toolchains(
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pip",
# Generate user friendly alias labels for each dependency that we have and
# enable entry_point support
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
Expand Down
8 changes: 5 additions & 3 deletions internal_deps.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,9 +53,11 @@ def rules_python_internal_deps():
maybe(
http_archive,
name = "io_bazel_stardoc",
url = "https://github.com/bazelbuild/stardoc/archive/6f274e903009158504a9d9130d7f7d5f3e9421ed.tar.gz",
sha256 = "b5d6891f869d5b5a224316ec4dd9e9d481885a9b1a1c81eb846e20180156f2fa",
strip_prefix = "stardoc-6f274e903009158504a9d9130d7f7d5f3e9421ed",
sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
"https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
],
)

# The below two deps are required for the integration test with bazel
Expand Down
3 changes: 2 additions & 1 deletion python/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ filegroup(
visibility = ["//:__pkg__"],
)

# ========= bzl_library targets end =========
# ========= bzl_library targets start =========

bzl_library(
name = "current_py_toolchain_bzl",
Expand DownExpand Up@@ -140,6 +140,7 @@ filegroup(
name = "bzl",
srcs = [
"defs.bzl",
"extensions.bzl",
"packaging.bzl",
"pip.bzl",
"repositories.bzl",
Expand Down
39 changes: 38 additions & 1 deletion python/extensions.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"Module extensions for use with bzlmod"
"""Module extensions for use with bzlmod.

# python

## python.toolchain

TODO

# pip

## pip.parse

You can use the `pip_parse` to access the generate entry_point targets as follows.
First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the
external spoke repository contents in distinct folders in the hub repo:
```starlark
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pypi",
# Generate aliases for more ergonomic consumption of dependencies from
# the `pypi` external repo.
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
use_repo(pip, "pip")
```

Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point:
```starlark
load("@pypi//flake8:bin.bzl", "bin")

alias(
name = "flake8",
actual = bin.flake8,
)
```"""

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
Expand DownExpand Up@@ -126,4 +162,5 @@ pip = module_extension(
tag_classes = {
"parse": tag_class(attrs = _pip_parse_ext_attrs()),
},
doc = "NOTE @aignas 2023-05-01: This will not appear in the docs generated with stardoc 0.5.3",
)
27 changes: 27 additions & 0 deletions python/pip_install/pip_repository.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -307,6 +307,33 @@ alias(
)
rctx.file("{}/BUILD.bazel".format(name), build_content)

# Re-export the opaque struct that contains labels to all of the
# entrypoint labels available in the package repository.
#
# NOTE: We have to do this because the hub repo created by `pip_parse`
# does not know about the existing entrypoints and therefor cannot
# generate the aliases for them. What it knows though is that the
# entrypoint labels will be available as a struct in a particular file
# when the external repository contents will be fetched from PyPI (or
# another index server).
#
# What is more, we should not put re-exports of all available `bin`
# structs into a single file, because that would mean eager fetches of
# all external repos even though the users are using only a single
# entry_point (or none at all, if the re-exports are done in the
# `requirements.bzl` and the user wants to use the `requirement`
# macro).
bin_content = """\
\"\"\"Autogenerated by 'pip_repository.bzl#_pkg_aliases'.\"\"\"
load("@{repo_name}_{dep}//:bin.bzl", _bin = "bin")

bin = _bin
""".format(
repo_name = repo_name,
dep = name,
)
rctx.file("{}/bin.bzl".format(name), bin_content)

def _bzlmod_pkg_aliases(repo_name, bzl_packages):
"""Create alias declarations for each python dependency.

Expand Down
32 changes: 32 additions & 0 deletions python/pip_install/tools/wheel_installer/wheel_installer.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@
import subprocess
import sys
import textwrap
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Dict, Iterable, List, Optional, Set, Tuple
Expand DownExpand Up@@ -289,6 +290,31 @@ def _generate_build_file_contents(
)


def _generate_bin_bzl_contents(entrypoints: Dict[str, str]) -> str:
"""Generate the contents of bin.bzl for each package.

The presence of the `bin.bzl` and the struct in it named `bin` becomes the API to
the users and to the hub repo on how to consume the labels to the entry_point
targets.
"""
struct_params = sorted(
[
'{attr} = Label("//:{target_name}"),'.format(
attr=attr, target_name=target_name
)
for attr, target_name in entrypoints.items()
]
)
if not struct_params:
struct_def = "struct()"
else:
struct_def = "struct(\n {}\n)" "".format("\n ".join(struct_params))

return "\n# Autogenerated by wheel_installer.py#_generate_bin_bzl_contents\n\nbin = {}\n".format(
struct_def
)


def _extract_wheel(
wheel_file: str,
extras: Dict[str, Set[str]],
Expand DownExpand Up@@ -329,6 +355,7 @@ def _extract_wheel(
]

entry_points = []
entry_point_targets = {}
for name, (module, attribute) in sorted(whl.entry_points().items()):
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
Expand All@@ -340,6 +367,8 @@ def _extract_wheel(
(installation_dir / entry_point_script_name).write_text(
_generate_entry_point_contents(module, attribute)
)

entry_point_targets[entry_point_without_py] = entry_point_target_name
entry_points.append(
_generate_entry_point_rule(
entry_point_target_name,
Expand All@@ -348,6 +377,9 @@ def _extract_wheel(
)
)

with open(os.path.join(installation_dir, "bin.bzl"), "w") as bin_bzl:
bin_bzl.write(_generate_bin_bzl_contents(entry_point_targets))

with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file:
additional_content = entry_points
data = []
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
feat: expose python package entrypoints via an opaque struct by aignas · Pull Request #1189 · bazel-contrib/rules_python · GitHub
Skip to content
12 changes: 12 additions & 0 deletions docs/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0

_DOCS = {
"extensions": "//docs:extensions-docs",
"packaging": "//docs:packaging-docs",
"pip": "//docs:pip-docs",
"pip_repository": "//docs:pip-repository",
Expand DownExpand Up@@ -101,6 +102,17 @@ stardoc(
deps = [":defs"],
)

stardoc(
name = "extensions-docs",
out = "extensions.md_",
input = "//python:extensions.bzl",
target_compatible_with = _NOT_WINDOWS,
deps = [
":pip_install_bzl",
"@bazel_skylib//lib:versions",
],
)

stardoc(
name = "pip-docs",
out = "pip.md_",
Expand Down
40 changes: 40 additions & 0 deletions docs/extensions.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions examples/bzlmod/BUILD.bazel
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
load("@pip//:requirements.bzl", "requirement")
load("@pip//yamllint:bin.bzl", yamllint_bin = "bin")
load("@python3_9//:defs.bzl", py_test_with_transition = "py_test")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
Expand DownExpand Up@@ -43,3 +44,11 @@ py_test_with_transition(
main = "test.py",
deps = [":lib"],
)

alias(
name = "yamllint",
# This is using the struct defined by the spoke repo 'yamllint' and it is
# re-exported by the hub repo named 'pip'. This allows bzlmod and
# non-bzlmod users to access the entry point targets.
actual = yamllint_bin.yamllint,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really follow this part -- why does it have to be put into a bzl file and loaded like this? This is pointing to a target, right? So why can't it just be "@pip//yamllint:bin"?

(I'm guessing the answer has something to do with bzlmod, repo mapping, and repo visibility, but I'd like to understand more).

)
3 changes: 3 additions & 0 deletions examples/bzlmod/MODULE.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,9 @@ register_toolchains(
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pip",
# Generate user friendly alias labels for each dependency that we have and
# enable entry_point support
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
Expand Down
8 changes: 5 additions & 3 deletions internal_deps.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,9 +53,11 @@ def rules_python_internal_deps():
maybe(
http_archive,
name = "io_bazel_stardoc",
url = "https://github.com/bazelbuild/stardoc/archive/6f274e903009158504a9d9130d7f7d5f3e9421ed.tar.gz",
sha256 = "b5d6891f869d5b5a224316ec4dd9e9d481885a9b1a1c81eb846e20180156f2fa",
strip_prefix = "stardoc-6f274e903009158504a9d9130d7f7d5f3e9421ed",
sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
"https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
],
)

# The below two deps are required for the integration test with bazel
Expand Down
3 changes: 2 additions & 1 deletion python/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ filegroup(
visibility = ["//:__pkg__"],
)

# ========= bzl_library targets end =========
# ========= bzl_library targets start =========

bzl_library(
name = "current_py_toolchain_bzl",
Expand DownExpand Up@@ -140,6 +140,7 @@ filegroup(
name = "bzl",
srcs = [
"defs.bzl",
"extensions.bzl",
"packaging.bzl",
"pip.bzl",
"repositories.bzl",
Expand Down
39 changes: 38 additions & 1 deletion python/extensions.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"Module extensions for use with bzlmod"
"""Module extensions for use with bzlmod.

# python

## python.toolchain

TODO

# pip

## pip.parse

You can use the `pip_parse` to access the generate entry_point targets as follows.
First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the
external spoke repository contents in distinct folders in the hub repo:
```starlark
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pypi",
# Generate aliases for more ergonomic consumption of dependencies from
# the `pypi` external repo.
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
use_repo(pip, "pip")
```

Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point:
```starlark
load("@pypi//flake8:bin.bzl", "bin")

alias(
name = "flake8",
actual = bin.flake8,
)
```"""

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
Expand DownExpand Up@@ -126,4 +162,5 @@ pip = module_extension(
tag_classes = {
"parse": tag_class(attrs = _pip_parse_ext_attrs()),
},
doc = "NOTE @aignas 2023-05-01: This will not appear in the docs generated with stardoc 0.5.3",
)
27 changes: 27 additions & 0 deletions python/pip_install/pip_repository.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -307,6 +307,33 @@ alias(
)
rctx.file("{}/BUILD.bazel".format(name), build_content)

# Re-export the opaque struct that contains labels to all of the
# entrypoint labels available in the package repository.
#
# NOTE: We have to do this because the hub repo created by `pip_parse`
# does not know about the existing entrypoints and therefor cannot
# generate the aliases for them. What it knows though is that the
# entrypoint labels will be available as a struct in a particular file
# when the external repository contents will be fetched from PyPI (or
# another index server).
#
# What is more, we should not put re-exports of all available `bin`
# structs into a single file, because that would mean eager fetches of
# all external repos even though the users are using only a single
# entry_point (or none at all, if the re-exports are done in the
# `requirements.bzl` and the user wants to use the `requirement`
# macro).
bin_content = """\
\"\"\"Autogenerated by 'pip_repository.bzl#_pkg_aliases'.\"\"\"
load("@{repo_name}_{dep}//:bin.bzl", _bin = "bin")

bin = _bin
""".format(
repo_name = repo_name,
dep = name,
)
rctx.file("{}/bin.bzl".format(name), bin_content)

def _bzlmod_pkg_aliases(repo_name, bzl_packages):
"""Create alias declarations for each python dependency.

Expand Down
32 changes: 32 additions & 0 deletions python/pip_install/tools/wheel_installer/wheel_installer.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@
import subprocess
import sys
import textwrap
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Dict, Iterable, List, Optional, Set, Tuple
Expand DownExpand Up@@ -289,6 +290,31 @@ def _generate_build_file_contents(
)


def _generate_bin_bzl_contents(entrypoints: Dict[str, str]) -> str:
"""Generate the contents of bin.bzl for each package.

The presence of the `bin.bzl` and the struct in it named `bin` becomes the API to
the users and to the hub repo on how to consume the labels to the entry_point
targets.
"""
struct_params = sorted(
[
'{attr} = Label("//:{target_name}"),'.format(
attr=attr, target_name=target_name
)
for attr, target_name in entrypoints.items()
]
)
if not struct_params:
struct_def = "struct()"
else:
struct_def = "struct(\n {}\n)" "".format("\n ".join(struct_params))

return "\n# Autogenerated by wheel_installer.py#_generate_bin_bzl_contents\n\nbin = {}\n".format(
struct_def
)


def _extract_wheel(
wheel_file: str,
extras: Dict[str, Set[str]],
Expand DownExpand Up@@ -329,6 +355,7 @@ def _extract_wheel(
]

entry_points = []
entry_point_targets = {}
for name, (module, attribute) in sorted(whl.entry_points().items()):
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
Expand All@@ -340,6 +367,8 @@ def _extract_wheel(
(installation_dir / entry_point_script_name).write_text(
_generate_entry_point_contents(module, attribute)
)

entry_point_targets[entry_point_without_py] = entry_point_target_name
entry_points.append(
_generate_entry_point_rule(
entry_point_target_name,
Expand All@@ -348,6 +377,9 @@ def _extract_wheel(
)
)

with open(os.path.join(installation_dir, "bin.bzl"), "w") as bin_bzl:
bin_bzl.write(_generate_bin_bzl_contents(entry_point_targets))

with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file:
additional_content = entry_points
data = []
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat: expose python package entrypoints via an opaque struct by aignas · Pull Request #1189 · bazel-contrib/rules_python · GitHub
Skip to content
12 changes: 12 additions & 0 deletions docs/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0

_DOCS = {
"extensions": "//docs:extensions-docs",
"packaging": "//docs:packaging-docs",
"pip": "//docs:pip-docs",
"pip_repository": "//docs:pip-repository",
Expand DownExpand Up@@ -101,6 +102,17 @@ stardoc(
deps = [":defs"],
)

stardoc(
name = "extensions-docs",
out = "extensions.md_",
input = "//python:extensions.bzl",
target_compatible_with = _NOT_WINDOWS,
deps = [
":pip_install_bzl",
"@bazel_skylib//lib:versions",
],
)

stardoc(
name = "pip-docs",
out = "pip.md_",
Expand Down
40 changes: 40 additions & 0 deletions docs/extensions.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions examples/bzlmod/BUILD.bazel
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
load("@pip//:requirements.bzl", "requirement")
load("@pip//yamllint:bin.bzl", yamllint_bin = "bin")
load("@python3_9//:defs.bzl", py_test_with_transition = "py_test")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
Expand DownExpand Up@@ -43,3 +44,11 @@ py_test_with_transition(
main = "test.py",
deps = [":lib"],
)

alias(
name = "yamllint",
# This is using the struct defined by the spoke repo 'yamllint' and it is
# re-exported by the hub repo named 'pip'. This allows bzlmod and
# non-bzlmod users to access the entry point targets.
actual = yamllint_bin.yamllint,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really follow this part -- why does it have to be put into a bzl file and loaded like this? This is pointing to a target, right? So why can't it just be "@pip//yamllint:bin"?

(I'm guessing the answer has something to do with bzlmod, repo mapping, and repo visibility, but I'd like to understand more).

)
3 changes: 3 additions & 0 deletions examples/bzlmod/MODULE.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,9 @@ register_toolchains(
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pip",
# Generate user friendly alias labels for each dependency that we have and
# enable entry_point support
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
Expand Down
8 changes: 5 additions & 3 deletions internal_deps.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,9 +53,11 @@ def rules_python_internal_deps():
maybe(
http_archive,
name = "io_bazel_stardoc",
url = "https://github.com/bazelbuild/stardoc/archive/6f274e903009158504a9d9130d7f7d5f3e9421ed.tar.gz",
sha256 = "b5d6891f869d5b5a224316ec4dd9e9d481885a9b1a1c81eb846e20180156f2fa",
strip_prefix = "stardoc-6f274e903009158504a9d9130d7f7d5f3e9421ed",
sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
"https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
],
)

# The below two deps are required for the integration test with bazel
Expand Down
3 changes: 2 additions & 1 deletion python/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ filegroup(
visibility = ["//:__pkg__"],
)

# ========= bzl_library targets end =========
# ========= bzl_library targets start =========

bzl_library(
name = "current_py_toolchain_bzl",
Expand DownExpand Up@@ -140,6 +140,7 @@ filegroup(
name = "bzl",
srcs = [
"defs.bzl",
"extensions.bzl",
"packaging.bzl",
"pip.bzl",
"repositories.bzl",
Expand Down
39 changes: 38 additions & 1 deletion python/extensions.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"Module extensions for use with bzlmod"
"""Module extensions for use with bzlmod.

# python

## python.toolchain

TODO

# pip

## pip.parse

You can use the `pip_parse` to access the generate entry_point targets as follows.
First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the
external spoke repository contents in distinct folders in the hub repo:
```starlark
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pypi",
# Generate aliases for more ergonomic consumption of dependencies from
# the `pypi` external repo.
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
use_repo(pip, "pip")
```

Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point:
```starlark
load("@pypi//flake8:bin.bzl", "bin")

alias(
name = "flake8",
actual = bin.flake8,
)
```"""

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
Expand DownExpand Up@@ -126,4 +162,5 @@ pip = module_extension(
tag_classes = {
"parse": tag_class(attrs = _pip_parse_ext_attrs()),
},
doc = "NOTE @aignas 2023-05-01: This will not appear in the docs generated with stardoc 0.5.3",
)
27 changes: 27 additions & 0 deletions python/pip_install/pip_repository.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -307,6 +307,33 @@ alias(
)
rctx.file("{}/BUILD.bazel".format(name), build_content)

# Re-export the opaque struct that contains labels to all of the
# entrypoint labels available in the package repository.
#
# NOTE: We have to do this because the hub repo created by `pip_parse`
# does not know about the existing entrypoints and therefor cannot
# generate the aliases for them. What it knows though is that the
# entrypoint labels will be available as a struct in a particular file
# when the external repository contents will be fetched from PyPI (or
# another index server).
#
# What is more, we should not put re-exports of all available `bin`
# structs into a single file, because that would mean eager fetches of
# all external repos even though the users are using only a single
# entry_point (or none at all, if the re-exports are done in the
# `requirements.bzl` and the user wants to use the `requirement`
# macro).
bin_content = """\
\"\"\"Autogenerated by 'pip_repository.bzl#_pkg_aliases'.\"\"\"
load("@{repo_name}_{dep}//:bin.bzl", _bin = "bin")

bin = _bin
""".format(
repo_name = repo_name,
dep = name,
)
rctx.file("{}/bin.bzl".format(name), bin_content)

def _bzlmod_pkg_aliases(repo_name, bzl_packages):
"""Create alias declarations for each python dependency.

Expand Down
32 changes: 32 additions & 0 deletions python/pip_install/tools/wheel_installer/wheel_installer.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@
import subprocess
import sys
import textwrap
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Dict, Iterable, List, Optional, Set, Tuple
Expand DownExpand Up@@ -289,6 +290,31 @@ def _generate_build_file_contents(
)


def _generate_bin_bzl_contents(entrypoints: Dict[str, str]) -> str:
"""Generate the contents of bin.bzl for each package.

The presence of the `bin.bzl` and the struct in it named `bin` becomes the API to
the users and to the hub repo on how to consume the labels to the entry_point
targets.
"""
struct_params = sorted(
[
'{attr} = Label("//:{target_name}"),'.format(
attr=attr, target_name=target_name
)
for attr, target_name in entrypoints.items()
]
)
if not struct_params:
struct_def = "struct()"
else:
struct_def = "struct(\n {}\n)" "".format("\n ".join(struct_params))

return "\n# Autogenerated by wheel_installer.py#_generate_bin_bzl_contents\n\nbin = {}\n".format(
struct_def
)


def _extract_wheel(
wheel_file: str,
extras: Dict[str, Set[str]],
Expand DownExpand Up@@ -329,6 +355,7 @@ def _extract_wheel(
]

entry_points = []
entry_point_targets = {}
for name, (module, attribute) in sorted(whl.entry_points().items()):
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
Expand All@@ -340,6 +367,8 @@ def _extract_wheel(
(installation_dir / entry_point_script_name).write_text(
_generate_entry_point_contents(module, attribute)
)

entry_point_targets[entry_point_without_py] = entry_point_target_name
entry_points.append(
_generate_entry_point_rule(
entry_point_target_name,
Expand All@@ -348,6 +377,9 @@ def _extract_wheel(
)
)

with open(os.path.join(installation_dir, "bin.bzl"), "w") as bin_bzl:
bin_bzl.write(_generate_bin_bzl_contents(entry_point_targets))

with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file:
additional_content = entry_points
data = []
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat: expose python package entrypoints via an opaque struct by aignas · Pull Request #1189 · bazel-contrib/rules_python · GitHub
Skip to content
12 changes: 12 additions & 0 deletions docs/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0

_DOCS = {
"extensions": "//docs:extensions-docs",
"packaging": "//docs:packaging-docs",
"pip": "//docs:pip-docs",
"pip_repository": "//docs:pip-repository",
Expand DownExpand Up@@ -101,6 +102,17 @@ stardoc(
deps = [":defs"],
)

stardoc(
name = "extensions-docs",
out = "extensions.md_",
input = "//python:extensions.bzl",
target_compatible_with = _NOT_WINDOWS,
deps = [
":pip_install_bzl",
"@bazel_skylib//lib:versions",
],
)

stardoc(
name = "pip-docs",
out = "pip.md_",
Expand Down
40 changes: 40 additions & 0 deletions docs/extensions.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions examples/bzlmod/BUILD.bazel
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
load("@pip//:requirements.bzl", "requirement")
load("@pip//yamllint:bin.bzl", yamllint_bin = "bin")
load("@python3_9//:defs.bzl", py_test_with_transition = "py_test")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
Expand DownExpand Up@@ -43,3 +44,11 @@ py_test_with_transition(
main = "test.py",
deps = [":lib"],
)

alias(
name = "yamllint",
# This is using the struct defined by the spoke repo 'yamllint' and it is
# re-exported by the hub repo named 'pip'. This allows bzlmod and
# non-bzlmod users to access the entry point targets.
actual = yamllint_bin.yamllint,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really follow this part -- why does it have to be put into a bzl file and loaded like this? This is pointing to a target, right? So why can't it just be "@pip//yamllint:bin"?

(I'm guessing the answer has something to do with bzlmod, repo mapping, and repo visibility, but I'd like to understand more).

)
3 changes: 3 additions & 0 deletions examples/bzlmod/MODULE.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,9 @@ register_toolchains(
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pip",
# Generate user friendly alias labels for each dependency that we have and
# enable entry_point support
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
Expand Down
8 changes: 5 additions & 3 deletions internal_deps.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,9 +53,11 @@ def rules_python_internal_deps():
maybe(
http_archive,
name = "io_bazel_stardoc",
url = "https://github.com/bazelbuild/stardoc/archive/6f274e903009158504a9d9130d7f7d5f3e9421ed.tar.gz",
sha256 = "b5d6891f869d5b5a224316ec4dd9e9d481885a9b1a1c81eb846e20180156f2fa",
strip_prefix = "stardoc-6f274e903009158504a9d9130d7f7d5f3e9421ed",
sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
"https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
],
)

# The below two deps are required for the integration test with bazel
Expand Down
3 changes: 2 additions & 1 deletion python/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ filegroup(
visibility = ["//:__pkg__"],
)

# ========= bzl_library targets end =========
# ========= bzl_library targets start =========

bzl_library(
name = "current_py_toolchain_bzl",
Expand DownExpand Up@@ -140,6 +140,7 @@ filegroup(
name = "bzl",
srcs = [
"defs.bzl",
"extensions.bzl",
"packaging.bzl",
"pip.bzl",
"repositories.bzl",
Expand Down
39 changes: 38 additions & 1 deletion python/extensions.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"Module extensions for use with bzlmod"
"""Module extensions for use with bzlmod.

# python

## python.toolchain

TODO

# pip

## pip.parse

You can use the `pip_parse` to access the generate entry_point targets as follows.
First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the
external spoke repository contents in distinct folders in the hub repo:
```starlark
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pypi",
# Generate aliases for more ergonomic consumption of dependencies from
# the `pypi` external repo.
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
use_repo(pip, "pip")
```

Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point:
```starlark
load("@pypi//flake8:bin.bzl", "bin")

alias(
name = "flake8",
actual = bin.flake8,
)
```"""

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
Expand DownExpand Up@@ -126,4 +162,5 @@ pip = module_extension(
tag_classes = {
"parse": tag_class(attrs = _pip_parse_ext_attrs()),
},
doc = "NOTE @aignas 2023-05-01: This will not appear in the docs generated with stardoc 0.5.3",
)
27 changes: 27 additions & 0 deletions python/pip_install/pip_repository.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -307,6 +307,33 @@ alias(
)
rctx.file("{}/BUILD.bazel".format(name), build_content)

# Re-export the opaque struct that contains labels to all of the
# entrypoint labels available in the package repository.
#
# NOTE: We have to do this because the hub repo created by `pip_parse`
# does not know about the existing entrypoints and therefor cannot
# generate the aliases for them. What it knows though is that the
# entrypoint labels will be available as a struct in a particular file
# when the external repository contents will be fetched from PyPI (or
# another index server).
#
# What is more, we should not put re-exports of all available `bin`
# structs into a single file, because that would mean eager fetches of
# all external repos even though the users are using only a single
# entry_point (or none at all, if the re-exports are done in the
# `requirements.bzl` and the user wants to use the `requirement`
# macro).
bin_content = """\
\"\"\"Autogenerated by 'pip_repository.bzl#_pkg_aliases'.\"\"\"
load("@{repo_name}_{dep}//:bin.bzl", _bin = "bin")

bin = _bin
""".format(
repo_name = repo_name,
dep = name,
)
rctx.file("{}/bin.bzl".format(name), bin_content)

def _bzlmod_pkg_aliases(repo_name, bzl_packages):
"""Create alias declarations for each python dependency.

Expand Down
32 changes: 32 additions & 0 deletions python/pip_install/tools/wheel_installer/wheel_installer.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@
import subprocess
import sys
import textwrap
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Dict, Iterable, List, Optional, Set, Tuple
Expand DownExpand Up@@ -289,6 +290,31 @@ def _generate_build_file_contents(
)


def _generate_bin_bzl_contents(entrypoints: Dict[str, str]) -> str:
"""Generate the contents of bin.bzl for each package.

The presence of the `bin.bzl` and the struct in it named `bin` becomes the API to
the users and to the hub repo on how to consume the labels to the entry_point
targets.
"""
struct_params = sorted(
[
'{attr} = Label("//:{target_name}"),'.format(
attr=attr, target_name=target_name
)
for attr, target_name in entrypoints.items()
]
)
if not struct_params:
struct_def = "struct()"
else:
struct_def = "struct(\n {}\n)" "".format("\n ".join(struct_params))

return "\n# Autogenerated by wheel_installer.py#_generate_bin_bzl_contents\n\nbin = {}\n".format(
struct_def
)


def _extract_wheel(
wheel_file: str,
extras: Dict[str, Set[str]],
Expand DownExpand Up@@ -329,6 +355,7 @@ def _extract_wheel(
]

entry_points = []
entry_point_targets = {}
for name, (module, attribute) in sorted(whl.entry_points().items()):
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
Expand All@@ -340,6 +367,8 @@ def _extract_wheel(
(installation_dir / entry_point_script_name).write_text(
_generate_entry_point_contents(module, attribute)
)

entry_point_targets[entry_point_without_py] = entry_point_target_name
entry_points.append(
_generate_entry_point_rule(
entry_point_target_name,
Expand All@@ -348,6 +377,9 @@ def _extract_wheel(
)
)

with open(os.path.join(installation_dir, "bin.bzl"), "w") as bin_bzl:
bin_bzl.write(_generate_bin_bzl_contents(entry_point_targets))

with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file:
additional_content = entry_points
data = []
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' feat: expose python package entrypoints via an opaque struct by aignas · Pull Request #1189 · bazel-contrib/rules_python · GitHub
Skip to content
12 changes: 12 additions & 0 deletions docs/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0

_DOCS = {
"extensions": "//docs:extensions-docs",
"packaging": "//docs:packaging-docs",
"pip": "//docs:pip-docs",
"pip_repository": "//docs:pip-repository",
Expand DownExpand Up@@ -101,6 +102,17 @@ stardoc(
deps = [":defs"],
)

stardoc(
name = "extensions-docs",
out = "extensions.md_",
input = "//python:extensions.bzl",
target_compatible_with = _NOT_WINDOWS,
deps = [
":pip_install_bzl",
"@bazel_skylib//lib:versions",
],
)

stardoc(
name = "pip-docs",
out = "pip.md_",
Expand Down
40 changes: 40 additions & 0 deletions docs/extensions.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions examples/bzlmod/BUILD.bazel
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
load("@pip//:requirements.bzl", "requirement")
load("@pip//yamllint:bin.bzl", yamllint_bin = "bin")
load("@python3_9//:defs.bzl", py_test_with_transition = "py_test")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
Expand DownExpand Up@@ -43,3 +44,11 @@ py_test_with_transition(
main = "test.py",
deps = [":lib"],
)

alias(
name = "yamllint",
# This is using the struct defined by the spoke repo 'yamllint' and it is
# re-exported by the hub repo named 'pip'. This allows bzlmod and
# non-bzlmod users to access the entry point targets.
actual = yamllint_bin.yamllint,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really follow this part -- why does it have to be put into a bzl file and loaded like this? This is pointing to a target, right? So why can't it just be "@pip//yamllint:bin"?

(I'm guessing the answer has something to do with bzlmod, repo mapping, and repo visibility, but I'd like to understand more).

)
3 changes: 3 additions & 0 deletions examples/bzlmod/MODULE.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,9 @@ register_toolchains(
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pip",
# Generate user friendly alias labels for each dependency that we have and
# enable entry_point support
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
Expand Down
8 changes: 5 additions & 3 deletions internal_deps.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,9 +53,11 @@ def rules_python_internal_deps():
maybe(
http_archive,
name = "io_bazel_stardoc",
url = "https://github.com/bazelbuild/stardoc/archive/6f274e903009158504a9d9130d7f7d5f3e9421ed.tar.gz",
sha256 = "b5d6891f869d5b5a224316ec4dd9e9d481885a9b1a1c81eb846e20180156f2fa",
strip_prefix = "stardoc-6f274e903009158504a9d9130d7f7d5f3e9421ed",
sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
"https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
],
)

# The below two deps are required for the integration test with bazel
Expand Down
3 changes: 2 additions & 1 deletion python/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ filegroup(
visibility = ["//:__pkg__"],
)

# ========= bzl_library targets end =========
# ========= bzl_library targets start =========

bzl_library(
name = "current_py_toolchain_bzl",
Expand DownExpand Up@@ -140,6 +140,7 @@ filegroup(
name = "bzl",
srcs = [
"defs.bzl",
"extensions.bzl",
"packaging.bzl",
"pip.bzl",
"repositories.bzl",
Expand Down
39 changes: 38 additions & 1 deletion python/extensions.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"Module extensions for use with bzlmod"
"""Module extensions for use with bzlmod.

# python

## python.toolchain

TODO

# pip

## pip.parse

You can use the `pip_parse` to access the generate entry_point targets as follows.
First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the
external spoke repository contents in distinct folders in the hub repo:
```starlark
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pypi",
# Generate aliases for more ergonomic consumption of dependencies from
# the `pypi` external repo.
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
use_repo(pip, "pip")
```

Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point:
```starlark
load("@pypi//flake8:bin.bzl", "bin")

alias(
name = "flake8",
actual = bin.flake8,
)
```"""

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
Expand DownExpand Up@@ -126,4 +162,5 @@ pip = module_extension(
tag_classes = {
"parse": tag_class(attrs = _pip_parse_ext_attrs()),
},
doc = "NOTE @aignas 2023-05-01: This will not appear in the docs generated with stardoc 0.5.3",
)
27 changes: 27 additions & 0 deletions python/pip_install/pip_repository.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -307,6 +307,33 @@ alias(
)
rctx.file("{}/BUILD.bazel".format(name), build_content)

# Re-export the opaque struct that contains labels to all of the
# entrypoint labels available in the package repository.
#
# NOTE: We have to do this because the hub repo created by `pip_parse`
# does not know about the existing entrypoints and therefor cannot
# generate the aliases for them. What it knows though is that the
# entrypoint labels will be available as a struct in a particular file
# when the external repository contents will be fetched from PyPI (or
# another index server).
#
# What is more, we should not put re-exports of all available `bin`
# structs into a single file, because that would mean eager fetches of
# all external repos even though the users are using only a single
# entry_point (or none at all, if the re-exports are done in the
# `requirements.bzl` and the user wants to use the `requirement`
# macro).
bin_content = """\
\"\"\"Autogenerated by 'pip_repository.bzl#_pkg_aliases'.\"\"\"
load("@{repo_name}_{dep}//:bin.bzl", _bin = "bin")

bin = _bin
""".format(
repo_name = repo_name,
dep = name,
)
rctx.file("{}/bin.bzl".format(name), bin_content)

def _bzlmod_pkg_aliases(repo_name, bzl_packages):
"""Create alias declarations for each python dependency.

Expand Down
32 changes: 32 additions & 0 deletions python/pip_install/tools/wheel_installer/wheel_installer.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@
import subprocess
import sys
import textwrap
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Dict, Iterable, List, Optional, Set, Tuple
Expand DownExpand Up@@ -289,6 +290,31 @@ def _generate_build_file_contents(
)


def _generate_bin_bzl_contents(entrypoints: Dict[str, str]) -> str:
"""Generate the contents of bin.bzl for each package.

The presence of the `bin.bzl` and the struct in it named `bin` becomes the API to
the users and to the hub repo on how to consume the labels to the entry_point
targets.
"""
struct_params = sorted(
[
'{attr} = Label("//:{target_name}"),'.format(
attr=attr, target_name=target_name
)
for attr, target_name in entrypoints.items()
]
)
if not struct_params:
struct_def = "struct()"
else:
struct_def = "struct(\n {}\n)" "".format("\n ".join(struct_params))

return "\n# Autogenerated by wheel_installer.py#_generate_bin_bzl_contents\n\nbin = {}\n".format(
struct_def
)


def _extract_wheel(
wheel_file: str,
extras: Dict[str, Set[str]],
Expand DownExpand Up@@ -329,6 +355,7 @@ def _extract_wheel(
]

entry_points = []
entry_point_targets = {}
for name, (module, attribute) in sorted(whl.entry_points().items()):
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
Expand All@@ -340,6 +367,8 @@ def _extract_wheel(
(installation_dir / entry_point_script_name).write_text(
_generate_entry_point_contents(module, attribute)
)

entry_point_targets[entry_point_without_py] = entry_point_target_name
entry_points.append(
_generate_entry_point_rule(
entry_point_target_name,
Expand All@@ -348,6 +377,9 @@ def _extract_wheel(
)
)

with open(os.path.join(installation_dir, "bin.bzl"), "w") as bin_bzl:
bin_bzl.write(_generate_bin_bzl_contents(entry_point_targets))

with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file:
additional_content = entry_points
data = []
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat: expose python package entrypoints via an opaque struct by aignas · Pull Request #1189 · bazel-contrib/rules_python · GitHub
Skip to content
12 changes: 12 additions & 0 deletions docs/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0

_DOCS = {
"extensions": "//docs:extensions-docs",
"packaging": "//docs:packaging-docs",
"pip": "//docs:pip-docs",
"pip_repository": "//docs:pip-repository",
Expand DownExpand Up@@ -101,6 +102,17 @@ stardoc(
deps = [":defs"],
)

stardoc(
name = "extensions-docs",
out = "extensions.md_",
input = "//python:extensions.bzl",
target_compatible_with = _NOT_WINDOWS,
deps = [
":pip_install_bzl",
"@bazel_skylib//lib:versions",
],
)

stardoc(
name = "pip-docs",
out = "pip.md_",
Expand Down
40 changes: 40 additions & 0 deletions docs/extensions.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions examples/bzlmod/BUILD.bazel
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
load("@pip//:requirements.bzl", "requirement")
load("@pip//yamllint:bin.bzl", yamllint_bin = "bin")
load("@python3_9//:defs.bzl", py_test_with_transition = "py_test")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
Expand DownExpand Up@@ -43,3 +44,11 @@ py_test_with_transition(
main = "test.py",
deps = [":lib"],
)

alias(
name = "yamllint",
# This is using the struct defined by the spoke repo 'yamllint' and it is
# re-exported by the hub repo named 'pip'. This allows bzlmod and
# non-bzlmod users to access the entry point targets.
actual = yamllint_bin.yamllint,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really follow this part -- why does it have to be put into a bzl file and loaded like this? This is pointing to a target, right? So why can't it just be "@pip//yamllint:bin"?

(I'm guessing the answer has something to do with bzlmod, repo mapping, and repo visibility, but I'd like to understand more).

)
3 changes: 3 additions & 0 deletions examples/bzlmod/MODULE.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,9 @@ register_toolchains(
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pip",
# Generate user friendly alias labels for each dependency that we have and
# enable entry_point support
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
Expand Down
8 changes: 5 additions & 3 deletions internal_deps.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,9 +53,11 @@ def rules_python_internal_deps():
maybe(
http_archive,
name = "io_bazel_stardoc",
url = "https://github.com/bazelbuild/stardoc/archive/6f274e903009158504a9d9130d7f7d5f3e9421ed.tar.gz",
sha256 = "b5d6891f869d5b5a224316ec4dd9e9d481885a9b1a1c81eb846e20180156f2fa",
strip_prefix = "stardoc-6f274e903009158504a9d9130d7f7d5f3e9421ed",
sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
"https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
],
)

# The below two deps are required for the integration test with bazel
Expand Down
3 changes: 2 additions & 1 deletion python/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ filegroup(
visibility = ["//:__pkg__"],
)

# ========= bzl_library targets end =========
# ========= bzl_library targets start =========

bzl_library(
name = "current_py_toolchain_bzl",
Expand DownExpand Up@@ -140,6 +140,7 @@ filegroup(
name = "bzl",
srcs = [
"defs.bzl",
"extensions.bzl",
"packaging.bzl",
"pip.bzl",
"repositories.bzl",
Expand Down
39 changes: 38 additions & 1 deletion python/extensions.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"Module extensions for use with bzlmod"
"""Module extensions for use with bzlmod.

# python

## python.toolchain

TODO

# pip

## pip.parse

You can use the `pip_parse` to access the generate entry_point targets as follows.
First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the
external spoke repository contents in distinct folders in the hub repo:
```starlark
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pypi",
# Generate aliases for more ergonomic consumption of dependencies from
# the `pypi` external repo.
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
use_repo(pip, "pip")
```

Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point:
```starlark
load("@pypi//flake8:bin.bzl", "bin")

alias(
name = "flake8",
actual = bin.flake8,
)
```"""

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
Expand DownExpand Up@@ -126,4 +162,5 @@ pip = module_extension(
tag_classes = {
"parse": tag_class(attrs = _pip_parse_ext_attrs()),
},
doc = "NOTE @aignas 2023-05-01: This will not appear in the docs generated with stardoc 0.5.3",
)
27 changes: 27 additions & 0 deletions python/pip_install/pip_repository.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -307,6 +307,33 @@ alias(
)
rctx.file("{}/BUILD.bazel".format(name), build_content)

# Re-export the opaque struct that contains labels to all of the
# entrypoint labels available in the package repository.
#
# NOTE: We have to do this because the hub repo created by `pip_parse`
# does not know about the existing entrypoints and therefor cannot
# generate the aliases for them. What it knows though is that the
# entrypoint labels will be available as a struct in a particular file
# when the external repository contents will be fetched from PyPI (or
# another index server).
#
# What is more, we should not put re-exports of all available `bin`
# structs into a single file, because that would mean eager fetches of
# all external repos even though the users are using only a single
# entry_point (or none at all, if the re-exports are done in the
# `requirements.bzl` and the user wants to use the `requirement`
# macro).
bin_content = """\
\"\"\"Autogenerated by 'pip_repository.bzl#_pkg_aliases'.\"\"\"
load("@{repo_name}_{dep}//:bin.bzl", _bin = "bin")

bin = _bin
""".format(
repo_name = repo_name,
dep = name,
)
rctx.file("{}/bin.bzl".format(name), bin_content)

def _bzlmod_pkg_aliases(repo_name, bzl_packages):
"""Create alias declarations for each python dependency.

Expand Down
32 changes: 32 additions & 0 deletions python/pip_install/tools/wheel_installer/wheel_installer.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@
import subprocess
import sys
import textwrap
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Dict, Iterable, List, Optional, Set, Tuple
Expand DownExpand Up@@ -289,6 +290,31 @@ def _generate_build_file_contents(
)


def _generate_bin_bzl_contents(entrypoints: Dict[str, str]) -> str:
"""Generate the contents of bin.bzl for each package.

The presence of the `bin.bzl` and the struct in it named `bin` becomes the API to
the users and to the hub repo on how to consume the labels to the entry_point
targets.
"""
struct_params = sorted(
[
'{attr} = Label("//:{target_name}"),'.format(
attr=attr, target_name=target_name
)
for attr, target_name in entrypoints.items()
]
)
if not struct_params:
struct_def = "struct()"
else:
struct_def = "struct(\n {}\n)" "".format("\n ".join(struct_params))

return "\n# Autogenerated by wheel_installer.py#_generate_bin_bzl_contents\n\nbin = {}\n".format(
struct_def
)


def _extract_wheel(
wheel_file: str,
extras: Dict[str, Set[str]],
Expand DownExpand Up@@ -329,6 +355,7 @@ def _extract_wheel(
]

entry_points = []
entry_point_targets = {}
for name, (module, attribute) in sorted(whl.entry_points().items()):
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
Expand All@@ -340,6 +367,8 @@ def _extract_wheel(
(installation_dir / entry_point_script_name).write_text(
_generate_entry_point_contents(module, attribute)
)

entry_point_targets[entry_point_without_py] = entry_point_target_name
entry_points.append(
_generate_entry_point_rule(
entry_point_target_name,
Expand All@@ -348,6 +377,9 @@ def _extract_wheel(
)
)

with open(os.path.join(installation_dir, "bin.bzl"), "w") as bin_bzl:
bin_bzl.write(_generate_bin_bzl_contents(entry_point_targets))

with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file:
additional_content = entry_points
data = []
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat: expose python package entrypoints via an opaque struct by aignas · Pull Request #1189 · bazel-contrib/rules_python · GitHub
Skip to content
12 changes: 12 additions & 0 deletions docs/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0

_DOCS = {
"extensions": "//docs:extensions-docs",
"packaging": "//docs:packaging-docs",
"pip": "//docs:pip-docs",
"pip_repository": "//docs:pip-repository",
Expand DownExpand Up@@ -101,6 +102,17 @@ stardoc(
deps = [":defs"],
)

stardoc(
name = "extensions-docs",
out = "extensions.md_",
input = "//python:extensions.bzl",
target_compatible_with = _NOT_WINDOWS,
deps = [
":pip_install_bzl",
"@bazel_skylib//lib:versions",
],
)

stardoc(
name = "pip-docs",
out = "pip.md_",
Expand Down
40 changes: 40 additions & 0 deletions docs/extensions.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions examples/bzlmod/BUILD.bazel
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
load("@pip//:requirements.bzl", "requirement")
load("@pip//yamllint:bin.bzl", yamllint_bin = "bin")
load("@python3_9//:defs.bzl", py_test_with_transition = "py_test")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
Expand DownExpand Up@@ -43,3 +44,11 @@ py_test_with_transition(
main = "test.py",
deps = [":lib"],
)

alias(
name = "yamllint",
# This is using the struct defined by the spoke repo 'yamllint' and it is
# re-exported by the hub repo named 'pip'. This allows bzlmod and
# non-bzlmod users to access the entry point targets.
actual = yamllint_bin.yamllint,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really follow this part -- why does it have to be put into a bzl file and loaded like this? This is pointing to a target, right? So why can't it just be "@pip//yamllint:bin"?

(I'm guessing the answer has something to do with bzlmod, repo mapping, and repo visibility, but I'd like to understand more).

)
3 changes: 3 additions & 0 deletions examples/bzlmod/MODULE.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,9 @@ register_toolchains(
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pip",
# Generate user friendly alias labels for each dependency that we have and
# enable entry_point support
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
Expand Down
8 changes: 5 additions & 3 deletions internal_deps.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,9 +53,11 @@ def rules_python_internal_deps():
maybe(
http_archive,
name = "io_bazel_stardoc",
url = "https://github.com/bazelbuild/stardoc/archive/6f274e903009158504a9d9130d7f7d5f3e9421ed.tar.gz",
sha256 = "b5d6891f869d5b5a224316ec4dd9e9d481885a9b1a1c81eb846e20180156f2fa",
strip_prefix = "stardoc-6f274e903009158504a9d9130d7f7d5f3e9421ed",
sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
"https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
],
)

# The below two deps are required for the integration test with bazel
Expand Down
3 changes: 2 additions & 1 deletion python/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ filegroup(
visibility = ["//:__pkg__"],
)

# ========= bzl_library targets end =========
# ========= bzl_library targets start =========

bzl_library(
name = "current_py_toolchain_bzl",
Expand DownExpand Up@@ -140,6 +140,7 @@ filegroup(
name = "bzl",
srcs = [
"defs.bzl",
"extensions.bzl",
"packaging.bzl",
"pip.bzl",
"repositories.bzl",
Expand Down
39 changes: 38 additions & 1 deletion python/extensions.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"Module extensions for use with bzlmod"
"""Module extensions for use with bzlmod.

# python

## python.toolchain

TODO

# pip

## pip.parse

You can use the `pip_parse` to access the generate entry_point targets as follows.
First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the
external spoke repository contents in distinct folders in the hub repo:
```starlark
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pypi",
# Generate aliases for more ergonomic consumption of dependencies from
# the `pypi` external repo.
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
use_repo(pip, "pip")
```

Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point:
```starlark
load("@pypi//flake8:bin.bzl", "bin")

alias(
name = "flake8",
actual = bin.flake8,
)
```"""

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
Expand DownExpand Up@@ -126,4 +162,5 @@ pip = module_extension(
tag_classes = {
"parse": tag_class(attrs = _pip_parse_ext_attrs()),
},
doc = "NOTE @aignas 2023-05-01: This will not appear in the docs generated with stardoc 0.5.3",
)
27 changes: 27 additions & 0 deletions python/pip_install/pip_repository.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -307,6 +307,33 @@ alias(
)
rctx.file("{}/BUILD.bazel".format(name), build_content)

# Re-export the opaque struct that contains labels to all of the
# entrypoint labels available in the package repository.
#
# NOTE: We have to do this because the hub repo created by `pip_parse`
# does not know about the existing entrypoints and therefor cannot
# generate the aliases for them. What it knows though is that the
# entrypoint labels will be available as a struct in a particular file
# when the external repository contents will be fetched from PyPI (or
# another index server).
#
# What is more, we should not put re-exports of all available `bin`
# structs into a single file, because that would mean eager fetches of
# all external repos even though the users are using only a single
# entry_point (or none at all, if the re-exports are done in the
# `requirements.bzl` and the user wants to use the `requirement`
# macro).
bin_content = """\
\"\"\"Autogenerated by 'pip_repository.bzl#_pkg_aliases'.\"\"\"
load("@{repo_name}_{dep}//:bin.bzl", _bin = "bin")

bin = _bin
""".format(
repo_name = repo_name,
dep = name,
)
rctx.file("{}/bin.bzl".format(name), bin_content)

def _bzlmod_pkg_aliases(repo_name, bzl_packages):
"""Create alias declarations for each python dependency.

Expand Down
32 changes: 32 additions & 0 deletions python/pip_install/tools/wheel_installer/wheel_installer.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@
import subprocess
import sys
import textwrap
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Dict, Iterable, List, Optional, Set, Tuple
Expand DownExpand Up@@ -289,6 +290,31 @@ def _generate_build_file_contents(
)


def _generate_bin_bzl_contents(entrypoints: Dict[str, str]) -> str:
"""Generate the contents of bin.bzl for each package.

The presence of the `bin.bzl` and the struct in it named `bin` becomes the API to
the users and to the hub repo on how to consume the labels to the entry_point
targets.
"""
struct_params = sorted(
[
'{attr} = Label("//:{target_name}"),'.format(
attr=attr, target_name=target_name
)
for attr, target_name in entrypoints.items()
]
)
if not struct_params:
struct_def = "struct()"
else:
struct_def = "struct(\n {}\n)" "".format("\n ".join(struct_params))

return "\n# Autogenerated by wheel_installer.py#_generate_bin_bzl_contents\n\nbin = {}\n".format(
struct_def
)


def _extract_wheel(
wheel_file: str,
extras: Dict[str, Set[str]],
Expand DownExpand Up@@ -329,6 +355,7 @@ def _extract_wheel(
]

entry_points = []
entry_point_targets = {}
for name, (module, attribute) in sorted(whl.entry_points().items()):
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
Expand All@@ -340,6 +367,8 @@ def _extract_wheel(
(installation_dir / entry_point_script_name).write_text(
_generate_entry_point_contents(module, attribute)
)

entry_point_targets[entry_point_without_py] = entry_point_target_name
entry_points.append(
_generate_entry_point_rule(
entry_point_target_name,
Expand All@@ -348,6 +377,9 @@ def _extract_wheel(
)
)

with open(os.path.join(installation_dir, "bin.bzl"), "w") as bin_bzl:
bin_bzl.write(_generate_bin_bzl_contents(entry_point_targets))

with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file:
additional_content = entry_points
data = []
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); feat: expose python package entrypoints via an opaque struct by aignas · Pull Request #1189 · bazel-contrib/rules_python · GitHub
Skip to content
12 changes: 12 additions & 0 deletions docs/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0

_DOCS = {
"extensions": "//docs:extensions-docs",
"packaging": "//docs:packaging-docs",
"pip": "//docs:pip-docs",
"pip_repository": "//docs:pip-repository",
Expand DownExpand Up@@ -101,6 +102,17 @@ stardoc(
deps = [":defs"],
)

stardoc(
name = "extensions-docs",
out = "extensions.md_",
input = "//python:extensions.bzl",
target_compatible_with = _NOT_WINDOWS,
deps = [
":pip_install_bzl",
"@bazel_skylib//lib:versions",
],
)

stardoc(
name = "pip-docs",
out = "pip.md_",
Expand Down
40 changes: 40 additions & 0 deletions docs/extensions.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions examples/bzlmod/BUILD.bazel
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
load("@pip//:requirements.bzl", "requirement")
load("@pip//yamllint:bin.bzl", yamllint_bin = "bin")
load("@python3_9//:defs.bzl", py_test_with_transition = "py_test")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
Expand DownExpand Up@@ -43,3 +44,11 @@ py_test_with_transition(
main = "test.py",
deps = [":lib"],
)

alias(
name = "yamllint",
# This is using the struct defined by the spoke repo 'yamllint' and it is
# re-exported by the hub repo named 'pip'. This allows bzlmod and
# non-bzlmod users to access the entry point targets.
actual = yamllint_bin.yamllint,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really follow this part -- why does it have to be put into a bzl file and loaded like this? This is pointing to a target, right? So why can't it just be "@pip//yamllint:bin"?

(I'm guessing the answer has something to do with bzlmod, repo mapping, and repo visibility, but I'd like to understand more).

)
3 changes: 3 additions & 0 deletions examples/bzlmod/MODULE.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,9 @@ register_toolchains(
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pip",
# Generate user friendly alias labels for each dependency that we have and
# enable entry_point support
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
Expand Down
8 changes: 5 additions & 3 deletions internal_deps.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,9 +53,11 @@ def rules_python_internal_deps():
maybe(
http_archive,
name = "io_bazel_stardoc",
url = "https://github.com/bazelbuild/stardoc/archive/6f274e903009158504a9d9130d7f7d5f3e9421ed.tar.gz",
sha256 = "b5d6891f869d5b5a224316ec4dd9e9d481885a9b1a1c81eb846e20180156f2fa",
strip_prefix = "stardoc-6f274e903009158504a9d9130d7f7d5f3e9421ed",
sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
"https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz",
],
)

# The below two deps are required for the integration test with bazel
Expand Down
3 changes: 2 additions & 1 deletion python/BUILD.bazel
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,7 @@ filegroup(
visibility = ["//:__pkg__"],
)

# ========= bzl_library targets end =========
# ========= bzl_library targets start =========

bzl_library(
name = "current_py_toolchain_bzl",
Expand DownExpand Up@@ -140,6 +140,7 @@ filegroup(
name = "bzl",
srcs = [
"defs.bzl",
"extensions.bzl",
"packaging.bzl",
"pip.bzl",
"repositories.bzl",
Expand Down
39 changes: 38 additions & 1 deletion python/extensions.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"Module extensions for use with bzlmod"
"""Module extensions for use with bzlmod.

# python

## python.toolchain

TODO

# pip

## pip.parse

You can use the `pip_parse` to access the generate entry_point targets as follows.
First, ensure you use the `incompatible_generate_aliases=True` feature to re-export the
external spoke repository contents in distinct folders in the hub repo:
```starlark
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
pip.parse(
name = "pypi",
# Generate aliases for more ergonomic consumption of dependencies from
# the `pypi` external repo.
incompatible_generate_aliases = True,
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)
use_repo(pip, "pip")
```

Then, similarly to the legacy usage, you can create an alias for the `flake8` entry_point:
```starlark
load("@pypi//flake8:bin.bzl", "bin")

alias(
name = "flake8",
actual = bin.flake8,
)
```"""

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
Expand DownExpand Up@@ -126,4 +162,5 @@ pip = module_extension(
tag_classes = {
"parse": tag_class(attrs = _pip_parse_ext_attrs()),
},
doc = "NOTE @aignas 2023-05-01: This will not appear in the docs generated with stardoc 0.5.3",
)
27 changes: 27 additions & 0 deletions python/pip_install/pip_repository.bzl
Original file line numberDiff line numberDiff line change
Expand Up@@ -307,6 +307,33 @@ alias(
)
rctx.file("{}/BUILD.bazel".format(name), build_content)

# Re-export the opaque struct that contains labels to all of the
# entrypoint labels available in the package repository.
#
# NOTE: We have to do this because the hub repo created by `pip_parse`
# does not know about the existing entrypoints and therefor cannot
# generate the aliases for them. What it knows though is that the
# entrypoint labels will be available as a struct in a particular file
# when the external repository contents will be fetched from PyPI (or
# another index server).
#
# What is more, we should not put re-exports of all available `bin`
# structs into a single file, because that would mean eager fetches of
# all external repos even though the users are using only a single
# entry_point (or none at all, if the re-exports are done in the
# `requirements.bzl` and the user wants to use the `requirement`
# macro).
bin_content = """\
\"\"\"Autogenerated by 'pip_repository.bzl#_pkg_aliases'.\"\"\"
load("@{repo_name}_{dep}//:bin.bzl", _bin = "bin")

bin = _bin
""".format(
repo_name = repo_name,
dep = name,
)
rctx.file("{}/bin.bzl".format(name), bin_content)

def _bzlmod_pkg_aliases(repo_name, bzl_packages):
"""Create alias declarations for each python dependency.

Expand Down
32 changes: 32 additions & 0 deletions python/pip_install/tools/wheel_installer/wheel_installer.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@
import subprocess
import sys
import textwrap
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Dict, Iterable, List, Optional, Set, Tuple
Expand DownExpand Up@@ -289,6 +290,31 @@ def _generate_build_file_contents(
)


def _generate_bin_bzl_contents(entrypoints: Dict[str, str]) -> str:
"""Generate the contents of bin.bzl for each package.

The presence of the `bin.bzl` and the struct in it named `bin` becomes the API to
the users and to the hub repo on how to consume the labels to the entry_point
targets.
"""
struct_params = sorted(
[
'{attr} = Label("//:{target_name}"),'.format(
attr=attr, target_name=target_name
)
for attr, target_name in entrypoints.items()
]
)
if not struct_params:
struct_def = "struct()"
else:
struct_def = "struct(\n {}\n)" "".format("\n ".join(struct_params))

return "\n# Autogenerated by wheel_installer.py#_generate_bin_bzl_contents\n\nbin = {}\n".format(
struct_def
)


def _extract_wheel(
wheel_file: str,
extras: Dict[str, Set[str]],
Expand DownExpand Up@@ -329,6 +355,7 @@ def _extract_wheel(
]

entry_points = []
entry_point_targets = {}
for name, (module, attribute) in sorted(whl.entry_points().items()):
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
Expand All@@ -340,6 +367,8 @@ def _extract_wheel(
(installation_dir / entry_point_script_name).write_text(
_generate_entry_point_contents(module, attribute)
)

entry_point_targets[entry_point_without_py] = entry_point_target_name
entry_points.append(
_generate_entry_point_rule(
entry_point_target_name,
Expand All@@ -348,6 +377,9 @@ def _extract_wheel(
)
)

with open(os.path.join(installation_dir, "bin.bzl"), "w") as bin_bzl:
bin_bzl.write(_generate_bin_bzl_contents(entry_point_targets))

with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file:
additional_content = entry_points
data = []
Expand Down