From 3eaf55d43dabe1cc0ca8aa3a9c04af1ef5de1acd Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 13:12:59 +0900 Subject: [PATCH 1/8] wip --- python/private/pypi/extension.bzl | 17 +- .../pypi/generate_whl_library_build_bazel.bzl | 153 +++++++-- python/private/pypi/hub_builder.bzl | 55 ++- python/private/pypi/labels.bzl | 2 + python/private/pypi/whl_extract.bzl | 5 + python/private/pypi/whl_library.bzl | 81 ++++- python/private/pypi/whl_library_targets.bzl | 325 ++++++++++-------- 7 files changed, 454 insertions(+), 184 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index b79d04c075..9dd50308c2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -31,7 +31,7 @@ load(":platform.bzl", _plat = "platform") load(":pypi_cache.bzl", "pypi_cache") load(":simpleapi_download.bzl", "simpleapi_download") load(":unified_hub_repo.bzl", "unified_hub_repo") -load(":whl_library.bzl", "whl_library") +load(":whl_library.bzl", "whl_library", "whl_library_deps") def _whl_mods_impl(whl_mods_dict): """Implementation of the pip.whl_mods tag class. @@ -459,14 +459,21 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + whl_library_deps_map = {} for hub in pip_hub_map.values(): out = hub.build() for whl_name, lib in out.whl_libraries.items(): if whl_name in whl_libraries: - fail("'{}' already in created".format(whl_name)) + print("'{}' already in created".format(whl_name)) + + whl_libraries[whl_name] = lib + + for deps_name, deps_args in out.whl_library_deps.items(): + if deps_name in whl_library_deps_map: + fail("'{}' already in created".format(deps_name)) else: - whl_libraries[whl_name] = lib + whl_library_deps_map[deps_name] = deps_args exposed_packages[hub.name] = out.exposed_packages extra_aliases[hub.name] = out.extra_aliases @@ -483,6 +490,7 @@ You cannot use both the additive_build_content and additive_build_content_file a hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, whl_libraries = whl_libraries, + whl_library_deps = whl_library_deps_map, whl_mods = whl_mods, platform_config_settings = { hub_name: { @@ -614,6 +622,9 @@ def _pip_impl(module_ctx): for name, args in mods.whl_libraries.items(): whl_library(name = name, **args) + for name, args in mods.whl_library_deps.items(): + whl_library_deps(name = name, **args) + for hub_name, whl_map in mods.hub_whl_map.items(): hub_repository( name = hub_name, diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 71eab26c0e..5dc70e8291 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -16,7 +16,9 @@ load("//python/private:text_util.bzl", "render") -_RENDER = { +# These are functions on how to render particular args, should be reused across all rendering +# invocations to make things easier. +_RENDER_FNS = { "copy_executables": render.dict, "copy_files": render.dict, "data": render.list, @@ -29,6 +31,11 @@ _RENDER = { "srcs_exclude": render.list, "tags": render.list, } +def _render(**kwargs): + return { + arg: _RENDER_FNS.get(arg, repr)(value) + for arg, value in kwargs.items + } # NOTE @aignas 2024-10-25: We have to keep this so that files in # this repository can be publicly visible without the need for @@ -38,19 +45,12 @@ _TEMPLATE = """\ package(default_visibility = ["//visibility:public"]) -package_metadata( - name = "package_metadata", - purl = {purl}, - visibility = ["//:__subpackages__"], -) - -{fn}( -{kwargs} -) +{macros} """ def generate_whl_library_build_bazel( *, + # TODO @aignas 2026-07-04: add extra args that are used in this function annotation = None, config_load, purl = None, @@ -72,21 +72,41 @@ def generate_whl_library_build_bazel( loads = [ """load("@package_metadata//rules:package_metadata.bzl", "package_metadata")""", + """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist")""" ] - fn = "whl_library_targets_from_requires" - if not requires_dist: - # no deps, we can leave the extra loads out - pass - else: - loads.append("""load("{}", "{}")""".format(config_load, "packages")) - kwargs["include"] = "packages" - kwargs["requires_dist"] = requires_dist - - loads.extend([ - """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "{}")""".format(fn), - ]) + srcs_kwargs = dict( + name = name, + data = [], + sdist_filename = sdist_filename, + data_exclude = list(data_exclude), + srcs_exclude = list(srcs_exclude), + tags = [ + "pypi_name={}".format(metadata_name), + "pypi_version={}".format(metadata_version), + ], + entry_points = entry_points, + enable_implicit_namespace_pkgs = enable_implicit_namespace_pkgs, + copy_files = copy_files, + copy_executables = copy_executables, + namespace_package_files = namespace_package_files, + data = [], + visibility = visibility, + ) + from_requires_kwargs = dict( + name = name, + metadata_name = metadata_name, + metadata_version = metadata_version, + requires_dist = requires_dist, + extras = extras, + group_deps = group_deps, + dep_template = dep_template, + group_name = group_name, + ) + # NOTE, if users specify annotations, the wheel downloads are not reused this + # is to ensure that we don't break users config and also to ensure that we + # can have predictable results. additional_content = [] if annotation: kwargs["data"] = annotation.data @@ -97,19 +117,96 @@ def generate_whl_library_build_bazel( if annotation.additive_build_content: additional_content.append(annotation.additive_build_content) + macro_parts = [ + render.call( + "package_metadata", + **_render( + name = "package_metadata", + purl = purl, + visibility = ["//:__subpackages__"], + ), + ), + render.call( + "whl_library_srcs", + **_render(**srcs_kwargs) + ) + ] + + if config_load: + loads.append("""load("{}", "{}")""".format(config_load, "packages")) + from_requires_kwargs["include"] = "packages" + + macro_parts.append(render.call( + "whl_library_from_requires_dist", + **_render(**from_requires_kwargs), + )) + contents = "\n".join( [ _TEMPLATE.format( loads = "\n".join(loads), - fn = fn, - kwargs = render.indent("\n".join([ - "{} = {},".format(k, _RENDER.get(k, repr)(v)) - for k, v in sorted(kwargs.items()) - ])), - purl = repr(purl), + macros = "\n\n".join(macro_parts), ), ] + additional_content, ) # NOTE: Ensure that we terminate with a new line return contents.rstrip() + "\n" + +def generate_whl_library_deps_build_bazel( + *, + # TODO @aignas 2026-07-04: add extra args that are used in this function + **kwargs): + """Generate a BUILD file for an unzipped Wheel + + + Returns: + A complete BUILD file as a string + """ + + loads = [ + """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist")""" + ] + + from_requires_kwargs = dict( + name = name, + metadata_name = metadata_name, + metadata_version = metadata_version, + requires_dist = requires_dist, + extras = extras, + group_deps = group_deps, + dep_template = dep_template, + group_name = group_name, + ) + + macro_parts = [ + render.call( + "alias", + name=target, + actual=whl_library.same_package_label(target), + ) + for target in [ + # TODO @aignas 2026-07-04: use ./labels.bzl for the following + "package_metadata", + "data", + "dist_info", + "extracted_whl_files", + ] + ] + + if config_load: + loads.append("""load("{}", "{}")""".format(config_load, "packages")) + from_requires_kwargs["include"] = "packages" + + macro_parts.append(render.call( + "whl_library_from_requires_dist", + **_render(**from_requires_kwargs), + )) + + contents = _TEMPLATE.format( + loads = "\n".join(loads), + macros = "\n\n".join(macro_parts), + ) + + # NOTE: Ensure that we terminate with a new line + return contents.rstrip() + "\n" diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 01dc8494d6..fa91a1ec03 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -71,6 +71,9 @@ def hub_builder( # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] _whl_libraries = {}, # modified by _add_whl_library + # Mapping of whl_library_deps repo names and their kwargs. + # dict[str repo_name, dict[str, object] kwargs] + _whl_library_deps = {}, # modified by _add_whl_library # Map of repos and their config settings, and repo the config # setting originated from. # dict[str whl_name, dict[str config_setting, str repo_name]] @@ -113,6 +116,7 @@ def _build(self): extra_aliases = {}, exposed_packages = [], whl_libraries = {}, + whl_library_deps = {}, ) if self._logger.failed(): return ret @@ -142,6 +146,10 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + + # Mapping of whl_library_deps repo names and their kwargs. + # dict[str repo_name, dict[str, object] kwargs] + whl_library_deps = self._whl_library_deps, ) def _pip_parse(self, module_ctx, pip_attr, python_version = None): @@ -308,17 +316,43 @@ def _add_whl_library(self, *, python_version, whl, repo): # are more platforms defined than there are wheels for and users # disallow building from sdist. return + + forbidden_args = { + "annotation": True, + "extra_pip_args": True, + "python_interpreter": True, + "python_interpreter_target": True, + } - # TODO @aignas 2025-06-29: we should not need the version in the repo_name if - # we are using pipstar and we are downloading the wheel using the downloader - # - # However, for that we should first have a different way to reference closures with - # extras. For example, if some package depends on `foo[extra]` and another depends on - # `foo`, we should have 2 py_library targets. - repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) + if [arg for arg in repo.args if arg in forbidden_args]: + # no reuse of the whl_library because we have args that force the extraction of the whl + # in the hub context. If we have whl-only pipstar extraction, then we can reuse the + # extracted sources. + repos_dict = self._whl_libraries + deps_args = repo.args + else: + whl_repo_name = "whl_{}".format(repo.whl_repo_name) - if repo_name in self._whl_libraries: - diff = _diff_dict(self._whl_libraries[repo_name], repo.args) + self._whl_libraries[whl_repo_name] = { + k: v for k, v in repo.args.items() + if k not in forbidden_args | { + "config_load": True + } + } + + args = repo.args + + deps_args = {} + for key in ("config_load", "dep_template", "group_deps", "group_name", "annotation", "pip_data_exclude"): + if key in args and args[key] != None: + deps_args[key] = args[key] + + deps_args["whl_library"] = "@{}//:BUILD.bazel".format(whl_repo_name) + repos_dict = self._whl_library_deps + + repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) + if repo_name in repos_dict: + diff = _diff_dict(repos_dict[repo_name], deps_args) if diff: self._logger.fail(lambda: ( "Attempting to create a duplicate library {repo_name} for {whl_name} with different arguments. Already existing declaration has:\n".format( @@ -331,7 +365,7 @@ def _add_whl_library(self, *, python_version, whl, repo): ]) )) return - self._whl_libraries[repo_name] = repo.args + repos_dict[repo_name] = deps_args mapping = self._whl_map.setdefault(whl.name, {}) if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: @@ -685,6 +719,7 @@ def _whl_repo( return struct( repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms), + whl_repo_name = whl_repo_name(src.filename, src.sha256), args = args, config_setting = whl_config_setting( version = python_version, diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index 8f91a03b4c..c06174d216 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -22,3 +22,5 @@ PY_LIBRARY_IMPL_LABEL = "_pkg" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" NODEPS_LABEL = "no_deps" +NODEPS_WHL_FILE_LABEL = "_whl_file" +NODEPS_PY_LIBRARY_LABEL = "_srcs" diff --git a/python/private/pypi/whl_extract.bzl b/python/private/pypi/whl_extract.bzl index 0d61b9a07b..9b4d6e6fc6 100644 --- a/python/private/pypi/whl_extract.bzl +++ b/python/private/pypi/whl_extract.bzl @@ -26,6 +26,11 @@ def whl_extract(rctx, *, whl_path, logger): logger = logger, ) + # Symlink the METADATA file to be able to refer to it from another repository_rule. This allows + # us to split the extracted sources and the closure itself to 2 different repository rules + # allowing us to not extract the same wheels multiple times. + rctx.symlink(metadata_file, "METADATA") + # Get the .dist_info dir name dist_info_dir = metadata_file.dirname rctx.file( diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 37cc36492e..a4fb5fa9f0 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -21,13 +21,13 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":attrs.bzl", "ATTRS", "use_isolated") load(":deps.bzl", "all_repo_names", "record_files") -load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") +load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel", "generate_whl_library_deps_build_bazel") load(":patch_whl.bzl", "patch_whl") load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":urllib.bzl", "urllib") load(":whl_extract.bzl", "whl_extract") -load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") +load(":whl_metadata.bzl", "parse_entry_points", "parse_whl_metadata", "whl_metadata") _CPPFLAGS = "CPPFLAGS" _COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" @@ -624,3 +624,80 @@ wheel contents without building an `sdist` first. REPO_DEBUG_ENV_VAR, ], ) + +def _whl_library_deps_impl(rctx): + logger = repo_utils.logger(rctx) + + # Load the METADATA file from a different repository + metadata_path = rctx.path(rctx.attr.whl_library.same_package_label("METADATA")) + metadata = parse_whl_metadata(rctx.read(metadata_path)) + + if not (metadata.name and metadata.version): + logger.fail("Failed to parse METADATA from {}".format(rctx.attr.whl_library)) + return + + build_file_contents = generate_whl_library_deps_build_bazel( + dep_template = rctx.attr.dep_template, + config_load = rctx.attr.config_load, + metadata_name = metadata.name, + metadata_version = metadata.version, + requires_dist = metadata.requires_dist, + group_deps = rctx.attr.group_deps, + group_name = rctx.attr.group_name, + extras = requirement(rctx.attr.requirement).extras, + whl_library=rctx.attr.whl_library, + ) + + rctx.file("WORKSPACE") + rctx.file("WORKSPACE.bazel") + rctx.file("MODULE.bazel") + rctx.file("REPO.bazel") + rctx.file("BUILD.bazel", build_file_contents) + +whl_library_deps = repository_rule( + attrs = { + "annotation": attr.label( + doc = "Optional json encoded file containing annotation to apply to the extracted wheel.", + allow_files = True, + ), + "config_load": attr.string( + doc = "The load location for configuration for pipstar.", + ), + "dep_template": attr.string( + doc = "The dep template to use for referencing the dependencies.", + ), + "extras": attr.string_list( + doc = "The list of extras.", + default = [], + ), + "group_deps": attr.string_list( + doc = "List of dependencies to skip in order to break the cycles within a dependency group.", + default = [], + ), + "group_name": attr.string( + doc = "Name of the group, if any.", + ), + "pip_data_exclude": attr.string_list( + doc = "Additional data exclude patterns.", + default = [], + ), + "whl_library": attr.label( + doc = "The whl_library repository label, use BUILD.bazel file for this.", + mandatory = True, + ), + }, + doc = """ +Uses a downloaded and extracted whl_library to generate a BUILD.bazel file with dependencies +inferred by parsing the METADATA file that comes with the wheel. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +:::{seealso} +See the {obj}`whl_library` that is used for downloading and extracting the wheel. +::: +""", + implementation = _whl_library_deps_impl, + environ = [ + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 933b529053..a6b199b481 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -24,6 +24,8 @@ load( "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", + "NODEPS_PY_LIBRARY_LABEL", + "NODEPS_WHL_FILE_LABEL", "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_FILE_IMPL_LABEL", @@ -47,14 +49,13 @@ _BAZEL_REPO_FILE_GLOBS = [ _IS_VENV_SITE_PACKAGES_YES = Label("//python/config_settings:_is_venvs_site_packages_yes") _VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages") -def whl_library_targets_from_requires( +def whl_library_from_requires_dist( *, name, metadata_name = "", metadata_version = "", requires_dist = [], extras = [], - entry_points = {}, include = [], group_deps = [], **kwargs): @@ -71,7 +72,6 @@ def whl_library_targets_from_requires( requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. - entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. **kwargs: Extra args passed to the {obj}`whl_library_targets` """ @@ -87,7 +87,6 @@ def whl_library_targets_from_requires( name = name, dependencies = package_deps.deps, dependencies_with_markers = package_deps.deps_select, - entry_points = entry_points, tags = [ "pypi_name={}".format(metadata_name), "pypi_version={}".format(metadata_version), @@ -95,6 +94,10 @@ def whl_library_targets_from_requires( **kwargs ) +def whl_library_targets_from_requires(*args, **kwargs): + """Deprecated alias for {obj}`whl_library_from_requires_dist`.""" + whl_library_from_requires_dist(*args, **kwargs) + def _parse_requires_dist( *, name, @@ -114,21 +117,165 @@ def whl_library_targets( *, name, dep_template, + tags = [], + dependencies = [], + dependencies_with_markers = {}, + group_name = "", + native = native, + src_pkg = Label("//:BUILD.bazel"), + rules = struct( + copy_file = copy_file, + py_binary = py_binary, + py_library = py_library, + venv_entry_point = venv_entry_point, + venv_rewrite_shebang = venv_rewrite_shebang, + env_marker_setting = env_marker_setting, + create_inits = _create_inits, + )): + """Create all of the whl_library targets. + + Args: + name: {type}`str` The file to match for including it into the `whl` + filegroup. This may be also parsed to generate extra metadata. + dep_template: {type}`str` The dep_template to use for dependency + interpolation. + tags: {type}`list[str]` The tags set on the `py_library`. + dependencies: {type}`list[str]` A list of dependencies. + dependencies_with_markers: {type}`dict[str, str]` A marker to evaluate + in order for the dep to be included. + group_name: {type}`str` name of the dependency group (if any) which + contains this library. If set, this library will behave as a shim + to group implementation rules which will provide simultaneously + installed dependencies which would otherwise form a cycle. + src_pkg: TODO + native: {type}`native` The native struct for overriding in tests. + rules: {type}`struct` A struct with references to rules for creating targets. + """ + src_pkg = Label(src_pkg) + + _config_settings( + dependencies_with_markers = dependencies_with_markers, + rules = rules, + visibility = ["//visibility:private"], + ) + deps_conditional = { + d: "is_include_{}_true".format(d) + for d in dependencies_with_markers + } + + # If this library is a member of a group, its public label aliases need to + # point to the group implementation rule not the implementation rules. We + # also need to mark the implementation rules as visible to the group + # implementation. + if group_name and "//:" in dep_template: + # This is the legacy behaviour where the group library is outside the hub repo + # + # It is expected to disappear when we drop WORKSPACE or drop the vendoring of + # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add + # another argument to the macro, but it is already full of arguments. + label_tmpl = dep_template.format( + name = "_config", + target = normalize_name(group_name) + "_{}", + ).replace( + "//:", + "//_groups:", + ) + impl_vis = [dep_template.format( + name = "_config", + target = "__pkg__", + ).replace( + "//:", + "//_groups:", + )] + + native.alias( + name = PY_LIBRARY_PUBLIC_LABEL, + actual = label_tmpl.format(PY_LIBRARY_PUBLIC_LABEL), + visibility = ["//visibility:public"], + ) + native.alias( + name = WHEEL_FILE_PUBLIC_LABEL, + actual = label_tmpl.format(WHEEL_FILE_PUBLIC_LABEL), + visibility = ["//visibility:public"], + ) + py_library_label = PY_LIBRARY_IMPL_LABEL + whl_file_label = WHEEL_FILE_IMPL_LABEL + + elif group_name: + py_library_label = PY_LIBRARY_PUBLIC_LABEL + whl_file_label = WHEEL_FILE_PUBLIC_LABEL + impl_vis = [dep_template.format(name = "", target = "__subpackages__")] + + else: + py_library_label = PY_LIBRARY_PUBLIC_LABEL + whl_file_label = WHEEL_FILE_PUBLIC_LABEL + impl_vis = ["//visibility:public"] + + dependencies = sorted([normalize_name(d) for d in dependencies]) + native.filegroup( + name = whl_file_label, + srcs = [src_pkg.same_package_label(NODEPS_WHL_FILE_LABEL)], + data = _deps( + deps = dependencies, + deps_conditional = deps_conditional, + tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), + ), + visibility = impl_vis, + ) + rules.py_library( + name = py_library_label, + deps = [src_pkg.same_package_label(NODEPS_PY_LIBRARY_LABEL)] + _deps( + deps = dependencies, + deps_conditional = deps_conditional, + tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), + ), + tags = tags, + visibility = impl_vis, + ) + +def _config_settings(dependencies_with_markers, rules, **kwargs): + """Generate config settings for the targets. + + Args: + dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by + each dep. + rules: used for testing + **kwargs: Extra kwargs to pass to the rule. + """ + for dep, expression in dependencies_with_markers.items(): + rules.env_marker_setting( + name = "include_{}".format(dep), + expression = expression, + **kwargs + ) + +def _deps(deps, deps_conditional, tmpl): + deps = [tmpl.format(d) for d in sorted(deps)] + + for dep, setting in deps_conditional.items(): + deps = deps + select({ + ":{}".format(setting): [tmpl.format(dep)], + "//conditions:default": [], + }) + + return deps + +def whl_library_srcs( + *, + name, sdist_filename = None, data_exclude = [], srcs_exclude = [], + copy_files = {}, + copy_executables = {}, tags = [], - dependencies = [], filegroups = None, - dependencies_with_markers = {}, entry_points = {}, - group_name = "", data = [], - copy_files = {}, - copy_executables = {}, native = native, enable_implicit_namespace_pkgs = False, namespace_package_files = [], + visibility = ["//visibility:public"], rules = struct( copy_file = copy_file, py_binary = py_binary, @@ -143,43 +290,52 @@ def whl_library_targets( Args: name: {type}`str` The file to match for including it into the `whl` filegroup. This may be also parsed to generate extra metadata. - dep_template: {type}`str` The dep_template to use for dependency - interpolation. sdist_filename: {type}`str | None` If the wheel was built from an sdist, the filename of the sdist. tags: {type}`list[str]` The tags set on the `py_library`. - dependencies: {type}`list[str]` A list of dependencies. - dependencies_with_markers: {type}`dict[str, str]` A marker to evaluate - in order for the dep to be included. entry_points: {type}`list[dict]` A list of parsed entry point definitions. filegroups: {type}`dict[str, list[str]] | None` A dictionary of the target names and the glob matches. If `None`, defaults will be used. - group_name: {type}`str` name of the dependency group (if any) which - contains this library. If set, this library will behave as a shim - to group implementation rules which will provide simultaneously - installed dependencies which would otherwise form a cycle. - copy_executables: {type}`dict[str, str]` The mapping between src and - dest locations for the targets. - copy_files: {type}`dict[str, str]` The mapping between src and - dest locations for the targets. data_exclude: {type}`list[str]` The globs for data attribute exclusion in `py_library`. srcs_exclude: {type}`list[str]` The globs for srcs attribute exclusion in `py_library`. + copy_executables: {type}`dict[str, str]` The mapping between src and + dest locations for the targets. + copy_files: {type}`dict[str, str]` The mapping between src and + dest locations for the targets. data: {type}`list[str]` A list of labels to include as part of the `data` attribute in `py_library`. enable_implicit_namespace_pkgs: {type}`boolean` generate __init__.py files for namespace pkgs. + visibility: {type}`list[str]` The visibility for the targets. native: {type}`native` The native struct for overriding in tests. namespace_package_files: {type}`list[str]` A list of labels of files whose directories are namespace packages. rules: {type}`struct` A struct with references to rules for creating targets. """ - dependencies = sorted([normalize_name(d) for d in dependencies]) tags = sorted(tags) data = [] + data bins_for_data_label = [] + for src, dest in copy_files.items(): + rules.copy_file( + name = dest + ".copy", + src = src, + out = dest, + visibility = visibility, + ) + data.append(dest) + for src, dest in copy_executables.items(): + rules.copy_file( + name = dest + ".copy", + src = src, + out = dest, + is_executable = True, + visibility = visibility, + ) + data.append(dest) + for ep_dict in entry_points.values(): kwargs = dict(ep_dict) ep_name = kwargs.pop("name") @@ -233,95 +389,14 @@ def whl_library_targets( native.filegroup( name = filegroup_name, srcs = srcs, - visibility = ["//visibility:public"], + visibility = visibility, ) - for src, dest in copy_files.items(): - rules.copy_file( - name = dest + ".copy", - src = src, - out = dest, - visibility = ["//visibility:public"], - ) - data.append(dest) - for src, dest in copy_executables.items(): - rules.copy_file( - name = dest + ".copy", - src = src, - out = dest, - is_executable = True, - visibility = ["//visibility:public"], - ) - data.append(dest) - - _config_settings( - dependencies_with_markers = dependencies_with_markers, - rules = rules, - visibility = ["//visibility:private"], - ) - deps_conditional = { - d: "is_include_{}_true".format(d) - for d in dependencies_with_markers - } - - # If this library is a member of a group, its public label aliases need to - # point to the group implementation rule not the implementation rules. We - # also need to mark the implementation rules as visible to the group - # implementation. - if group_name and "//:" in dep_template: - # This is the legacy behaviour where the group library is outside the hub repo - # - # It is expected to disappear when we drop WORKSPACE or drop the vendoring of - # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add - # another argument to the macro, but it is already full of arguments. - label_tmpl = dep_template.format( - name = "_config", - target = normalize_name(group_name) + "_{}", - ).replace( - "//:", - "//_groups:", - ) - impl_vis = [dep_template.format( - name = "_config", - target = "__pkg__", - ).replace( - "//:", - "//_groups:", - )] - - native.alias( - name = PY_LIBRARY_PUBLIC_LABEL, - actual = label_tmpl.format(PY_LIBRARY_PUBLIC_LABEL), - visibility = ["//visibility:public"], - ) - native.alias( - name = WHEEL_FILE_PUBLIC_LABEL, - actual = label_tmpl.format(WHEEL_FILE_PUBLIC_LABEL), - visibility = ["//visibility:public"], - ) - py_library_label = PY_LIBRARY_IMPL_LABEL - whl_file_label = WHEEL_FILE_IMPL_LABEL - - elif group_name: - py_library_label = PY_LIBRARY_PUBLIC_LABEL - whl_file_label = WHEEL_FILE_PUBLIC_LABEL - impl_vis = [dep_template.format(name = "", target = "__subpackages__")] - - else: - py_library_label = PY_LIBRARY_PUBLIC_LABEL - whl_file_label = WHEEL_FILE_PUBLIC_LABEL - impl_vis = ["//visibility:public"] - if hasattr(native, "filegroup"): native.filegroup( - name = whl_file_label, + name = NODEPS_WHL_FILE_LABEL, srcs = [name], - data = _deps( - deps = dependencies, - deps_conditional = deps_conditional, - tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), - ), - visibility = impl_vis, + visibility = visibility, ) if hasattr(rules, "py_library"): @@ -375,47 +450,15 @@ def whl_library_targets( data = data + [DATA_LABEL] rules.py_library( - name = py_library_label, + name = NODEPS_PY_LIBRARY_LABEL, srcs = srcs, pyi_srcs = pyi_srcs, data = data, # This makes this directory a top-level in the python import # search path for anything that depends on this. imports = ["site-packages"], - deps = _deps( - deps = dependencies, - deps_conditional = deps_conditional, - tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), - ), tags = tags, - visibility = impl_vis, + visibility = visibility, experimental_venvs_site_packages = _VENV_SITE_PACKAGES_FLAG, namespace_package_files = namespace_package_files, ) - -def _config_settings(dependencies_with_markers, rules, **kwargs): - """Generate config settings for the targets. - - Args: - dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by - each dep. - rules: used for testing - **kwargs: Extra kwargs to pass to the rule. - """ - for dep, expression in dependencies_with_markers.items(): - rules.env_marker_setting( - name = "include_{}".format(dep), - expression = expression, - **kwargs - ) - -def _deps(deps, deps_conditional, tmpl): - deps = [tmpl.format(d) for d in sorted(deps)] - - for dep, setting in deps_conditional.items(): - deps = deps + select({ - ":{}".format(setting): [tmpl.format(dep)], - "//conditions:default": [], - }) - - return deps From c5b5153d1d5504b4ff062297c0fa850ac2b474fc Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 13:52:54 +0900 Subject: [PATCH 2/8] artisanal inteligence spike --- python/private/pypi/extension.bzl | 63 ++++++++++++++++- .../pypi/generate_whl_library_build_bazel.bzl | 69 ++++++++++++------- python/private/pypi/hub_builder.bzl | 45 ++++++------ python/private/pypi/labels.bzl | 1 + python/private/pypi/whl_library.bzl | 12 ++-- python/private/pypi/whl_library_targets.bzl | 18 +++-- 6 files changed, 147 insertions(+), 61 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 9dd50308c2..7f68f1f6f4 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -22,6 +22,7 @@ load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_requires_python") load("//python/private:repo_utils.bzl", "repo_utils") +load("//python/private:text_util.bzl", "render") load(":hub_builder.bzl", "hub_builder") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") @@ -464,8 +465,26 @@ You cannot use both the additive_build_content and additive_build_content_file a out = hub.build() for whl_name, lib in out.whl_libraries.items(): + # NOTE @aignas 2026-07-04: if the same wheel is downloaded from multiple + # indexes, this will fail, forcing the user to actually download the wheel + # from the same and deterministic location. This is usually the case for + # public wheels and users should setup the defaults.index_url to correct + # fall-back in rules_python we should handle the default index to substitute + # any index-url in requirements pointing to the public PyPI mirrors. if whl_name in whl_libraries: - print("'{}' already in created".format(whl_name)) + existing = whl_libraries[whl_name] + + # TODO @aignas 2026-07-04: stop ignoring the index_url + diff = _diff_dict(existing, lib, ignore_keys = {"index_url": True}) + if diff: + fail("'{}' already in created:\n{}".format( + whl_name, + "\n".join([ + " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) + for key, value in diff.items() + if value + ]), + )) whl_libraries[whl_name] = lib @@ -1223,3 +1242,45 @@ This rule creates json files based on the whl_mods attribute. ), }, ) + +# TODO dedupe code + +def _diff_dict(first, second, *, ignore_keys = {}): + """A simple utility to shallow compare dictionaries. + + Args: + first: The first dictionary to compare. + second: The second dictionary to compare. + + Returns: + A dictionary containing the differences, with keys "common", "different", + "extra", and "missing", or None if the dictionaries are identical. + """ + missing = {} + extra = { + key: value + for key, value in second.items() + if key not in first and key not in ignore_keys + } + common = {} + different = {} + + for key, value in first.items(): + if key in ignore_keys: + continue + elif key not in second: + missing[key] = value + elif value == second[key]: + common[key] = value + else: + different[key] = (value, second[key]) + + if missing or extra or different: + return { + "common": common, + "different": different, + "extra": extra, + "missing": missing, + } + else: + return None diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 5dc70e8291..8a0c45da26 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -15,6 +15,7 @@ """Generate the BUILD.bazel contents for a repo defined by a whl_library.""" load("//python/private:text_util.bzl", "render") +load(":labels.bzl", "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", "PACKAGE_METADATA_LABEL") # These are functions on how to render particular args, should be reused across all rendering # invocations to make things easier. @@ -31,10 +32,11 @@ _RENDER_FNS = { "srcs_exclude": render.list, "tags": render.list, } + def _render(**kwargs): return { arg: _RENDER_FNS.get(arg, repr)(value) - for arg, value in kwargs.items + for arg, value in kwargs.items() } # NOTE @aignas 2024-10-25: We have to keep this so that files in @@ -50,11 +52,26 @@ package(default_visibility = ["//visibility:public"]) def generate_whl_library_build_bazel( *, - # TODO @aignas 2026-07-04: add extra args that are used in this function + name, annotation = None, config_load, + copy_executables = {}, + copy_files = {}, + data_exclude = [], + dep_template, + enable_implicit_namespace_pkgs = False, + entry_points = {}, + extras = [], + group_deps = [], + group_name = None, + metadata_name, + metadata_version, + namespace_package_files = {}, purl = None, requires_dist = [], + sdist_filename = None, + srcs_exclude = [], + visibility = ["//visibility:public"], **kwargs): """Generate a BUILD file for an unzipped Wheel @@ -72,7 +89,7 @@ def generate_whl_library_build_bazel( loads = [ """load("@package_metadata//rules:package_metadata.bzl", "package_metadata")""", - """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist")""" + """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist")""", ] srcs_kwargs = dict( @@ -90,13 +107,11 @@ def generate_whl_library_build_bazel( copy_files = copy_files, copy_executables = copy_executables, namespace_package_files = namespace_package_files, - data = [], visibility = visibility, ) from_requires_kwargs = dict( - name = name, - metadata_name = metadata_name, - metadata_version = metadata_version, + name = metadata_name, + version = metadata_version, requires_dist = requires_dist, extras = extras, group_deps = group_deps, @@ -121,15 +136,15 @@ def generate_whl_library_build_bazel( render.call( "package_metadata", **_render( - name = "package_metadata", + name = PACKAGE_METADATA_LABEL, purl = purl, visibility = ["//:__subpackages__"], - ), + ) ), render.call( "whl_library_srcs", **_render(**srcs_kwargs) - ) + ), ] if config_load: @@ -138,7 +153,7 @@ def generate_whl_library_build_bazel( macro_parts.append(render.call( "whl_library_from_requires_dist", - **_render(**from_requires_kwargs), + **_render(**from_requires_kwargs) )) contents = "\n".join( @@ -155,7 +170,15 @@ def generate_whl_library_build_bazel( def generate_whl_library_deps_build_bazel( *, - # TODO @aignas 2026-07-04: add extra args that are used in this function + name, + version, + config_load, + dep_template, + extras, + group_deps, + group_name, + requires_dist, + whl_library, **kwargs): """Generate a BUILD file for an unzipped Wheel @@ -165,32 +188,32 @@ def generate_whl_library_deps_build_bazel( """ loads = [ - """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist")""" + """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist")""", ] from_requires_kwargs = dict( name = name, - metadata_name = metadata_name, - metadata_version = metadata_version, + version = version, requires_dist = requires_dist, extras = extras, group_deps = group_deps, dep_template = dep_template, group_name = group_name, + src_pkg = str(whl_library), ) macro_parts = [ render.call( "alias", - name=target, - actual=whl_library.same_package_label(target), + **_render( + name = target, + actual = str(whl_library.same_package_label(target)), + ) ) for target in [ - # TODO @aignas 2026-07-04: use ./labels.bzl for the following - "package_metadata", - "data", - "dist_info", - "extracted_whl_files", + DATA_LABEL, + DIST_INFO_LABEL, + EXTRACTED_WHEEL_FILES, ] ] @@ -200,7 +223,7 @@ def generate_whl_library_deps_build_bazel( macro_parts.append(render.call( "whl_library_from_requires_dist", - **_render(**from_requires_kwargs), + **_render(**from_requires_kwargs) )) contents = _TEMPLATE.format( diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index fa91a1ec03..0c4fa6c0d9 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -316,7 +316,7 @@ def _add_whl_library(self, *, python_version, whl, repo): # are more platforms defined than there are wheels for and users # disallow building from sdist. return - + forbidden_args = { "annotation": True, "extra_pip_args": True, @@ -332,13 +332,14 @@ def _add_whl_library(self, *, python_version, whl, repo): deps_args = repo.args else: whl_repo_name = "whl_{}".format(repo.whl_repo_name) - - self._whl_libraries[whl_repo_name] = { - k: v for k, v in repo.args.items() + _add_library(self, repos_dict = self._whl_libraries, name = whl_repo_name, args = { + k: v + for k, v in repo.args.items() if k not in forbidden_args | { - "config_load": True + "config_load": None, + "dep_template": None, } - } + }) args = repo.args @@ -351,21 +352,7 @@ def _add_whl_library(self, *, python_version, whl, repo): repos_dict = self._whl_library_deps repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) - if repo_name in repos_dict: - diff = _diff_dict(repos_dict[repo_name], deps_args) - if diff: - self._logger.fail(lambda: ( - "Attempting to create a duplicate library {repo_name} for {whl_name} with different arguments. Already existing declaration has:\n".format( - repo_name = repo_name, - whl_name = whl.name, - ) + "\n".join([ - " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) - for key, value in diff.items() - if value - ]) - )) - return - repos_dict[repo_name] = deps_args + _add_library(self, repos_dict = repos_dict, name = repo_name, args = deps_args) mapping = self._whl_map.setdefault(whl.name, {}) if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: @@ -381,6 +368,22 @@ def _add_whl_library(self, *, python_version, whl, repo): ### end of setters, below we have various functions to implement the public methods +def _add_library(self, *, repos_dict, name, args): + if name in repos_dict: + diff = _diff_dict(repos_dict[name], args) + if diff: + self._logger.fail(lambda: ( + "Attempting to create a duplicate library {name} with different arguments. Already existing declaration has:\n".format( + repo_name = name, + ) + "\n".join([ + " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) + for key, value in diff.items() + if value + ]) + )) + return + repos_dict[name] = args + def _set_get_index_urls(self, mctx, pip_attr): # Resolve the index URL through envsubst so the ``$VAR`` / ``${VAR:-default}`` # form is honored when deciding whether the experimental index-url mode is diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index c06174d216..4baedc0b28 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -21,6 +21,7 @@ PY_LIBRARY_PUBLIC_LABEL = "pkg" PY_LIBRARY_IMPL_LABEL = "_pkg" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" +PACKAGE_METADATA_LABEL = "package_metadata" NODEPS_LABEL = "no_deps" NODEPS_WHL_FILE_LABEL = "_whl_file" NODEPS_PY_LIBRARY_LABEL = "_srcs" diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index a4fb5fa9f0..fb246538c8 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -637,15 +637,15 @@ def _whl_library_deps_impl(rctx): return build_file_contents = generate_whl_library_deps_build_bazel( - dep_template = rctx.attr.dep_template, + name = metadata.name, + version = metadata.version, config_load = rctx.attr.config_load, - metadata_name = metadata.name, - metadata_version = metadata.version, - requires_dist = metadata.requires_dist, + dep_template = rctx.attr.dep_template, + extras = rctx.attr.extras, group_deps = rctx.attr.group_deps, group_name = rctx.attr.group_name, - extras = requirement(rctx.attr.requirement).extras, - whl_library=rctx.attr.whl_library, + requires_dist = metadata.requires_dist, + whl_library = rctx.attr.whl_library, ) rctx.file("WORKSPACE") diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index a6b199b481..917685fb0b 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -51,9 +51,8 @@ _VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages") def whl_library_from_requires_dist( *, - name, - metadata_name = "", - metadata_version = "", + name = None, + version = "", requires_dist = [], extras = [], include = [], @@ -62,9 +61,8 @@ def whl_library_from_requires_dist( """The macro to create whl targets from the METADATA. Args: - name: {type}`str` The wheel filename - metadata_name: {type}`str` The package name as written in wheel `METADATA`. - metadata_version: {type}`str` The package version as written in wheel `METADATA`. + name: {type}`str` Unused + version: {type}`str` The package version as written in wheel `METADATA`. group_deps: {type}`list[str]` names of fellow members of the group (if any). These will be excluded from generated deps lists so as to avoid direct cycles. These dependencies will be provided at runtime by the @@ -76,7 +74,7 @@ def whl_library_from_requires_dist( **kwargs: Extra args passed to the {obj}`whl_library_targets` """ package_deps = _parse_requires_dist( - name = metadata_name, + name = name, requires_dist = requires_dist, excludes = group_deps, extras = extras, @@ -84,12 +82,12 @@ def whl_library_from_requires_dist( ) whl_library_targets( - name = name, + name = normalize_name(name), dependencies = package_deps.deps, dependencies_with_markers = package_deps.deps_select, tags = [ - "pypi_name={}".format(metadata_name), - "pypi_version={}".format(metadata_version), + "pypi_name={}".format(name), + "pypi_version={}".format(version), ], **kwargs ) From c3a21b3f10e7a669fe3ab2c70b5ec191b8595560 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:19:16 +0900 Subject: [PATCH 3/8] wip --- .../pypi/generate_whl_library_build_bazel.bzl | 59 +++++++++---------- python/private/pypi/whl_library.bzl | 8 ++- python/private/pypi/whl_library_targets.bzl | 8 +++ 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 8a0c45da26..407d3e8428 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -20,6 +20,7 @@ load(":labels.bzl", "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", "P # These are functions on how to render particular args, should be reused across all rendering # invocations to make things easier. _RENDER_FNS = { + "aliases": render.list, "copy_executables": render.dict, "copy_files": render.dict, "data": render.list, @@ -109,15 +110,6 @@ def generate_whl_library_build_bazel( namespace_package_files = namespace_package_files, visibility = visibility, ) - from_requires_kwargs = dict( - name = metadata_name, - version = metadata_version, - requires_dist = requires_dist, - extras = extras, - group_deps = group_deps, - dep_template = dep_template, - group_name = group_name, - ) # NOTE, if users specify annotations, the wheel downloads are not reused this # is to ensure that we don't break users config and also to ensure that we @@ -147,14 +139,25 @@ def generate_whl_library_build_bazel( ), ] - if config_load: - loads.append("""load("{}", "{}")""".format(config_load, "packages")) - from_requires_kwargs["include"] = "packages" + if dep_template: + from_requires_kwargs = dict( + name = metadata_name, + version = metadata_version, + requires_dist = requires_dist, + extras = extras, + group_deps = group_deps, + dep_template = dep_template, + group_name = group_name, + ) - macro_parts.append(render.call( - "whl_library_from_requires_dist", - **_render(**from_requires_kwargs) - )) + if config_load: + loads.append("""load("{}", "{}")""".format(config_load, "packages")) + from_requires_kwargs["include"] = "packages" + + macro_parts.append(render.call( + "whl_library_from_requires_dist", + **_render(**from_requires_kwargs) + )) contents = "\n".join( [ @@ -174,6 +177,7 @@ def generate_whl_library_deps_build_bazel( version, config_load, dep_template, + entry_points, extras, group_deps, group_name, @@ -200,31 +204,24 @@ def generate_whl_library_deps_build_bazel( dep_template = dep_template, group_name = group_name, src_pkg = str(whl_library), - ) - - macro_parts = [ - render.call( - "alias", - **_render( - name = target, - actual = str(whl_library.same_package_label(target)), - ) - ) - for target in [ + aliases = [ DATA_LABEL, DIST_INFO_LABEL, EXTRACTED_WHEEL_FILES, - ] - ] + ] + [ + "bin/{}".format(entry_point) + for entry_point in entry_points + ], + ) if config_load: loads.append("""load("{}", "{}")""".format(config_load, "packages")) from_requires_kwargs["include"] = "packages" - macro_parts.append(render.call( + macro_parts = [render.call( "whl_library_from_requires_dist", **_render(**from_requires_kwargs) - )) + )] contents = _TEMPLATE.format( loads = "\n".join(loads), diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index fb246538c8..c64765a075 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -442,7 +442,7 @@ def _whl_library_impl(rctx): sdist_filename = sdist_filename, dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format( rctx.attr.repo_prefix, - ), + ) if rctx.attr.dep_template or rctx.attr.repo_prefix else "", config_load = rctx.attr.config_load, metadata_name = metadata.name, metadata_version = metadata.version, @@ -631,6 +631,11 @@ def _whl_library_deps_impl(rctx): # Load the METADATA file from a different repository metadata_path = rctx.path(rctx.attr.whl_library.same_package_label("METADATA")) metadata = parse_whl_metadata(rctx.read(metadata_path)) + entry_points = _get_entry_points( + rctx, + rctx.path(metadata_path).dirname.get_child("site-packages"), + metadata, + ) if not (metadata.name and metadata.version): logger.fail("Failed to parse METADATA from {}".format(rctx.attr.whl_library)) @@ -644,6 +649,7 @@ def _whl_library_deps_impl(rctx): extras = rctx.attr.extras, group_deps = rctx.attr.group_deps, group_name = rctx.attr.group_name, + entry_points = [e["name"] for e in entry_points.values()], requires_dist = metadata.requires_dist, whl_library = rctx.attr.whl_library, ) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 917685fb0b..8d8ce6a510 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -120,6 +120,7 @@ def whl_library_targets( dependencies_with_markers = {}, group_name = "", native = native, + aliases = [], src_pkg = Label("//:BUILD.bazel"), rules = struct( copy_file = copy_file, @@ -231,6 +232,13 @@ def whl_library_targets( visibility = impl_vis, ) + for target in aliases: + native.alias( + name = target, + actual = src_pkg.same_package_label(target), + visibility = ["//visibility:public"], + ) + def _config_settings(dependencies_with_markers, rules, **kwargs): """Generate config settings for the targets. From 3608e1d52193536a12d251bcaeb5fccc771edfbe Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:35:02 +0900 Subject: [PATCH 4/8] fixup --- python/private/pypi/extension.bzl | 4 +-- .../pypi/generate_whl_library_build_bazel.bzl | 30 +++++++++++++++++-- python/private/pypi/hub_builder.bzl | 18 +++++++---- python/private/pypi/whl_library_targets.bzl | 1 + 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 7f68f1f6f4..59f0fc47ae 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -474,8 +474,7 @@ You cannot use both the additive_build_content and additive_build_content_file a if whl_name in whl_libraries: existing = whl_libraries[whl_name] - # TODO @aignas 2026-07-04: stop ignoring the index_url - diff = _diff_dict(existing, lib, ignore_keys = {"index_url": True}) + diff = _diff_dict(existing, lib) if diff: fail("'{}' already in created:\n{}".format( whl_name, @@ -1251,6 +1250,7 @@ def _diff_dict(first, second, *, ignore_keys = {}): Args: first: The first dictionary to compare. second: The second dictionary to compare. + ignore_keys: A set of keys to ignore during comparison. Returns: A dictionary containing the differences, with keys "common", "different", diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 407d3e8428..35c9af8567 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -77,10 +77,26 @@ def generate_whl_library_build_bazel( """Generate a BUILD file for an unzipped Wheel Args: + name: The name of the target. annotation: The annotation for the build file. config_load: {type}`str` The location from where to load the config. + copy_executables: A mapping of file paths to executable names. + copy_files: A mapping of file paths to file names. + data_exclude: A list of files to exclude from data. + dep_template: The template for dependencies. + enable_implicit_namespace_pkgs: Whether to enable implicit namespace packages. + entry_points: A mapping of entry points. + extras: A list of extras. + group_deps: A list of grouped dependencies. + group_name: The name of the group. + metadata_name: The name of the package. + metadata_version: The version of the package. + namespace_package_files: A mapping of namespace package files. purl: The purl. requires_dist: {type}`list[str]` The list of dependencies from the METADATA file. + sdist_filename: The filename of the sdist. + srcs_exclude: A list of source files to exclude. + visibility: The visibility of the target. **kwargs: Extra args serialized to be passed to the {obj}`whl_library_targets`. @@ -182,10 +198,20 @@ def generate_whl_library_deps_build_bazel( group_deps, group_name, requires_dist, - whl_library, - **kwargs): + whl_library): """Generate a BUILD file for an unzipped Wheel + Args: + name: The name of the target. + version: The version of the package. + config_load: The location from where to load the config. + dep_template: The template for dependencies. + entry_points: A mapping of entry points. + extras: A list of extras. + group_deps: A list of grouped dependencies. + group_name: The name of the group. + requires_dist: The list of dependencies from the METADATA file. + whl_library: The wheel library target. Returns: A complete BUILD file as a string diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 0c4fa6c0d9..8fa66f2c33 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -90,6 +90,7 @@ def hub_builder( # Functions to download according to the config # dict[str python_version, callable] _get_index_urls = {}, + _default_index_url = {}, # Tells whether to use the downloader for a package. # dict[str python_version, dict[str package_name, bool use_downloader]] _use_downloader = {}, @@ -385,24 +386,25 @@ def _add_library(self, *, repos_dict, name, args): repos_dict[name] = args def _set_get_index_urls(self, mctx, pip_attr): + python_version = pip_attr.python_version + # Resolve the index URL through envsubst so the ``$VAR`` / ``${VAR:-default}`` # form is honored when deciding whether the experimental index-url mode is # active. Without this, an unsubstituted template like ``$RULES_PYTHON_PIP_INDEX_URL`` # is treated as truthy and the mode is forced on, even when the env var # would expand to the empty string. - default_index_url = envsubst( + self._default_index_url[python_version] = envsubst( pip_attr.experimental_index_url, pip_attr.envsubst, mctx.getenv, ) or self._config.index_url default_extra_index_urls = pip_attr.experimental_extra_index_urls or [] - if not default_index_url: + if not self._default_index_url[python_version]: # parallel_download is set to True by default, so we are not checking/validating it # here return False - python_version = pip_attr.python_version self._use_downloader.setdefault(python_version, {}).update({ normalize_name(s): False for s in pip_attr.simpleapi_skip @@ -410,7 +412,7 @@ def _set_get_index_urls(self, mctx, pip_attr): self._get_index_urls[python_version] = lambda ctx, distributions, *, index_url = None, extra_index_urls = None: self._simpleapi_download_fn( ctx, attr = struct( - index_url = (index_url or default_index_url).rstrip("/"), + index_url = (index_url or self._default_index_url[python_version]).rstrip("/"), extra_index_urls = [ x.rstrip("/") for x in (extra_index_urls or default_extra_index_urls) @@ -579,7 +581,13 @@ def _create_whl_repos( for src in whl.srcs: repo = _whl_repo( src = src, - index_url = whl.index_url, + index_url = ( + whl.index_url or + "{}/{}".format( + self._default_index_url[python_version], + whl.name, + ) + ).rstrip("/"), whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = self._config.netrc or pip_attr.netrc, diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 8d8ce6a510..1b7b1b8756 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -146,6 +146,7 @@ def whl_library_targets( contains this library. If set, this library will behave as a shim to group implementation rules which will provide simultaneously installed dependencies which would otherwise form a cycle. + aliases: {type}`list[str]` A list of aliases to create for the target. src_pkg: TODO native: {type}`native` The native struct for overriding in tests. rules: {type}`struct` A struct with references to rules for creating targets. From 67b05581c6446905b3f30455d6e6d58b9a53a533 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:35:37 +0900 Subject: [PATCH 5/8] add a note --- python/private/pypi/hub_builder.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 8fa66f2c33..ce9ac00e73 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -581,6 +581,8 @@ def _create_whl_repos( for src in whl.srcs: repo = _whl_repo( src = src, + # TODO @aignas 2026-07-04: add a test to ensure that overriding the default + # index url overrides the values here. index_url = ( whl.index_url or "{}/{}".format( From 983aa13cb6d357bb618119867816cd4cc54abdf7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:40:01 +0900 Subject: [PATCH 6/8] fixup --- python/private/pypi/hub_builder.bzl | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index ce9ac00e73..fa51c60182 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -332,15 +332,19 @@ def _add_whl_library(self, *, python_version, whl, repo): repos_dict = self._whl_libraries deps_args = repo.args else: - whl_repo_name = "whl_{}".format(repo.whl_repo_name) - _add_library(self, repos_dict = self._whl_libraries, name = whl_repo_name, args = { - k: v - for k, v in repo.args.items() - if k not in forbidden_args | { - "config_load": None, - "dep_template": None, - } - }) + _add_library( + self, + repos_dict = self._whl_libraries, + name = repo.whl_repo_name, + args = { + k: v + for k, v in repo.args.items() + if k not in forbidden_args | { + "config_load": None, + "dep_template": None, + } + }, + ) args = repo.args From c1caf89572cd5c106e023b6a9dab92ea6427c2af Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 15:09:06 +0900 Subject: [PATCH 7/8] address todos --- python/private/pypi/whl_library_targets.bzl | 3 +- ...generate_whl_library_build_bazel_tests.bzl | 258 +++++++++++++----- 2 files changed, 193 insertions(+), 68 deletions(-) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 1b7b1b8756..d29bda4d5d 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -147,7 +147,8 @@ def whl_library_targets( to group implementation rules which will provide simultaneously installed dependencies which would otherwise form a cycle. aliases: {type}`list[str]` A list of aliases to create for the target. - src_pkg: TODO + src_pkg: {type}`Label` The label of the repository where the wheel files + are located. native: {type}`native` The native struct for overriding in tests. rules: {type}`struct` A struct with references to rules for creating targets. """ diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index 1fd99205b1..42e80da8b3 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -15,15 +15,15 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") # buildifier: disable=bzl-visibility +load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel", "generate_whl_library_deps_build_bazel") # buildifier: disable=bzl-visibility _tests = [] def _test_all_workspace(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist") load("@pypi//:config.bzl", "packages") -load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -33,36 +33,42 @@ package_metadata( visibility = ["//:__subpackages__"], ) -whl_library_targets_from_requires( - copy_executables = { - "exec_src": "exec_dest", - }, - copy_files = { - "file_src": "file_dest", - }, - data = ["extra_target"], - data_exclude = [ - "exclude_via_attr", - "data_exclude_all", +whl_library_srcs( + name = "foo.whl", + data = [], + sdist_filename = None, + data_exclude = ["exclude_via_attr"], + srcs_exclude = [], + tags = [ + "pypi_name=foo", + "pypi_version=1.0.0", ], - dep_template = "@pypi//{name}:{target}", + entry_points = {}, + enable_implicit_namespace_pkgs = False, + copy_files = {}, + copy_executables = {}, + namespace_package_files = {}, + visibility = ["//visibility:public"], +) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = [ + "foo", + "bar-baz", + "qux", + ], + extras = [], group_deps = [ "foo", "fox", "qux", ], + dep_template = "@pypi//{name}:{target}", group_name = "qux", include = packages, - name = "foo.whl", - requires_dist = [ - "foo", - "bar-baz", - "qux", - ], - srcs_exclude = ["srcs_exclude_all"], ) - -# SOMETHING SPECIAL AT THE END """ actual = generate_whl_library_build_bazel( dep_template = "@pypi//{name}:{target}", @@ -80,7 +86,12 @@ whl_library_targets_from_requires( config_load = "@pypi//:config.bzl", group_name = "qux", group_deps = ["foo", "fox", "qux"], + metadata_name = "foo", + metadata_version = "1.0.0", ) + + # Strip the trailing newline and the additional_content from actual + actual = actual.split("# SOMETHING SPECIAL AT THE END")[0].rstrip() + "\n" env.expect.that_str(actual.replace("@@", "@")).equals(want) _tests.append(_test_all_workspace) @@ -88,8 +99,8 @@ _tests.append(_test_all_workspace) def _test_all(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist") load("@pypi//:config.bzl", "packages") -load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -99,36 +110,42 @@ package_metadata( visibility = ["//:__subpackages__"], ) -whl_library_targets_from_requires( - copy_executables = { - "exec_src": "exec_dest", - }, - copy_files = { - "file_src": "file_dest", - }, - data = ["extra_target"], - data_exclude = [ - "exclude_via_attr", - "data_exclude_all", +whl_library_srcs( + name = "foo.whl", + data = [], + sdist_filename = None, + data_exclude = ["exclude_via_attr"], + srcs_exclude = [], + tags = [ + "pypi_name=foo", + "pypi_version=1.0.0", ], - dep_template = "@pypi//{name}:{target}", + entry_points = {}, + enable_implicit_namespace_pkgs = False, + copy_files = {}, + copy_executables = {}, + namespace_package_files = {}, + visibility = ["//visibility:public"], +) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = [ + "foo", + "bar-baz", + "qux", + ], + extras = [], group_deps = [ "foo", "fox", "qux", ], + dep_template = "@pypi//{name}:{target}", group_name = "qux", include = packages, - name = "foo.whl", - requires_dist = [ - "foo", - "bar-baz", - "qux", - ], - srcs_exclude = ["srcs_exclude_all"], ) - -# SOMETHING SPECIAL AT THE END """ actual = generate_whl_library_build_bazel( dep_template = "@pypi//{name}:{target}", @@ -146,7 +163,12 @@ whl_library_targets_from_requires( config_load = "@pypi//:config.bzl", group_name = "qux", group_deps = ["foo", "fox", "qux"], + metadata_name = "foo", + metadata_version = "1.0.0", ) + + # Strip the trailing newline and the additional_content from actual + actual = actual.split("# SOMETHING SPECIAL AT THE END")[0].rstrip() + "\n" env.expect.that_str(actual.replace("@@", "@")).equals(want) _tests.append(_test_all) @@ -154,8 +176,8 @@ _tests.append(_test_all) def _test_all_with_loads(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist") load("@pypi//:config.bzl", "packages") -load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -165,36 +187,42 @@ package_metadata( visibility = ["//:__subpackages__"], ) -whl_library_targets_from_requires( - copy_executables = { - "exec_src": "exec_dest", - }, - copy_files = { - "file_src": "file_dest", - }, - data = ["extra_target"], - data_exclude = [ - "exclude_via_attr", - "data_exclude_all", +whl_library_srcs( + name = "foo.whl", + data = [], + sdist_filename = None, + data_exclude = ["exclude_via_attr"], + srcs_exclude = [], + tags = [ + "pypi_name=foo", + "pypi_version=1.0.0", ], - dep_template = "@pypi//{name}:{target}", + entry_points = {}, + enable_implicit_namespace_pkgs = False, + copy_files = {}, + copy_executables = {}, + namespace_package_files = {}, + visibility = ["//visibility:public"], +) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = [ + "foo", + "bar-baz", + "qux", + ], + extras = [], group_deps = [ "foo", "fox", "qux", ], + dep_template = "@pypi//{name}:{target}", group_name = "qux", include = packages, - name = "foo.whl", - requires_dist = [ - "foo", - "bar-baz", - "qux", - ], - srcs_exclude = ["srcs_exclude_all"], ) - -# SOMETHING SPECIAL AT THE END """ actual = generate_whl_library_build_bazel( dep_template = "@pypi//{name}:{target}", @@ -212,11 +240,107 @@ whl_library_targets_from_requires( group_name = "qux", config_load = "@pypi//:config.bzl", group_deps = ["foo", "fox", "qux"], + metadata_name = "foo", + metadata_version = "1.0.0", ) + + # Strip the trailing newline and the additional_content from actual + actual = actual.split("# SOMETHING SPECIAL AT THE END")[0].rstrip() + "\n" env.expect.that_str(actual.replace("@@", "@")).equals(want) _tests.append(_test_all_with_loads) +def _test_generate_whl_library_deps_build_bazel(env): + want = """\ +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist") + +package(default_visibility = ["//visibility:public"]) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = [], + extras = [], + group_deps = [], + dep_template = "template", + group_name = None, + src_pkg = "@//:pkg", + aliases = [ + "data", + "dist_info", + "extracted_whl_files", + ], +) +""" + actual = generate_whl_library_deps_build_bazel( + name = "foo", + version = "1.0.0", + config_load = None, + dep_template = "template", + entry_points = [], + extras = [], + group_deps = [], + group_name = None, + requires_dist = [], + whl_library = "@@//:pkg", + ) + env.expect.that_str(actual.replace("@@", "@")).equals(want) + +def _test_no_annotation(env): + want = """\ +load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist") + +package(default_visibility = ["//visibility:public"]) + +package_metadata( + name = "package_metadata", + purl = None, + visibility = ["//:__subpackages__"], +) + +whl_library_srcs( + name = "foo.whl", + data = [], + sdist_filename = None, + data_exclude = [], + srcs_exclude = [], + tags = [ + "pypi_name=foo", + "pypi_version=1.0.0", + ], + entry_points = {}, + enable_implicit_namespace_pkgs = False, + copy_files = {}, + copy_executables = {}, + namespace_package_files = {}, + visibility = ["//visibility:public"], +) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = ["foo"], + extras = [], + group_deps = [], + dep_template = "@pypi//{name}:{target}", + group_name = None, +) +""" + actual = generate_whl_library_build_bazel( + dep_template = "@pypi//{name}:{target}", + name = "foo.whl", + requires_dist = ["foo"], + annotation = None, + config_load = None, + metadata_name = "foo", + metadata_version = "1.0.0", + ) + env.expect.that_str(actual).equals(want) + +_tests.append(_test_generate_whl_library_deps_build_bazel) +_tests.append(_test_no_annotation) + def generate_whl_library_build_bazel_test_suite(name): """Create the test suite. From a014b8a8b3c0d889d5c1489bfa873b393cf67291 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 16:06:10 +0900 Subject: [PATCH 8/8] fixup - add index_url handling to uv.lock --- python/private/pypi/hub_builder.bzl | 15 ++++----------- python/private/pypi/parse_requirements.bzl | 8 +++++++- python/private/pypi/simpleapi_download.bzl | 3 +++ tests/pypi/hub_builder/hub_builder_tests.bzl | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index fa51c60182..237fa501e2 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -353,7 +353,7 @@ def _add_whl_library(self, *, python_version, whl, repo): if key in args and args[key] != None: deps_args[key] = args[key] - deps_args["whl_library"] = "@{}//:BUILD.bazel".format(whl_repo_name) + deps_args["whl_library"] = "@{}//:BUILD.bazel".format(repo.whl_repo_name) repos_dict = self._whl_library_deps repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) @@ -379,13 +379,14 @@ def _add_library(self, *, repos_dict, name, args): if diff: self._logger.fail(lambda: ( "Attempting to create a duplicate library {name} with different arguments. Already existing declaration has:\n".format( - repo_name = name, + name = name, ) + "\n".join([ " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) for key, value in diff.items() if value ]) )) + return repos_dict[name] = args @@ -585,15 +586,7 @@ def _create_whl_repos( for src in whl.srcs: repo = _whl_repo( src = src, - # TODO @aignas 2026-07-04: add a test to ensure that overriding the default - # index url overrides the values here. - index_url = ( - whl.index_url or - "{}/{}".format( - self._default_index_url[python_version], - whl.name, - ) - ).rstrip("/"), + index_url = whl.index_url.rstrip("/"), whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = self._config.netrc or pip_attr.netrc, diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index fd0399fc86..c87c586f7e 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -176,6 +176,12 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p "versions": {}, }) entry["versions"][version] = None + registry = pkg.get("source", {}).get("registry", "") + if registry.rstrip("/").endswith("simple"): + index_url = "{}/{}".format(registry, norm_name.replace("_", "-")) + else: + index_url = "" + entry["index_url"] = index_url pkg_extras = sorted(extras_map.get(name, [])) extra_str = "[{}]".format(",".join(pkg_extras)) if pkg_extras else "" @@ -287,7 +293,7 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p name = norm_name, is_exposed = True, is_multiple_versions = len(versions) > 1, - index_url = "", + index_url = info["index_url"], srcs = info["resolved_srcs"], ) ret.append(item) diff --git a/python/private/pypi/simpleapi_download.bzl b/python/private/pypi/simpleapi_download.bzl index 5377a08093..03c9dc3754 100644 --- a/python/private/pypi/simpleapi_download.bzl +++ b/python/private/pypi/simpleapi_download.bzl @@ -291,6 +291,9 @@ def _with_index_url(index_url, values): if not values: return values + if not index_url: + fail("BUG: no index_url") + return struct( sdists = values.sdists, whls = values.whls, diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 60017593fb..ca5c77b97c 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -1498,7 +1498,7 @@ def _test_err_duplicate_repos(env): env.expect.that_dict(logs).keys().contains_exactly(["rules_python:unit-test FAIL:"]) env.expect.that_collection(logs["rules_python:unit-test FAIL:"]).contains_exactly([ """\ -Attempting to create a duplicate library pypi_315_foo for foo with different arguments. Already existing declaration has: +Attempting to create a duplicate library pypi_315_foo with different arguments. Already existing declaration has: common: { "dep_template": "@pypi//{name}:{target}", "config_load": "@pypi//:config.bzl",