feat: enable pyproject.toml as single source of truth for Python version

news/3514.added.md

@@ -0,0 +1 @@
+(pip,python) Added `pyproject_toml` attribute to {obj}`pip.default`, {obj}`pip.parse` and {obj}`python.defaults` to read the default Python version from the `requires-python` field of `pyproject.toml`.

python/private/BUILD.bazel

@@ -709,6 +709,7 @@ bzl_library(
         ":full_version",
         ":pbs_manifest",
         ":platform_info",
+        ":pyproject_utils",
         ":python_register_toolchains",
         ":pythons_hub",
         ":repo_utils",
@@ -876,6 +877,15 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "pyproject_utils",
+    srcs = ["pyproject_utils.bzl"],
+    deps = [
+        ":version",
+        "@toml.bzl//:toml",
+    ],
+)
+
 bzl_library(
     name = "bzlmod_enabled",
     srcs = ["bzlmod_enabled.bzl"],

python/private/pypi/BUILD.bazel

@@ -125,6 +125,7 @@ bzl_library(
         ":whl_library",
         "//python/private:auth",
         "//python/private:normalize_name",
+        "//python/private:pyproject_utils",
         "//python/private:repo_utils",
         "@pythons_hub//:interpreters",
         "@pythons_hub//:versions",

python/private/pypi/extension.bzl

@@ -20,6 +20,7 @@ load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config")
 load("@toml.bzl", "toml")
 load("//python/private:auth.bzl", "AUTH_ATTRS")
 load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_requires_python")
 load("//python/private:repo_utils.bzl", "repo_utils")
 load(":hub_builder.bzl", "hub_builder")
 load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
@@ -208,6 +209,7 @@ def build_config(
     default_hub = None
     defaults = {
         "platforms": default_platforms(),
+        "python_version": None,
     }
     for mod in module_ctx.modules:
         if not (mod.is_root or mod.name == "rules_python"):
@@ -219,6 +221,11 @@ def build_config(
                     if default_hub:
                         fail("Duplicate pip.default tag: only one explicit default PyPI hub is allowed.")
                     default_hub = tag.default_hub
+            pyproject_toml = tag.pyproject_toml
+            if pyproject_toml:
+                pyproject = read_pyproject(module_ctx, pyproject_toml)
+                if pyproject.requires_python:
+                    defaults["python_version"] = version_from_requires_python(pyproject.requires_python)
 
             platform = tag.platform
             if platform:
@@ -256,6 +263,7 @@ def build_config(
         default_hub = default_hub,
         index_url = defaults.get("index_url", "https://pypi.org/simple").rstrip("/"),
         netrc = defaults.get("netrc", None),
+        python_version = defaults.get("python_version", None),
         platforms = {
             name: _plat(**values)
             for name, values in defaults["platforms"].items()
@@ -359,6 +367,15 @@ You cannot use both the additive_build_content and additive_build_content_file a
 
     for mod in module_ctx.modules:
         for pip_attr in mod.tags.parse:
+            python_version = pip_attr.python_version
+            if not python_version and pip_attr.pyproject_toml:
+                pyproject = read_pyproject(module_ctx, pip_attr.pyproject_toml)
+                if pyproject.requires_python:
+                    python_version = version_from_requires_python(pyproject.requires_python)
+            python_version = python_version or config.python_version
+            if not python_version:
+                _fail("pip.parse() requires one of `python_version`, `pyproject_toml`, or `pip.default(pyproject_toml=...)` to be set")
+
             hub_name = pip_attr.hub_name
             if hub_name == "pypi":
                 if is_pypi_hub_reserved:
@@ -422,6 +439,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
             builder.pip_parse(
                 module_ctx,
                 pip_attr = pip_attr,
+                python_version = python_version,
             )
 
     # Keeps track of all the hub's whl repos across the different versions.
@@ -699,6 +717,23 @@ If you are defining custom platforms in your project and don't want things to cl
 [isolation] feature.
 
 [isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate
+""",
+    ),
+    "pyproject_toml": attr.label(
+        mandatory = False,
+        doc = """\
+Label pointing to pyproject.toml file to read the default Python version from.
+When specified, reads the `requires-python` field from pyproject.toml and uses
+it as the default python_version for all `pip.parse()` calls that don't
+explicitly specify one.
+
+:::{note}
+The version must be specified as `==X.Y.Z` (exact version with full semver).
+This is designed to work with dependency management tools like Renovate.
+:::
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
 """,
     ),
     "whl_abi_tags": attr.string_list(
@@ -869,15 +904,39 @@ find in case extra indexes are specified.
 """,
             default = True,
         ),
+        "pyproject_toml": attr.label(
+            mandatory = False,
+            doc = """\
+Label pointing to a pyproject.toml file to read the Python version from.
+When specified, the `requires-python` field is used as the `python_version`
+for this `pip.parse()` call, unless `python_version` is set explicitly.
+
+:::{note}
+The version must be specified as `==X.Y.Z` (exact version with full semver).
+:::
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+""",
+        ),
         "python_version": attr.string(
-            mandatory = True,
+            mandatory = False,
             doc = """
 The Python version the dependencies are targetting, in Major.Minor format
 (e.g., "3.11") or patch level granularity (e.g. "3.11.1").
 
 If an interpreter isn't explicitly provided (using `python_interpreter` or
 `python_interpreter_target`), then the version specified here must have
 a corresponding `python.toolchain()` configured.
+
+:::{seealso}
+The {obj}`pyproject_toml` attribute for getting the version from a project file.
+:::
+
+:::{versionchanged} VERSION_NEXT_FEATURE
+No longer mandatory if the {obj}`pyproject_toml` attribute or
+{obj}`pip.default.pyproject_toml` is specified.
+:::
 """,
         ),
         "simpleapi_skip": attr.string_list(

python/private/pypi/hub_builder.bzl

@@ -144,8 +144,8 @@ def _build(self):
         whl_libraries = self._whl_libraries,
     )
 
-def _pip_parse(self, module_ctx, pip_attr):
-    python_version = pip_attr.python_version
+def _pip_parse(self, module_ctx, pip_attr, python_version = None):
+    python_version = python_version or pip_attr.python_version
     if python_version in self._platforms:
         fail((
             "Duplicate pip python version '{version}' for hub " +
@@ -191,7 +191,8 @@ def _pip_parse(self, module_ctx, pip_attr):
         self,
         module_ctx,
         pip_attr = pip_attr,
-        enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(pip_attr.python_version)),
+        python_version = python_version,
+        enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(python_version)),
     )
 
 ### end of PUBLIC methods
@@ -393,11 +394,11 @@ def _set_get_index_urls(self, mctx, pip_attr):
     )
     return True
 
-def _detect_interpreter(self, pip_attr):
+def _detect_interpreter(self, pip_attr, python_version):
     python_interpreter_target = pip_attr.python_interpreter_target
     if python_interpreter_target == None and not pip_attr.python_interpreter:
         python_name = "python_{}_host".format(
-            pip_attr.python_version.replace(".", "_"),
+            python_version.replace(".", "_"),
         )
         if python_name not in self._available_interpreters:
             fail((
@@ -407,7 +408,7 @@ def _detect_interpreter(self, pip_attr):
                 "Expected to find {python_name} among registered versions:\n  {labels}"
             ).format(
                 hub_name = self.name,
-                version = pip_attr.python_version,
+                version = python_version,
                 python_name = python_name,
                 labels = "  \n".join(self._available_interpreters),
             ))
@@ -476,17 +477,19 @@ def _create_whl_repos(
         module_ctx,
         *,
         pip_attr,
+        python_version,
         enable_pipstar_extract = False):
     """create all of the whl repositories
 
     Args:
         self: the builder.
         module_ctx: {type}`module_ctx`.
         pip_attr: {type}`struct` - the struct that comes from the tag class iteration.
+        python_version: {type}`str` - the resolved python version for this pip.parse call.
         enable_pipstar_extract: {type}`bool` - enable the pipstar extraction or not.
     """
     logger = self._logger
-    platforms = self._platforms[pip_attr.python_version]
+    platforms = self._platforms[python_version]
     requirements_by_platform = parse_requirements(
         module_ctx,
         requirements_by_platform = requirements_files_by_platform(
@@ -498,7 +501,7 @@ def _create_whl_repos(
             extra_pip_args = pip_attr.extra_pip_args,
             platforms = sorted(platforms),  # here we only need keys
             python_version = full_version(
-                version = pip_attr.python_version,
+                version = python_version,
                 minor_mapping = self._minor_mapping,
             ),
             logger = logger,
@@ -528,7 +531,7 @@ def _create_whl_repos(
         pip_attr = pip_attr,
     )
 
-    interpreter = _detect_interpreter(self, pip_attr)
+    interpreter = _detect_interpreter(self, pip_attr, python_version)
 
     for whl in requirements_by_platform:
         whl_library_args = common_args | _whl_library_args(
@@ -543,16 +546,16 @@ def _create_whl_repos(
                 whl_library_args = whl_library_args,
                 download_only = pip_attr.download_only,
                 netrc = self._config.netrc or pip_attr.netrc,
-                use_downloader = src.url and _use_downloader(self, pip_attr.python_version, whl.name),
+                use_downloader = src.url and _use_downloader(self, python_version, whl.name),
                 auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns,
-                python_version = _major_minor_version(pip_attr.python_version),
+                python_version = _major_minor_version(python_version),
                 is_multiple_versions = whl.is_multiple_versions,
                 interpreter = interpreter,
                 enable_pipstar_extract = enable_pipstar_extract,
             )
             _add_whl_library(
                 self,
-                python_version = pip_attr.python_version,
+                python_version = python_version,
                 whl = whl,
                 repo = repo,
             )

python/private/pyproject_utils.bzl

@@ -0,0 +1,51 @@
+"""Utilities for reading values from pyproject.toml."""
+
+load("@toml.bzl", "toml")
+load(":version.bzl", "version")
+
+def read_pyproject(module_ctx, pyproject):
+    """Read a pyproject.toml file and return the relevant fields.
+
+    The file is parsed with a pure-Starlark TOML decoder; no Python
+    interpreter is required. The raw `requires-python` value is returned
+    as-is so that callers can decide how to interpret it.
+
+    Args:
+        module_ctx: the module extension context (needs `path` and `read`).
+        pyproject: {type}`Label` pointing at the pyproject.toml file.
+
+    Returns:
+        {type}`struct` with the attributes:
+          * `requires_python`: {type}`str | None` the raw `requires-python`
+            value (e.g. `"==3.13.9"`), or `None` if it is not set.
+    """
+    data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes"))
+    return struct(
+        requires_python = data.get("project", {}).get("requires-python"),
+    )
+
+def version_from_requires_python(requires_python):
+    """Derive a concrete Python version from a `requires-python` value.
+
+    Currently only an exact `==X.Y.Z` specifier is supported. The value is
+    validated and normalized via {obj}`//python/private:version.bzl` so that
+    malformed input fails in a consistent way. Broader specifier support
+    (e.g. `>=`, `X.Y`) can be layered on here in the future.
+
+    Args:
+        requires_python: {type}`str` the raw `requires-python` value.
+
+    Returns:
+        {type}`str` the normalized version string (e.g. `"3.13.9"`).
+    """
+    if not requires_python:
+        fail("`requires-python` must be specified")
+
+    if not requires_python.startswith("=="):
+        fail("`requires-python` must pin an exact version with `==`, got: {}".format(requires_python))
+
+    bare_version = requires_python[len("=="):].strip()
+
+    # Parse strictly so malformed versions fail cleanly, then normalize.
+    version.parse(bare_version, strict = True)
+    return version.normalize(bare_version)