Merge remote-tracking branch 'upstream/main' into HEAD

Author: philsc

CHANGELOG.md

@@ -57,16 +57,30 @@ Unreleased changes template.
 * (rules) deprecation warnings for deprecated symbols have been turned off by
   default for now and can be enabled with `RULES_PYTHON_DEPRECATION_WARNINGS`
   env var.
+* (pypi) Downgraded versions of packages: `pip` from `24.3.2` to `24.0.0` and
+  `packaging` from `24.2` to `24.0`.
 
 {#v0-0-0-fixed}
 ### Fixed
+* (rules) `python_zip_file` output with `--bootstrap_impl=script` works again
+  ([#2596](https://github.com/bazelbuild/rules_python/issues/2596)).
+* (docs) Using `python_version` attribute for specifying python versions introduced in `v1.1.0`
 * (gazelle) Providing multiple input requirements files to `gazelle_python_manifest` now works correctly.
 * (pypi) Handle trailing slashes in pip index URLs in environment variables,
   fixes [#2554](https://github.com/bazelbuild/rules_python/issues/2554).
 * (runfiles) Runfile manifest and repository mapping files are now interpreted
   as UTF-8 on all platforms.
 * (coverage) Coverage with `--bootstrap_impl=script` is fixed
   ([#2572](https://github.com/bazelbuild/rules_python/issues/2572)).
+* (pypi) Non deterministic behaviour in requirement file usage has been fixed
+  by reverting [#2514](https://github.com/bazelbuild/rules_python/pull/2514).
+  The related issue is [#908](https://github.com/bazelbuild/rules_python/issue/908).
+* (sphinxdocs) Do not crash when `tag_class` does not have a populated `doc` value.
+  Fixes ([#2579](https://github.com/bazelbuild/rules_python/issues/2579)).
+* (binaries/tests) Fix packaging when using `--bootstrap_impl=script`: set
+  {obj}`--venvs_use_declare_symlink=no` to have it not create symlinks at
+  build time (they will be created at runtime instead).
+  (Fixes [#2489](https://github.com/bazelbuild/rules_python/issues/2489))
 
 {#v0-0-0-added}
 ### Added

MODULE.bazel

@@ -84,6 +84,7 @@ bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True)
 bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True)
 bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True)
 bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
+bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True)
 
 # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests.
 # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides.
@@ -175,7 +176,7 @@ use_repo(
 
 # EXPERIMENTAL: This is experimental and may be removed without notice
 uv = use_extension(
-    "//python/uv:extensions.bzl",
+    "//python/uv:uv.bzl",
     "uv",
     dev_dependency = True,
 )

docs/BUILD.bazel

@@ -16,7 +16,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
 load("@dev_pip//:requirements.bzl", "requirement")
 load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")  # buildifier: disable=bzl-visibility
 load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")  # buildifier: disable=bzl-visibility
-load("//python/uv/private:lock.bzl", "lock")  # buildifier: disable=bzl-visibility
+load("//python/uv:lock.bzl", "lock")  # buildifier: disable=bzl-visibility
 load("//sphinxdocs:readthedocs.bzl", "readthedocs_install")
 load("//sphinxdocs:sphinx.bzl", "sphinx_build_binary", "sphinx_docs")
 load("//sphinxdocs:sphinx_docs_library.bzl", "sphinx_docs_library")
@@ -105,6 +105,10 @@ sphinx_stardocs(
         "//python/private/api:py_common_api_bzl",
         "//python/private/pypi:config_settings_bzl",
         "//python/private/pypi:pkg_aliases_bzl",
+        "//python/uv:lock_bzl",
+        "//python/uv:uv_bzl",
+        "//python/uv:uv_toolchain_bzl",
+        "//python/uv:uv_toolchain_info_bzl",
     ] + ([
         # Bazel 6 + Stardoc isn't able to parse something about the python bzlmod extension
         "//python/extensions:python_bzl",

docs/_includes/py_console_script_binary.md

@@ -12,7 +12,8 @@ py_console_script_binary(
 )
 ```
 
-Or for more advanced setups you can also specify extra dependencies and the
+#### Specifying extra dependencies 
+You can also specify extra dependencies and the
 exact script name you want to call. It is useful for tools like `flake8`, `pylint`,
 `pytest`, which have plugin discovery methods and discover dependencies from the
 PyPI packages available in the `PYTHONPATH`.
@@ -34,17 +35,26 @@ py_console_script_binary(
 )
 ```
 
-A specific Python version can be forced by using the generated version-aware
-wrappers, e.g. to force Python 3.9:
+#### Using a specific Python version
+
+A specific Python version can be forced by passing the desired Python version, e.g. to force Python 3.9:
 ```starlark
-load("@python_versions//3.9:defs.bzl", "py_console_script_binary")
+load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
 
 py_console_script_binary(
     name = "yamllint",
     pkg = "@pip//yamllint",
+    python_version = "3.9"
 )
 ```
 
+#### Using a specific Python Version directly from a Toolchain
+:::{deprecated} 1.1.0
+The toolchain specific `py_binary` and `py_test` symbols are aliases to the regular rules. 
+i.e. Deprecated `load("@python_versions//3.11:defs.bzl", "py_binary")` and `load("@python_versions//3.11:defs.bzl", "py_test")`
+
+You should instead specify the desired python version with `python_version`; see above example.
+:::
 Alternatively, the [`py_console_script_binary.binary_rule`] arg can be passed
 the version-bound `py_binary` symbol, or any other `py_binary`-compatible rule
 of your choosing:
@@ -60,5 +70,4 @@ py_console_script_binary(
 ```
 
 [specification]: https://packaging.python.org/en/latest/specifications/entry-points/
-[`py_console_script_binary.binary_rule`]: #py_console_script_binary_binary_rule
-
+[`py_console_script_binary.binary_rule`]: #py_console_script_binary_binary_rule

docs/api/rules_python/python/config_settings/index.md

@@ -212,6 +212,7 @@ Values:
 :::
 ::::
 
+
 ::::{bzl:flag} bootstrap_impl
 Determine how programs implement their startup process.
 
@@ -258,3 +259,26 @@ Values:
 :::
 
 ::::
+
+::::{bzl:flag} venvs_use_declare_symlink
+
+Determines if relative symlinks are created using `declare_symlink()` at build
+time.
+
+This is only intended to work around
+[#2489](https://github.com/bazelbuild/rules_python/issues/2489), where some
+packaging rules don't support `declare_symlink()` artifacts.
+
+Values:
+* `yes`: Use `declare_symlink()` and create relative symlinks at build time.
+* `no`: Do not use `declare_symlink()`. Instead, the venv will be created at
+  runtime.
+
+:::{seealso}
+{envvar}`RULES_PYTHON_EXTRACT_ROOT` for customizing where the runtime venv
+is created.
+:::
+
+:::{versionadded} VERSION_NEXT_PATCH
+:::
+::::

docs/environment-variables.md

@@ -1,28 +1,56 @@
 # Environment Variables
 
-:::{envvar} RULES_PYTHON_REPO_DEBUG
+:::{envvar} RULES_PYTHON_BOOTSTRAP_VERBOSE
 
-When `1`, repository rules will print debug information about what they're
+When `1`, debug information about bootstrapping of a program is printed to
+stderr.
+:::
+
+:::{envvar} RULES_PYTHON_BZLMOD_DEBUG
+
+When `1`, bzlmod extensions will print debug information about what they're
 doing. This is mostly useful for development to debug errors.
 :::
 
-:::{envvar} RULES_PYTHON_REPO_DEBUG_VERBOSITY
+:::{envvar} RULES_PYTHON_DEPRECATION_WARNINGS
 
-Determines the verbosity of logging output for repo rules. Valid values:
+When `1`, the rules_python will warn users about deprecated functionality that will
+be removed in a subsequent major `rules_python` version. Defaults to `0` if unset.
+:::
 
-* `DEBUG`
-* `INFO`
-* `TRACE`
+:::{envvar} RULES_PYTHON_ENABLE_PYSTAR
+
+When `1`, the rules_python Starlark implementation of the core rules is used
+instead of the Bazel-builtin rules. Note this requires Bazel 7+.
 :::
 
-:::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH
+::::{envvar} RULES_PYTHON_EXTRACT_ROOT
 
-Determines the python interpreter platform to be used for a particular
-interpreter `(version, os, arch)` triple to be used in repository rules.
-Replace the `VERSION_OS_ARCH` part with actual values when using, e.g.
-`3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the
-os, arch values are the same as the ones mentioned in the
-`//python:versions.bzl` file.
+Directory to use as the root for creating files necessary for bootstrapping so
+that a binary can run.
+
+Only applicable when {bzl:flag}`--venvs_use_declare_symlink=no` is used.
+
+When set, a binary will attempt to find a unique, reusable, location within this
+directory for the files it needs to create to aid startup. The files may not be
+deleted upon program exit; it is the responsibility of the caller to ensure
+cleanup.
+
+Manually specifying the directory is useful to lower the overhead of
+extracting/creating files on every program execution. By using a location
+outside /tmp, longer lived programs don't have to worry about files in /tmp
+being cleaned up by the OS.
+
+If not set, then a temporary directory will be created and deleted upon program
+exit.
+
+:::{versionadded} VERSION_NEXT_PATCH
+:::
+::::
+
+:::{envvar} RULES_PYTHON_GAZELLE_VERBOSE
+
+When `1`, debug information from gazelle is printed to stderr.
 :::
 
 :::{envvar} RULES_PYTHON_PIP_ISOLATED
@@ -34,37 +62,32 @@ Valid values:
 * Other non-empty values mean to use isolated mode.
 :::
 
-:::{envvar} RULES_PYTHON_BZLMOD_DEBUG
+:::{envvar} RULES_PYTHON_REPO_DEBUG
 
-When `1`, bzlmod extensions will print debug information about what they're
+When `1`, repository rules will print debug information about what they're
 doing. This is mostly useful for development to debug errors.
 :::
 
-:::{envvar} RULES_PYTHON_DEPRECATION_WARNINGS
-
-When `1`, the rules_python will warn users about deprecated functionality that will
-be removed in a subsequent major `rules_python` version. Defaults to `0` if unset.
-:::
+:::{envvar} RULES_PYTHON_REPO_DEBUG_VERBOSITY
 
-:::{envvar} RULES_PYTHON_ENABLE_PYSTAR
+Determines the verbosity of logging output for repo rules. Valid values:
 
-When `1`, the rules_python Starlark implementation of the core rules is used
-instead of the Bazel-builtin rules. Note this requires Bazel 7+.
+* `DEBUG`
+* `INFO`
+* `TRACE`
 :::
 
-:::{envvar} RULES_PYTHON_BOOTSTRAP_VERBOSE
+:::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH
 
-When `1`, debug information about bootstrapping of a program is printed to
-stderr.
+Determines the python interpreter platform to be used for a particular
+interpreter `(version, os, arch)` triple to be used in repository rules.
+Replace the `VERSION_OS_ARCH` part with actual values when using, e.g.
+`3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the
+os, arch values are the same as the ones mentioned in the
+`//python:versions.bzl` file.
 :::
 
 :::{envvar} VERBOSE_COVERAGE
 
 When `1`, debug information about coverage behavior is printed to stderr.
 :::
-
-
-:::{envvar} RULES_PYTHON_GAZELLE_VERBOSE
-
-When `1`, debug information from gazelle is printed to stderr.
-:::

docs/toolchains.md

@@ -116,9 +116,9 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(python_version = "3.12")
 
 # BUILD.bazel
-load("@python_versions//3.12:defs.bzl", "py_binary")
+load("@rules_python//python:py_binary.bzl", "py_binary")
 
-py_binary(...)
+py_binary(..., python_version="3.12")
 ```
 
 ### Pinning to a Python version
@@ -132,21 +132,59 @@ is most useful for two cases:
    typically in a mono-repo situation.
 
 To configure a submodule with the version-aware rules, request the particular
-version you need, then use the `@python_versions` repo to use the rules that
-force specific versions:
+version you need when defining the toolchain:
 
 ```starlark
+# MODULE.bazel
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 
 python.toolchain(
     python_version = "3.11",
 )
-use_repo(python, "python_versions")
+use_repo(python)
+```
+
+Then use the `@rules_python` repo in your BUILD file to explicity pin the Python version when calling the rule:
+
+```starlark
+# BUILD.bazel
+load("@rules_python//python:py_binary.bzl", "py_binary")
+
+py_binary(..., python_version = "3.11")
+py_test(..., python_version = "3.11")
 ```
 
-Then use e.g. `load("@python_versions//3.11:defs.bzl", "py_binary")` to use
-the rules that force that particular version. Multiple versions can be specified
-and use within a single build.
+Multiple versions can be specified and used within a single build.
+
+```starlark
+# MODULE.bazel
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+
+python.toolchain(
+    python_version = "3.11",
+    is_default = True,
+)
+
+python.toolchain(
+    python_version = "3.12",
+)
+
+# BUILD.bazel
+load("@rules_python//python:py_binary.bzl", "py_binary")
+load("@rules_python//python:py_test.bzl", "py_test")
+
+# Defaults to 3.11
+py_binary(...)
+py_test(...)
+
+# Explicitly use Python 3.11
+py_binary(..., python_version = "3.11")
+py_test(..., python_version = "3.11")
+
+# Explicitly use Python 3.12
+py_binary(..., python_version = "3.12")
+py_test(..., python_version = "3.12")
+```
 
 For more documentation, see the bzlmod examples under the {gh-path}`examples`
 folder.  Look for the examples that contain a `MODULE.bazel` file.
@@ -159,6 +197,16 @@ The `python.toolchain()` call makes its contents available under a repo named
 Remember to call `use_repo()` to make repos visible to your module:
 `use_repo(python, "python_3_11")`
 
+
+:::{deprecated} 1.1.0
+The toolchain specific `py_binary` and `py_test` symbols are aliases to the regular rules. 
+i.e. Deprecated `load("@python_versions//3.11:defs.bzl", "py_binary")` & `load("@python_versions//3.11:defs.bzl", "py_test")`
+
+Usages of them should be changed to load the regular rules directly; 
+i.e.  Use `load("@rules_python//python:py_binary.bzl", "py_binary")` & `load("@rules_python//python:py_test.bzl", "py_test")` and then specify the `python_version` when using the rules corresponding to the python version you defined in your toolchain. {ref}`Library modules with version constraints`
+:::
+
+
 #### Toolchain usage in other rules
 
 Python toolchains can be utilized in other bazel rules, such as `genrule()`, by

examples/bzlmod/MODULE.bazel

@@ -105,7 +105,7 @@ python.single_version_platform_override(
 use_repo(python, "python_3_10", "python_3_9", "python_versions", "pythons_hub")
 
 # EXPERIMENTAL: This is experimental and may be removed without notice
-uv = use_extension("@rules_python//python/uv:extensions.bzl", "uv")
+uv = use_extension("@rules_python//python/uv:uv.bzl", "uv")
 uv.toolchain(uv_version = "0.4.25")
 use_repo(uv, "uv_toolchains")
 

python/config_settings/BUILD.bazel

@@ -9,6 +9,7 @@ load(
     "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
+    "VenvsUseDeclareSymlinkFlag",
 )
 load(
     "//python/private/pypi:flags.bzl",
@@ -121,6 +122,13 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+string_flag(
+    name = "venvs_use_declare_symlink",
+    build_setting_default = VenvsUseDeclareSymlinkFlag.YES,
+    values = VenvsUseDeclareSymlinkFlag.flag_values(),
+    visibility = ["//visibility:public"],
+)
+
 # pip.parse related flags
 
 string_flag(