Merge branch 'main' into stop-hacking-tree-sitter

Author: dougthor42

.github/ISSUE_TEMPLATE/patch_release_request.md

@@ -0,0 +1,22 @@
+---
+name: "🏗️ Patch release or backport"
+about: Request a patch release or backport of a fix to a release.
+title: 'Patch release: MAJOR.MINOR.PATCH'
+labels: 'type: process'
+---
+
+<!--
+Thank you for requesting a backport. Before submitting, please read:
+
+* Backport policy: https://rules-python.readthedocs.io/en/latest/support.html#backports-and-patch-releases
+-->
+
+**What version of `rules_python` do you want to patch?**
+
+
+**What pull requests do you want to backport?**
+
+Please provide a list of pull request numbers.
+
+- #
+- #

CHANGELOG.md

@@ -72,6 +72,10 @@ END_UNRELEASED_TEMPLATE
 * (toolchain) Python 3.13 now references 3.13.6
 * (gazelle) Switched back to smacker/go-tree-sitter, fixing
   [#2630](https://github.com/bazel-contrib/rules_python/issues/2630)
+* (pypi) From now on the list of default platforms only includes `linux_x86_64`, `linux_aarch64`,
+  `osx_x86_64`, `osx_aarch64` and `windows_x86_64`. If you are on other platforms, you need to
+  use the `pip.default` to configure it yourself. If you are interested in graduating the
+  platform, consider helping set us up CI for them and update the documentation.
 * (ci) We are now testing on Ubuntu 22.04 for RBE and non-RBE configurations.
 * (core) `#!/usr/bin/env bash` is now used as a shebang in the stage1 bootstrap template.
 * (gazelle:docs) The Gazelle docs have been migrated from {gh-path}`gazelle/README.md` to
@@ -93,6 +97,10 @@ END_UNRELEASED_TEMPLATE
   ([#2503](https://github.com/bazel-contrib/rules_python/issues/2503)).
 * (pypi) The pipstar `defaults` configuration now supports any custom platform
   name.
+* (pypi) The selection of the whls has been changed and should no longer result
+  in ambiguous select matches ({gh-issue}`2759`) and should be much more efficient
+  when running `bazel query` due to fewer repositories being included 
+  ({gh-issue}`2849`).
 * Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle.
 * (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8
   ([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)).
@@ -123,7 +131,8 @@ END_UNRELEASED_TEMPLATE
   ([#3114](https://github.com/bazel-contrib/rules_python/pull/3114)).
 * (pypi) To configure the environment for `requirements.txt` evaluation, use the newly added
   developer preview of the `pip.default` tag class. Only `rules_python` and root modules can use
-  this feature. You can also configure custom `config_settings` using `pip.default`.
+  this feature. You can also configure custom `config_settings` using `pip.default`. It
+  can also be used to set the global `netrc` or `auth_patterns` variables.
 * (pypi) PyPI dependencies now expose an `:extracted_whl_files` filegroup target
   of all the files extracted from the wheel. This can be used in lieu of
   {obj}`whl_filegroup` to avoid copying/extracting wheel multiple times to

MODULE.bazel

@@ -74,16 +74,18 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
         env = {"platform_version": "0"},
         os_name = "linux",
         platform = "linux_{}".format(cpu),
+        whl_abi_tags = [
+            "abi3",
+            "cp{major}{minor}",
+        ],
+        whl_platform_tags = [
+            "linux_{}".format(cpu),
+            "manylinux_*_{}".format(cpu),
+        ],
     )
     for cpu in [
         "x86_64",
         "aarch64",
-        # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the
-        # `pip.default` extension. i.e. drop the below values - users will have to
-        # define themselves if they need them.
-        "arm",
-        "ppc",
-        "s390x",
     ]
 ]
 
@@ -99,26 +101,53 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
         env = {"platform_version": "14.0"},
         os_name = "osx",
         platform = "osx_{}".format(cpu),
+        whl_abi_tags = [
+            "abi3",
+            "cp{major}{minor}",
+        ],
+        whl_platform_tags = [
+            "macosx_*_{}".format(suffix)
+            for suffix in platform_tag_cpus
+        ],
     )
-    for cpu in [
-        "aarch64",
-        "x86_64",
-    ]
+    for cpu, platform_tag_cpus in {
+        "aarch64": [
+            "universal2",
+            "arm64",
+        ],
+        "x86_64": [
+            "universal2",
+            "x86_64",
+        ],
+    }.items()
+]
+
+[
+    pip.default(
+        arch_name = cpu,
+        config_settings = [
+            "@platforms//cpu:{}".format(cpu),
+            "@platforms//os:windows",
+        ],
+        env = {"platform_version": "0"},
+        os_name = "windows",
+        platform = "windows_{}".format(cpu),
+        whl_abi_tags = [
+            "abi3",
+            "cp{major}{minor}",
+        ],
+        whl_platform_tags = whl_platform_tags,
+    )
+    for cpu, whl_platform_tags in {
+        "x86_64": ["win_amd64"],
+    }.items()
 ]
 
-pip.default(
-    arch_name = "x86_64",
-    config_settings = [
-        "@platforms//cpu:x86_64",
-        "@platforms//os:windows",
-    ],
-    env = {"platform_version": "0"},
-    os_name = "windows",
-    platform = "windows_x86_64",
-)
 pip.parse(
     # NOTE @aignas 2024-10-26: We have an integration test that depends on us
     # being able to build sdists for this hub, so explicitly set this to False.
+    #
+    # how do we test sdists? Maybe just worth adding a single sdist somewhere?
     download_only = False,
     experimental_index_url = "https://pypi.org/simple",
     hub_name = "rules_python_publish_deps",

docs/devguide.md

@@ -116,3 +116,35 @@ to have everything self-documented, we have a special target,
 `//private:requirements.update`, which uses `rules_multirun` to run all
 of the requirement-updating scripts in sequence in one go. This can be done once per release as
 we prepare for releases.
+
+## Creating Backport PRs
+
+The steps to create a backport PR are:
+
+1.  Create an issue for the patch release; use the [patch relase
+    template][patch-release-issue].
+2.  Create a fork of `rules_python`.
+3.  Checkout the `release/X.Y` branch.
+4.  Use `git cherry-pick -x` to cherry pick the desired fixes.
+5.  Update the release's `CHANGELOG.md` file:
+    * Add a Major.Minor.Patch section if one doesn't exist
+    * Copy the changelog text from `main` to the release's changelog.
+6.  Send a PR with the backport's changes.
+    * The title should be `backport: PR#N to Major.Minor`
+    * The body must preserve the original PR's number, commit hash, description,
+      and authorship.
+      Use the following format (`git cherry-pick` will use this format):
+      ```
+      <original PR title>
+
+      <original PR body>
+      (cherry picked from commit <commit hash>)
+      -----
+      Co-authored-by: <original PR author; separate lines for each>
+      ```
+    * If the PR contains multiple backport commits, separate each's description
+      with `-----`.
+7.  Send a PR to update the `main` branch's `CHANGELOG.md` to reflect the
+    changes done in the patched release.
+
+[patch-release-issue]: https://github.com/bazelbuild/rules_python/issues/new?template=patch_release.md

docs/support.md

@@ -14,6 +14,22 @@ the willingness of volunteers.
 If you want or need particular functionality backported, then the best way
 is to open a PR to demonstrate the feasibility of the backport.
 
+### Backports and Patch Releases
+
+Backports and patch releases are provided on a best-effort basis. Only fixes are
+backported. Features are not backported.
+
+Backports can be done to older releases, but only if newer releases also have
+the fix backported. For example, if the current release is 1.5, in order to
+patch 1.4, version 1.5 must be patched first.
+
+Backports can be requested by [creating an issue with the patch release
+template][patch-release-issue] or by sending a pull request performing the backport.
+See the dev guide for [how to create a backport PR][backport-pr].
+
+[patch-release-issue]: https://github.com/bazelbuild/rules_python/issues/new?template=patch_release_request.md
+[backport-pr]: devguide.html#creating-backport-prs
+
 ## Supported Bazel Versions
 
 The supported Bazel versions are:

examples/bzlmod/MODULE.bazel

@@ -158,6 +158,23 @@ pip.whl_mods(
 )
 use_repo(pip, "whl_mods_hub")
 
+# Because below we are using `windows_aarch64` platform, we have to define various
+# properties for it.
+pip.default(
+    arch_name = "aarch64",
+    config_settings = [
+        "@platforms//os:windows",
+        "@platforms//cpu:aarch64",
+    ],
+    env = {
+        "platform_version": "0",
+    },
+    os_name = "windows",
+    platform = "windows_aarch64",
+    whl_abi_tags = [],  # default to all ABIs
+    whl_platform_tags = ["win_amd64"],
+)
+
 # To fetch pip dependencies, use pip.parse. We can pass in various options,
 # but typically we pass requirements and the Python version. The Python
 # version must have been configured by a corresponding `python.toolchain()`

gazelle/docs/installation_and_usage.md

@@ -86,6 +86,16 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
 modules_mapping(
     name = "modules_map",
     wheels = all_whl_requirements,
+
+    # include_stub_packages: bool (default: False)
+    # If set to True, this flag automatically includes any corresponding type stub packages
+    # for the third-party libraries that are present and used. For example, if you have
+    # `boto3` as a dependency, and this flag is enabled, the corresponding `boto3-stubs`
+    # package will be automatically included in the BUILD file.
+    # Enabling this feature helps ensure that type hints and stubs are readily available
+    # for tools like type checkers and IDEs, improving the development experience and
+    # reducing manual overhead in managing separate stub packages.
+    include_stub_packages = True,
 )
 
 # Gazelle python extension needs a manifest file mapping from
@@ -110,16 +120,6 @@ gazelle_python_manifest(
     # the integrity field is not added to the manifest which can help avoid
     # merge conflicts in large repos.
     requirements = "//:requirements_lock.txt",
-
-    # include_stub_packages: bool (default: False)
-    # If set to True, this flag automatically includes any corresponding type stub packages
-    # for the third-party libraries that are present and used. For example, if you have
-    # `boto3` as a dependency, and this flag is enabled, the corresponding `boto3-stubs`
-    # package will be automatically included in the BUILD file.
-    # Enabling this feature helps ensure that type hints and stubs are readily available
-    # for tools like type checkers and IDEs, improving the development experience and
-    # reducing manual overhead in managing separate stub packages.
-    include_stub_packages = True
 )
 ```
 
@@ -134,9 +134,9 @@ gazelle_binary(
     name = "gazelle_multilang",
     languages = [
         # List of language plugins.
-        # If you want to generate py_proto_library targets PR #3057), then
+        # If you want to generate py_proto_library targets (PR #3057), then
         # the proto language plugin _must_ come before the rules_python plugin.
-        #"@bazel_gazelle//lanugage/proto",
+        #"@bazel_gazelle//language/proto",
         "@rules_python_gazelle_plugin//python",
     ],
 )

python/private/get_local_runtime_info.py

@@ -23,7 +23,7 @@
 _IS_DARWIN = sys.platform == "darwin"
 
 
-def _search_directories(get_config):
+def _search_directories(get_config, base_executable):
     """Returns a list of library directories to search for shared libraries."""
     # There's several types of libraries with different names and a plethora
     # of settings, and many different config variables to check:
@@ -53,19 +53,23 @@ def _search_directories(get_config):
             if config_value and not config_value.endswith(multiarch):
                 lib_dirs.append(os.path.join(config_value, multiarch))
 
-    if _IS_WINDOWS:
-        # On Windows DLLs go in the same directory as the executable, while .lib
-        # files live in the lib/ or libs/ subdirectory.
-        lib_dirs.append(get_config("BINDIR"))
-        lib_dirs.append(os.path.join(os.path.dirname(sys.executable)))
-        lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "lib"))
-        lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "libs"))
-    elif not _IS_DARWIN:
-        # On most systems the executable is in a bin/ directory and the libraries
-        # are in a sibling lib/ directory.
-        lib_dirs.append(
-            os.path.join(os.path.dirname(os.path.dirname(sys.executable)), "lib")
-        )
+    if not _IS_DARWIN:
+        for exec_dir in (
+            os.path.dirname(base_executable) if base_executable else None,
+            get_config("BINDIR"),
+        ):
+            if not exec_dir:
+                continue
+            if _IS_WINDOWS:
+                # On Windows DLLs go in the same directory as the executable, while .lib
+                # files live in the lib/ or libs/ subdirectory.
+                lib_dirs.append(exec_dir)
+                lib_dirs.append(os.path.join(exec_dir, "lib"))
+                lib_dirs.append(os.path.join(exec_dir, "libs"))
+            else:
+                # On most systems the executable is in a bin/ directory and the libraries
+                # are in a sibling lib/ directory.
+                lib_dirs.append(os.path.join(os.path.dirname(exec_dir), "lib"))
 
     # Dedup and remove empty values, keeping the order.
     lib_dirs = [v for v in lib_dirs if v]
@@ -126,7 +130,7 @@ def _search_library_names(get_config):
     return {k: None for k in lib_names}.keys()
 
 
-def _get_python_library_info():
+def _get_python_library_info(base_executable):
     """Returns a dictionary with the static and dynamic python libraries."""
     config_vars = sysconfig.get_config_vars()
 
@@ -140,7 +144,7 @@ def _get_python_library_info():
                 f"{sys.version_info.major}.{sys.version_info.minor}"
             )
 
-    search_directories = _search_directories(config_vars.get)
+    search_directories = _search_directories(config_vars.get, base_executable)
     search_libnames = _search_library_names(config_vars.get)
 
     def _add_if_exists(target, path):
@@ -187,13 +191,28 @@ def _add_if_exists(target, path):
     }
 
 
+def _get_base_executable():
+    """Returns the base executable path."""
+    try:
+        if sys._base_executable:  # pylint: disable=protected-access
+            return sys._base_executable  # pylint: disable=protected-access
+    except AttributeError:
+        # Bug reports indicate sys._base_executable  doesn't exist in some cases,
+        # but it's not clear why.
+        # See https://github.com/bazel-contrib/rules_python/issues/3172
+        pass
+    # The normal sys.executable is the next-best guess if sys._base_executable
+    # is missing.
+    return sys.executable
+
+
 data = {
     "major": sys.version_info.major,
     "minor": sys.version_info.minor,
     "micro": sys.version_info.micro,
     "include": sysconfig.get_path("include"),
     "implementation_name": sys.implementation.name,
-    "base_executable": sys._base_executable,
+    "base_executable": _get_base_executable(),
 }
-data.update(_get_python_library_info())
+data.update(_get_python_library_info(_get_base_executable()))
 print(json.dumps(data))

python/private/py_executable.bzl

@@ -1578,7 +1578,11 @@ def _create_shared_native_deps_dso(
         feature_configuration,
         requested_features,
         cc_toolchain):
-    linkstamps = py_internal.linking_context_linkstamps(cc_info.linking_context)
+    linkstamps = [
+        py_internal.linkstamp_file(linkstamp)
+        for linker_input in cc_info.linking_context.linker_inputs.to_list()
+        for linkstamp in linker_input.linkstamps
+    ]
 
     partially_disabled_thin_lto = (
         cc_common.is_enabled(
@@ -1602,10 +1606,7 @@ def _create_shared_native_deps_dso(
             for input in cc_info.linking_context.linker_inputs.to_list()
             for flag in input.user_link_flags
         ],
-        linkstamps = [
-            py_internal.linkstamp_file(linkstamp)
-            for linkstamp in linkstamps.to_list()
-        ],
+        linkstamps = linkstamps,
         build_info_artifacts = _get_build_info(ctx, cc_toolchain) if linkstamps else [],
         features = requested_features,
         is_test_target_partially_disabled_thin_lto = is_test and partially_disabled_thin_lto,