Merge branch 'main' into aignas.chore.remove-root-error-and-chmod

Author: aignas

CHANGELOG.md

@@ -55,18 +55,23 @@ END_UNRELEASED_TEMPLATE
 
 {#v0-0-0-removed}
 ### Removed
+* (toolchain) Remove all of the python 3.8 toolchain support out of the box. Users need
+  to pass the `TOOL_VERSIONS` that include 3.8 toolchains or use the `bzlmod` APIs to add
+  them back. This means any hub `pip.parse` calls that target `3.8` will be ignored from
+  now on. ([#2704](https://github.com/bazel-contrib/rules_python/issues/2704))
+* (toolchain) Remove all of the python 3.9 toolchain versions except for the `3.9.25`.
+  This version has reached EOL and will no longer receive any security fixes, please update to
+  `3.10` or above. ([#2704](https://github.com/bazel-contrib/rules_python/issues/2704))
 * (toolchain) `ignore_root_user_error` has now been flipped to be always enabled and
   the `chmod` of the python toolchain directories have been removed. From now on `rules_python`
   always adds the `pyc` files to the glob excludes and in order to avoid any problems when using
   the toolchains in the repository phase, ensure that you pass `-B` to the python interpreter.
   ([#2016](https://github.com/bazel-contrib/rules_python/issues/2016))
-* (toolchain) Remove all of the python 3.9 toolchain versions except for the `3.9.25`.
-  This version has reached EOL and will no longer receive any security fixes, please update to
-  `3.10` or above.
 
 {#v0-0-0-changed}
 ### Changed
 * (toolchains) Use toolchains from the [20251031] release.
+* (gazelle) Internally split modules mapping generation to be per-wheel for concurrency and caching.
 
 {#v0-0-0-fixed}
 ### Fixed
@@ -78,8 +83,15 @@ END_UNRELEASED_TEMPLATE
   ([#3085](https://github.com/bazel-contrib/rules_python/issues/3085)).
 * (toolchains) local toolchains now tell the `sys.abiflags` value of the
   underlying runtime.
+* (toolchains) various local toolchain fixes: add abi3 header targets,
+  fixes to linking, Windows DLL detection, and defines for free threaded
+  runtimes.
+* (toolchains) The `python_headers` target is now compatible with
+  layering_check.
 * (performance) 90% reduction in py_binary/py_test analysis phase cost.
   ([#3381](https://github.com/bazel-contrib/rules_python/pull/3381)).
+* (gazelle) Fix `gazelle_python_manifest.test` so that it accesses manifest files via `runfile` path handling rather than directly ([#3397](https://github.com/bazel-contrib/rules_python/issues/3397)).
+
 
 {#v0-0-0-added}
 ### Added

docs/howto/common-deps-with-multiple-pypi-versions.md

@@ -43,8 +43,6 @@ rules_python_config.add_transition_setting(
 
 # File: BUILD.bazel
 
-```bzl
-
 load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
 
 string_flag(

examples/bzlmod/.bazelrc

@@ -1,3 +1,8 @@
+# Starting with Bazel 8, Windows requires this flag in order
+# for symlinks to work properly (namely, so that sh_test with
+# py_binary as a data dependency gets symlinks that are executable)
+startup --windows_enable_symlinks
+
 common --enable_bzlmod
 common --lockfile_mode=update
 

gazelle/manifest/defs.bzl

@@ -117,9 +117,9 @@ def gazelle_python_manifest(
     if requirements:
         attrs = {
             "env": {
-                "_TEST_MANIFEST": "$(rootpath {})".format(manifest),
+                "_TEST_MANIFEST": "$(rlocationpath {})".format(manifest),
                 "_TEST_MANIFEST_GENERATOR_HASH": "$(rlocationpath {})".format(manifest_generator_hash),
-                "_TEST_REQUIREMENTS": "$(rootpath {})".format(requirements),
+                "_TEST_REQUIREMENTS": "$(rlocationpath {})".format(requirements),
             },
             "size": "small",
         }

gazelle/manifest/test/test.go

@@ -26,45 +26,50 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/bazelbuild/rules_go/go/runfiles"
 	"github.com/bazel-contrib/rules_python/gazelle/manifest"
+	"github.com/bazelbuild/rules_go/go/runfiles"
 )
 
-func TestGazelleManifestIsUpdated(t *testing.T) {
-	requirementsPath := os.Getenv("_TEST_REQUIREMENTS")
-	if requirementsPath == "" {
-		t.Fatal("_TEST_REQUIREMENTS must be set")
+// getResolvedRunfile resolves an environment variable to a runfiles path.
+// It handles getting the env var, checking it's set, and resolving it through
+// the runfiles mechanism, providing detailed error messages if anything fails.
+func getResolvedRunfile(t *testing.T, envVar string) string {
+	t.Helper()
+	path := os.Getenv(envVar)
+	if path == "" {
+		t.Fatalf("%s must be set", envVar)
 	}
-
-	manifestPath := os.Getenv("_TEST_MANIFEST")
-	if manifestPath == "" {
-		t.Fatal("_TEST_MANIFEST must be set")
+	resolvedPath, err := runfiles.Rlocation(path)
+	if err != nil {
+		t.Fatalf("failed to resolve runfiles path for %s (%q): %v", envVar, path, err)
 	}
+	return resolvedPath
+}
+
+func TestGazelleManifestIsUpdated(t *testing.T) {
+	requirementsPathResolved := getResolvedRunfile(t, "_TEST_REQUIREMENTS")
+	manifestPathResolved := getResolvedRunfile(t, "_TEST_MANIFEST")
 
 	manifestFile := new(manifest.File)
-	if err := manifestFile.Decode(manifestPath); err != nil {
+	if err := manifestFile.Decode(manifestPathResolved); err != nil {
 		t.Fatalf("decoding manifest file: %v", err)
 	}
 
 	if manifestFile.Integrity == "" {
 		t.Fatal("failed to find the Gazelle manifest file integrity")
 	}
 
-	manifestGeneratorHashPath, err := runfiles.Rlocation(
-		os.Getenv("_TEST_MANIFEST_GENERATOR_HASH"))
-	if err != nil {
-		t.Fatalf("failed to resolve runfiles path of manifest: %v", err)
-	}
+	manifestGeneratorHashPath := getResolvedRunfile(t, "_TEST_MANIFEST_GENERATOR_HASH")
 
 	manifestGeneratorHash, err := os.Open(manifestGeneratorHashPath)
 	if err != nil {
 		t.Fatalf("opening %q: %v", manifestGeneratorHashPath, err)
 	}
 	defer manifestGeneratorHash.Close()
 
-	requirements, err := os.Open(requirementsPath)
+	requirements, err := os.Open(requirementsPathResolved)
 	if err != nil {
-		t.Fatalf("opening %q: %v", requirementsPath, err)
+		t.Fatalf("opening %q: %v", requirementsPathResolved, err)
 	}
 	defer requirements.Close()
 
@@ -73,9 +78,9 @@ func TestGazelleManifestIsUpdated(t *testing.T) {
 		t.Fatalf("verifying integrity: %v", err)
 	}
 	if !valid {
-		manifestRealpath, err := filepath.EvalSymlinks(manifestPath)
+		manifestRealpath, err := filepath.EvalSymlinks(manifestPathResolved)
 		if err != nil {
-			t.Fatalf("evaluating symlink %q: %v", manifestPath, err)
+			t.Fatalf("evaluating symlink %q: %v", manifestPathResolved, err)
 		}
 		t.Errorf(
 			"%q is out-of-date. Follow the update instructions in that file to resolve this",

gazelle/modules_mapping/BUILD.bazel

@@ -9,6 +9,12 @@ py_binary(
     visibility = ["//visibility:public"],
 )
 
+py_binary(
+    name = "merger",
+    srcs = ["merger.py"],
+    visibility = ["//visibility:public"],
+)
+
 copy_file(
     name = "pytest_wheel",
     src = "@pytest//file",
@@ -33,6 +39,18 @@ py_test(
     deps = [":generator"],
 )
 
+py_test(
+    name = "test_merger",
+    srcs = ["test_merger.py"],
+    data = [
+        "django_types_wheel",
+        "pytest_wheel",
+    ],
+    imports = ["."],
+    main = "test_merger.py",
+    deps = [":merger"],
+)
+
 filegroup(
     name = "distribution",
     srcs = glob(["**"]),

gazelle/modules_mapping/def.bzl

@@ -30,25 +30,39 @@ def _modules_mapping_impl(ctx):
         transitive = [dep[DefaultInfo].files for dep in ctx.attr.wheels] + [dep[DefaultInfo].data_runfiles.files for dep in ctx.attr.wheels],
     )
 
-    args = ctx.actions.args()
+    # Run the generator once per-wheel (to leverage caching)
+    per_wheel_outputs = []
+    for idx, whl in enumerate(all_wheels.to_list()):
+        wheel_modules_mapping = ctx.actions.declare_file("{}.{}".format(modules_mapping.short_path, idx))
+        args = ctx.actions.args()
+        args.add("--output_file", wheel_modules_mapping.path)
+        if ctx.attr.include_stub_packages:
+            args.add("--include_stub_packages")
+        args.add_all("--exclude_patterns", ctx.attr.exclude_patterns)
+        args.add("--wheel", whl.path)
 
-    # Spill parameters to a file prefixed with '@'. Note, the '@' prefix is the same
-    # prefix as used in the `generator.py` in `fromfile_prefix_chars` attribute.
-    args.use_param_file(param_file_arg = "@%s")
-    args.set_param_file_format(format = "multiline")
-    if ctx.attr.include_stub_packages:
-        args.add("--include_stub_packages")
-    args.add("--output_file", modules_mapping)
-    args.add_all("--exclude_patterns", ctx.attr.exclude_patterns)
-    args.add_all("--wheels", all_wheels)
+        ctx.actions.run(
+            inputs = [whl],
+            outputs = [wheel_modules_mapping],
+            executable = ctx.executable._generator,
+            arguments = [args],
+            use_default_shell_env = False,
+        )
+        per_wheel_outputs.append(wheel_modules_mapping)
+
+    # Then merge the individual JSONs together
+    merge_args = ctx.actions.args()
+    merge_args.add("--output", modules_mapping.path)
+    merge_args.add_all("--inputs", [f.path for f in per_wheel_outputs])
 
     ctx.actions.run(
-        inputs = all_wheels,
+        inputs = per_wheel_outputs,
         outputs = [modules_mapping],
-        executable = ctx.executable._generator,
-        arguments = [args],
+        executable = ctx.executable._merger,
+        arguments = [merge_args],
         use_default_shell_env = False,
     )
+
     return [DefaultInfo(files = depset([modules_mapping]))]
 
 modules_mapping = rule(
@@ -79,6 +93,11 @@ modules_mapping = rule(
             default = "//modules_mapping:generator",
             executable = True,
         ),
+        "_merger": attr.label(
+            cfg = "exec",
+            default = "//modules_mapping:merger",
+            executable = True,
+        ),
     },
     doc = "Creates a modules_mapping.json file for mapping module names to wheel distribution names.",
 )

gazelle/modules_mapping/generator.py

@@ -96,23 +96,28 @@ def module_for_path(self, path, whl):
                 ext = "".join(pathlib.Path(root).suffixes)
             module = root[: -len(ext)].replace("/", ".")
             if not self.is_excluded(module):
-                if not self.is_excluded(module):
-                    self.mapping[module] = wheel_name
+                self.mapping[module] = wheel_name
 
     def is_excluded(self, module):
         for pattern in self.excluded_patterns:
             if pattern.search(module):
                 return True
         return False
 
-    # run is the entrypoint for the generator.
-    def run(self, wheels):
-        for whl in wheels:
-            try:
-                self.dig_wheel(whl)
-            except AssertionError as error:
-                print(error, file=self.stderr)
-                return 1
+    def run(self, wheel: pathlib.Path) -> int:
+        """
+        Entrypoint for the generator.
+
+        Args:
+            wheel: The path to the wheel file (`.whl`)
+        Returns:
+            Exit code (for `sys.exit`)
+        """
+        try:
+            self.dig_wheel(wheel)
+        except AssertionError as error:
+            print(error, file=self.stderr)
+            return 1
         self.simplify()
         mapping_json = json.dumps(self.mapping)
         with open(self.output_file, "w") as f:
@@ -152,16 +157,13 @@ def data_has_purelib_or_platlib(path):
     parser = argparse.ArgumentParser(
         prog="generator",
         description="Generates the modules mapping used by the Gazelle manifest.",
-        # Automatically read parameters from a file. Note, the '@' is the same prefix
-        # as set in the 'args.use_param_file' in the bazel rule.
-        fromfile_prefix_chars="@",
     )
     parser.add_argument("--output_file", type=str)
     parser.add_argument("--include_stub_packages", action="store_true")
     parser.add_argument("--exclude_patterns", nargs="+", default=[])
-    parser.add_argument("--wheels", nargs="+", default=[])
+    parser.add_argument("--wheel", type=pathlib.Path)
     args = parser.parse_args()
     generator = Generator(
         sys.stderr, args.output_file, args.exclude_patterns, args.include_stub_packages
     )
-    sys.exit(generator.run(args.wheels))
+    sys.exit(generator.run(args.wheel))

gazelle/modules_mapping/merger.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""Merges multiple modules_mapping.json files into a single file."""
+
+import argparse
+import json
+from pathlib import Path
+
+
+def merge_modules_mappings(input_files: list[Path], output_file: Path) -> None:
+    """Merge multiple modules_mapping.json files into one.
+
+    Args:
+        input_files: List of paths to input JSON files to merge
+        output_file: Path where the merged output should be written
+    """
+    merged_mapping = {}
+    for input_file in input_files:
+        mapping = json.loads(input_file.read_text())
+        # Merge the mappings, with later files overwriting earlier ones
+        # if there are conflicts
+        merged_mapping.update(mapping)
+
+    output_file.write_text(json.dumps(merged_mapping))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Merge multiple modules_mapping.json files"
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        type=Path,
+        help="Output file path for merged mapping",
+    )
+    parser.add_argument(
+        "--inputs",
+        required=True,
+        nargs="+",
+        type=Path,
+        help="Input JSON files to merge",
+    )
+
+    args = parser.parse_args()
+    merge_modules_mappings(args.inputs, args.output)