Merge remote-tracking branch 'upstream/main' into remove-cc-defs

Author: rickeylev

.bazelci/presubmit.yml

@@ -65,15 +65,6 @@ buildifier:
 .reusable_build_test_all: &reusable_build_test_all
   build_targets: ["..."]
   test_targets: ["..."]
-.lockfile_mode_error: &lockfile_mode_error
-  # For testing lockfile support
-  skip_in_bazel_downstream_pipeline: "Lockfile depends on the bazel version"
-  build_flags:
-    - "--lockfile_mode=error"
-  test_flags:
-    - "--lockfile_mode=error"
-  coverage_flags:
-    - "--lockfile_mode=error"
 .coverage_targets_example_bzlmod: &coverage_targets_example_bzlmod
   coverage_targets: ["..."]
 .coverage_targets_example_bzlmod_build_file_generation: &coverage_targets_example_bzlmod_build_file_generation
@@ -94,6 +85,8 @@ buildifier:
   bazel: "7.x"
   environment:
     RULES_PYTHON_ENABLE_PYSTAR: "1"
+  build_flags:
+    - "--config=bazel7.x"
   test_flags:
     # The doc check tests fail because the Starlark implementation makes the
     # PyInfo and PyRuntimeInfo symbols become documented.
@@ -266,17 +259,23 @@ tasks:
   integration_test_bzlmod_ubuntu_lockfile:
     <<: *reusable_build_test_all
     <<: *coverage_targets_example_bzlmod
-    <<: *lockfile_mode_error
     name: "examples/bzlmod: Ubuntu with lockfile"
     working_directory: examples/bzlmod
     platform: ubuntu2004
+    shell_commands:
+    # Update the lockfiles and fail if it is different.
+    - "../../tools/private/update_bzlmod_lockfiles.sh"
+    - "git diff --exit-code"
   integration_test_bzlmod_macos_lockfile:
     <<: *reusable_build_test_all
     <<: *coverage_targets_example_bzlmod
-    <<: *lockfile_mode_error
     name: "examples/bzlmod: macOS with lockfile"
     working_directory: examples/bzlmod
     platform: macos
+    shell_commands:
+    # Update the lockfiles and fail if it is different.
+    - "../../tools/private/update_bzlmod_lockfiles.sh"
+    - "git diff --exit-code"
 
   integration_test_bzlmod_generate_build_file_generation_ubuntu_min:
     <<: *minimum_supported_version
@@ -302,7 +301,7 @@ tasks:
   integration_test_bzlmod_build_file_generation_debian:
     <<: *reusable_build_test_all
     <<: *coverage_targets_example_bzlmod_build_file_generation
-    name: "examples/bzlmod_build_file_integration: Debian"
+    name: "examples/bzlmod_build_file_generation: Debian"
     working_directory: examples/bzlmod_build_file_generation
     platform: debian11
   integration_test_bzlmod_build_file_generation_macos:

.bazelrc

@@ -33,4 +33,6 @@ build:rtd --stamp
 # Some bzl files contain repos only available under bzlmod
 build:rtd --enable_bzlmod
 
+common:bazel7.x --incompatible_python_disallow_native_rules
+
 build --lockfile_mode=update

.bcr/metadata.template.json

@@ -10,6 +10,11 @@
       "name": "Thulio Ferraz Assis",
       "email": "[email protected]",
       "github": "f0rmiga"
+    },
+    {
+      "name": "Ignas Anikevicius",
+      "email": "[email protected]",
+      "github": "aignas"
     }
   ],
   "repository": [

.gitattributes

@@ -1 +1,2 @@
+python/features.bzl export-subst
 tools/publish/*.txt linguist-generated=true

.github/workflows/create_archive_and_notes.sh

@@ -25,9 +25,10 @@ git archive --format=tar --prefix=${PREFIX}/ ${TAG} | gzip > $ARCHIVE
 SHA=$(shasum -a 256 $ARCHIVE | awk '{print $1}')
 
 cat > release_notes.txt << EOF
-## Using Bzlmod with Bazel 6
 
-**NOTE: bzlmod support is still beta. APIs subject to change.**
+For more detailed setup instructions, see https://rules-python.readthedocs.io/en/latest/getting-started.html
+
+## Using Bzlmod
 
 Add to your \`MODULE.bazel\` file:
 

CHANGELOG.md

@@ -27,19 +27,101 @@ A brief description of the categories of changes:
 
 {#v0-0-0-changed}
 ### Changed
-- Nothing yet
+* (deps) bazel_skylib 1.6.1 -> 1.7.1
 
 {#v0-0-0-fixed}
 ### Fixed
-- Nothing yet
+* Nothing yet
 
 {#v0-0-0-added}
 ### Added
-- Nothing yet
+* Nothing yet
 
 {#v0-0-0-removed}
 ### Removed
-- Nothing yet
+* Nothing yet
+
+{#v0-38-0}
+## [0.38.0] - 2024-11-08
+
+[0.38.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.38.0
+
+{#v0-38-0-changed}
+### Changed
+* (deps) (WORKSPACE only) rules_cc 0.0.13 and protobuf 27.0 is now the default
+  version used; this for Bazel 8+ support (previously version was rules_cc 0.0.9
+  and no protobuf version specified)
+  ([2310](https://github.com/bazelbuild/rules_python/issues/2310)).
+* (publish) The dependencies have been updated to the latest available versions
+  for the `twine` publishing rule.
+* (whl_library) Remove `--no-build-isolation` to allow non-hermetic sdist builds
+  by default. Users wishing to keep this argument and to enforce more hermetic
+  builds can do so by passing the argument in
+  [`pip.parse#extra_pip_args`](https://rules-python.readthedocs.io/en/latest/api/rules_python/python/extensions/pip.html#pip.parse.extra_pip_args)
+* (pip.parse) {attr}`pip.parse.whl_modifications` now normalizes the given whl names
+  and now `pyyaml` and `PyYAML` will both work.
+* (bzlmod) `pip.parse` spoke repository naming will be changed in an upcoming
+  release in places where the users specify different package versions per
+  platform in the same hub repository. The naming of the spoke repos is
+  considered an implementation detail and we advise the users to use the `hub`
+  repository directly and make use of {bzl:obj}`pip.parse.extra_hub_aliases`
+  feature added in this release.
+
+{#v0-38-0-fixed}
+### Fixed
+* (pypi) (Bazel 7.4+) Allow spaces in filenames included in `whl_library`s
+  ([617](https://github.com/bazelbuild/rules_python/issues/617)).
+* (pypi) When {attr}`pip.parse.experimental_index_url` is set, we need to still
+  pass the `extra_pip_args` value when building an `sdist`.
+* (pypi) The patched wheel filenames from now on are using local version specifiers
+  which fixes usage of the said wheels using standard package managers.
+* (bzlmod) The extension evaluation has been adjusted to always generate the
+  same lock file irrespective if `experimental_index_url` is set by any module
+  or not. To opt into this behavior, set
+  {bzl:obj}`pip.parse.parse_all_requirements_files`, which will become the
+  default in future releases leading up to `1.0.0`. Fixes
+  [#2268](https://github.com/bazelbuild/rules_python/issues/2268). A known
+  issue is that it may break `bazel query` and in these use cases it is
+  advisable to use `cquery` or switch to `download_only = True`
+
+{#v0-38-0-added}
+### Added
+* (publish) The requirements file for the `twine` publishing rules have been
+  updated to have a new convention: `requirements_darwin.txt`,
+  `requirements_linux.txt`, `requirements_windows.txt` for each respective OS
+  and one extra file `requirements_universal.txt` if you prefer a single file.
+  The `requirements.txt` file may be removed in the future.
+* The rules_python version is now reported in `//python/features.bzl#features.version`
+* (pip.parse) {attr}`pip.parse.extra_hub_aliases` can now be used to expose extra
+  targets created by annotations in whl repositories.
+  Fixes [#2187](https://github.com/bazelbuild/rules_python/issues/2187).
+* (bzlmod) `pip.parse` now supports `whl-only` setup using
+  `download_only = True` where users can specify multiple requirements files
+  and use the `pip` backend to do the downloading. This was only available for
+  users setting {bzl:obj}`pip.parse.experimental_index_url`, but now users have
+  more options whilst we continue to work on stabilizing the experimental feature.
+
+{#v0-37-2}
+## [0.37.2] - 2024-10-27
+
+[0.37.2]: https://github.com/bazelbuild/rules_python/releases/tag/0.37.2
+
+{#v0-37-2-fixed}
+### Fixed
+* (bzlmod) Generate `config_setting` values for all available toolchains instead
+  of only the registered toolchains, which restores the previous behaviour that
+  `bzlmod` users would have observed.
+
+{#v0-37-1}
+## [0.37.1] - 2024-10-22
+
+[0.37.1]: https://github.com/bazelbuild/rules_python/releases/tag/0.37.1
+
+{#v0-37-1-fixed}
+### Fixed
+* (rules) Setting `--incompatible_python_disallow_native_rules` no longer
+  causes rules_python rules to fail
+  ([#2326](https://github.com/bazelbuild/rules_python/issues/2326)).
 
 {#v0-37-0}
 ## [0.37.0] - 2024-10-18

CONTRIBUTING.md

@@ -65,6 +65,15 @@ and setup. Subsequent runs will be faster, but there are many tests, and some of
 them are slow. If you're working on a particular area of code, you can run just
 the tests in those directories instead, which can speed up your edit-run cycle.
 
+## Updating tool dependencies
+
+It's suggested to routinely update the tool versions within our repo - some of the
+tools are using requirement files compiled by `uv` and others use other means. In order
+to have everything self-documented, we have a special target -
+`//private:requirements.update`, which uses `rules_multirun` to run in sequence all
+of the requirement updating scripts in one go. This can be done once per release as
+we prepare for releases.
+
 ## Formatting
 
 Starlark files should be formatted by

DEVELOPING.md

@@ -4,7 +4,11 @@
 
 1. Modify the `./python/private/pypi/requirements.txt` file and run:
    ```
-   bazel run //tools/private/update_deps:update_pip_deps
+   bazel run //private:whl_library_requirements.update
+   ```
+1. Run the following target to update `twine` dependencies:
+   ```
+   bazel run //private:requirements.update
    ```
 1. Bump the coverage dependencies using the script using:
    ```
@@ -20,6 +24,8 @@ Start from a clean checkout at `main`.
 Before running through the release it's good to run the build and the tests locally, and make sure CI is passing. You can
 also test-drive the commit in an existing Bazel workspace to sanity check functionality.
 
+### Releasing from HEAD
+
 #### Steps
 1. [Determine the next semantic version number](#determining-semantic-version)
 1. Create a tag and push, e.g. `git tag 0.5.0 upstream/main && git push upstream --tags`
@@ -38,9 +44,31 @@ To find if there were any features added or incompatible changes made, review
 the commit history. This can be done using github by going to the url:
 `https://github.com/bazelbuild/rules_python/compare/<VERSION>...main`.
 
+### Patch release with cherry picks
+
+If a patch release from head would contain changes that aren't appropriate for
+a patch release, then the patch release needs to be based on the original
+release tag and the patch changes cherry-picked into it.
+
+In this example, release `0.37.0` is being patched to create release `0.37.1`.
+The fix being included is commit `deadbeef`.
+
+1. `git checkout -b release/0.37 0.37.0`
+1. `git push upstream release/0.37`
+1. `git cherry-pick -x deadbeef`
+1. Fix merge conflicts, if any. If `MODULE.bazel.lock` conflicts occur, then
+   run `pre-commit run update-bzlmod-lockfiles -a`
+1. `git cherry-pick --continue` (if applicable)
+1. `git push upstream`
+
+If multiple commits need to be applied, repeat the `git cherry-pick` step for
+each.
+
+Once the release branch is in the desired state, use `git tag` to tag it, as
+done with a release from head. Release automation will do the rest.
+
 #### After release creation in Github
 
-1. Ping @philwo to get the new release added to mirror.bazel.build. See [this comment on issue #400](https://github.com/bazelbuild/rules_python/issues/400#issuecomment-779159530) for more context.
 1. Announce the release in the #python channel in the Bazel slack (bazelbuild.slack.com).
 
 ## Secrets

MODULE.bazel

@@ -5,7 +5,7 @@ module(
 )
 
 bazel_dep(name = "bazel_features", version = "1.9.1")
-bazel_dep(name = "bazel_skylib", version = "1.6.1")
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
 bazel_dep(name = "rules_cc", version = "0.0.13")
 bazel_dep(name = "platforms", version = "0.0.4")
 
@@ -56,11 +56,15 @@ register_toolchains("@pythons_hub//:all")
 
 pip = use_extension("//python/private/pypi:pip.bzl", "pip_internal")
 pip.parse(
+    # NOTE @aignas 2024-10-26: We have an integration test that depends on us
+    # being able to build sdists for this hub, so explicitly set this to False.
+    download_only = False,
+    experimental_index_url = "https://pypi.org/simple",
     hub_name = "rules_python_publish_deps",
     python_version = "3.11",
     requirements_by_platform = {
-        "//tools/publish:requirements.txt": "linux_*",
         "//tools/publish:requirements_darwin.txt": "osx_*",
+        "//tools/publish:requirements_linux.txt": "linux_*",
         "//tools/publish:requirements_windows.txt": "windows_*",
     },
 )
@@ -70,14 +74,31 @@ use_repo(pip, "rules_python_publish_deps")
 bazel_dep(name = "stardoc", version = "0.6.2", repo_name = "io_bazel_stardoc")
 
 # ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
-bazel_dep(name = "rules_bazel_integration_test", version = "0.20.0", dev_dependency = True)
+bazel_dep(name = "rules_bazel_integration_test", version = "0.26.1", dev_dependency = True)
 bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True)
-bazel_dep(name = "rules_shell", version = "0.2.0", dev_dependency = True)
+bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True)
+bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True)
+bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
 
 # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests.
 # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides.
 bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_name = "io_bazel_rules_go")
 bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle")
+bazel_dep(name = "rules_python_gazelle_plugin", version = "0", dev_dependency = True)
+
+internal_dev_deps = use_extension(
+    "//python/private:internal_dev_deps.bzl",
+    "internal_dev_deps",
+    dev_dependency = True,
+)
+use_repo(internal_dev_deps, "buildkite_config", "wheel_for_testing")
+
+# Add gazelle plugin so that we can run the gazelle example as an e2e integration
+# test and include the distribution files.
+local_path_override(
+    module_name = "rules_python_gazelle_plugin",
+    path = "gazelle",
+)
 
 dev_python = use_extension(
     "//python/extensions:python.bzl",
@@ -89,17 +110,20 @@ dev_python.override(
 )
 
 dev_pip = use_extension(
-    "//python/private/pypi:pip.bzl",
-    "pip_internal",
+    "//python/extensions:pip.bzl",
+    "pip",
     dev_dependency = True,
 )
 dev_pip.parse(
-    download_only = True,  # this will not add the `sdist` values to the transitive closures at all.
+    download_only = True,
+    experimental_index_url = "https://pypi.org/simple",
     hub_name = "dev_pip",
     python_version = "3.11",
     requirements_lock = "//docs:requirements.txt",
 )
 dev_pip.parse(
+    download_only = True,
+    experimental_index_url = "https://pypi.org/simple",
     hub_name = "pypiserver",
     python_version = "3.11",
     requirements_lock = "//examples/wheel:requirements_server.txt",
@@ -140,7 +164,7 @@ uv = use_extension(
     "uv",
     dev_dependency = True,
 )
-uv.toolchain(uv_version = "0.2.23")
+uv.toolchain(uv_version = "0.4.25")
 use_repo(uv, "uv_toolchains")
 
 register_toolchains(