Merge branch 'main' into bootstrap-fix

Author: rickeylev

CHANGELOG.md

@@ -95,6 +95,8 @@ Other changes:
   ([2169](https://github.com/bazelbuild/rules_python/issues/2169)).
 * (workspace) Corrected protobuf's name to com_google_protobuf, the name is
   hardcoded in Bazel, WORKSPACE mode.
+* (pypi): {bzl:obj}`compile_pip_requirements` no longer fails on Windows when `--enable_runfiles` is not enabled.
+* (pypi): {bzl:obj}`compile_pip_requirements` now correctly updates files in the source tree on Windows when `--windows_enable_symlinks` is not enabled.
 * (repositories): Add libs/python3.lib and pythonXY.dll to the `libpython` target
   defined by a repository template. This enables stable ABI builds of Python extensions
   on Windows (by defining Py_LIMITED_API).

docs/toolchains.md

@@ -444,8 +444,17 @@ toolchain(
     ],
     exec_comaptible_with = ["@platforms/os:linux"]
 )
+
+# File: MODULE.bazel or WORKSPACE.bazel
+# These toolchains will considered before others
+register_toolchains("//toolchains:all")
 ```
 
+When registering custom toolchains, be aware of the the [toolchain registration
+order](https://bazel.build/extending/toolchains#toolchain-resolution). In brief,
+toolchain order is the BFS-order of the modules; see the bazel docs for a more
+detailed description.
+
 :::{note}
 The toolchain() calls should be in a separate BUILD file from everything else.
 This avoids Bazel having to perform unnecessary work when it discovers the list

python/private/BUILD.bazel

@@ -30,6 +30,7 @@ licenses(["notice"])
 filegroup(
     name = "distribution",
     srcs = glob(["**"]) + [
+        "//python/private/api:distribution",
         "//python/private/proto:distribution",
         "//python/private/pypi:distribution",
         "//python/private/whl_filegroup:distribution",

python/private/api/BUILD.bazel

@@ -19,6 +19,11 @@ package(
     default_visibility = ["//:__subpackages__"],
 )
 
+filegroup(
+    name = "distribution",
+    srcs = glob(["**"]),
+)
+
 py_common_api(
     name = "py_common_api",
     # NOTE: Not actually public. Implicit dependency of public rules.

python/private/py_cc_link_params_info.bzl

@@ -16,290 +16,6 @@
 load("@rules_cc//cc/common:cc_info.bzl", "CcInfo")
 load(":util.bzl", "define_bazel_6_provider")
 
-DEFAULT_STUB_SHEBANG = "#!/usr/bin/env python3"
-
-DEFAULT_BOOTSTRAP_TEMPLATE = Label("//python/private:bootstrap_template")
-
-_PYTHON_VERSION_VALUES = ["PY2", "PY3"]
-
-def _optional_int(value):
-    return int(value) if value != None else None
-
-def interpreter_version_info_struct_from_dict(info_dict):
-    """Create a struct of interpreter version info from a dict from an attribute.
-
-    Args:
-        info_dict: (dict | None) of version info fields. See interpreter_version_info
-            provider field docs.
-
-    Returns:
-        struct of version info; see interpreter_version_info provider field docs.
-    """
-    info_dict = dict(info_dict or {})  # Copy in case the original is frozen
-    if info_dict:
-        if not ("major" in info_dict and "minor" in info_dict):
-            fail("interpreter_version_info must have at least two keys, 'major' and 'minor'")
-    version_info_struct = struct(
-        major = _optional_int(info_dict.pop("major", None)),
-        minor = _optional_int(info_dict.pop("minor", None)),
-        micro = _optional_int(info_dict.pop("micro", None)),
-        releaselevel = str(info_dict.pop("releaselevel")) if "releaselevel" in info_dict else None,
-        serial = _optional_int(info_dict.pop("serial", None)),
-    )
-
-    if len(info_dict.keys()) > 0:
-        fail("unexpected keys {} in interpreter_version_info".format(
-            str(info_dict.keys()),
-        ))
-
-    return version_info_struct
-
-def _PyRuntimeInfo_init(
-        *,
-        implementation_name = None,
-        interpreter_path = None,
-        interpreter = None,
-        files = None,
-        coverage_tool = None,
-        coverage_files = None,
-        pyc_tag = None,
-        python_version,
-        stub_shebang = None,
-        bootstrap_template = None,
-        interpreter_version_info = None,
-        stage2_bootstrap_template = None,
-        zip_main_template = None):
-    if (interpreter_path and interpreter) or (not interpreter_path and not interpreter):
-        fail("exactly one of interpreter or interpreter_path must be specified")
-
-    if interpreter_path and files != None:
-        fail("cannot specify 'files' if 'interpreter_path' is given")
-
-    if (coverage_tool and not coverage_files) or (not coverage_tool and coverage_files):
-        fail(
-            "coverage_tool and coverage_files must both be set or neither must be set, " +
-            "got coverage_tool={}, coverage_files={}".format(
-                coverage_tool,
-                coverage_files,
-            ),
-        )
-
-    if python_version not in _PYTHON_VERSION_VALUES:
-        fail("invalid python_version: '{}'; must be one of {}".format(
-            python_version,
-            _PYTHON_VERSION_VALUES,
-        ))
-
-    if files != None and type(files) != type(depset()):
-        fail("invalid files: got value of type {}, want depset".format(type(files)))
-
-    if interpreter:
-        if files == None:
-            files = depset()
-    else:
-        files = None
-
-    if coverage_files == None:
-        coverage_files = depset()
-
-    if not stub_shebang:
-        stub_shebang = DEFAULT_STUB_SHEBANG
-
-    return {
-        "bootstrap_template": bootstrap_template,
-        "coverage_files": coverage_files,
-        "coverage_tool": coverage_tool,
-        "files": files,
-        "implementation_name": implementation_name,
-        "interpreter": interpreter,
-        "interpreter_path": interpreter_path,
-        "interpreter_version_info": interpreter_version_info_struct_from_dict(interpreter_version_info),
-        "pyc_tag": pyc_tag,
-        "python_version": python_version,
-        "stage2_bootstrap_template": stage2_bootstrap_template,
-        "stub_shebang": stub_shebang,
-        "zip_main_template": zip_main_template,
-    }
-
-PyRuntimeInfo, _unused_raw_py_runtime_info_ctor = define_bazel_6_provider(
-    doc = """Contains information about a Python runtime, as returned by the `py_runtime`
-rule.
-
-A Python runtime describes either a *platform runtime* or an *in-build runtime*.
-A platform runtime accesses a system-installed interpreter at a known path,
-whereas an in-build runtime points to a `File` that acts as the interpreter. In
-both cases, an "interpreter" is really any executable binary or wrapper script
-that is capable of running a Python script passed on the command line, following
-the same conventions as the standard CPython interpreter.
-""",
-    init = _PyRuntimeInfo_init,
-    fields = {
-        "bootstrap_template": """
-:type: File
-
-A template of code responsible for the initial startup of a program.
-
-This code is responsible for:
-
-* Locating the target interpreter. Typically it is in runfiles, but not always.
-* Setting necessary environment variables, command line flags, or other
-  configuration that can't be modified after the interpreter starts.
-* Invoking the appropriate entry point. This is usually a second-stage bootstrap
-  that performs additional setup prior to running a program's actual entry point.
-
-The {obj}`--bootstrap_impl` flag affects how this stage 1 bootstrap
-is expected to behave and the substutitions performed.
-
-* `--bootstrap_impl=system_python` substitutions: `%is_zipfile%`, `%python_binary%`,
-  `%target%`, `%workspace_name`, `%coverage_tool%`, `%import_all%`, `%imports%`,
-  `%main%`, `%shebang%`
-* `--bootstrap_impl=script` substititions: `%is_zipfile%`, `%python_binary%`,
-  `%target%`, `%workspace_name`, `%shebang%, `%stage2_bootstrap%`
-
-Substitution definitions:
-
-* `%shebang%`: The shebang to use with the bootstrap; the bootstrap template
-  may choose to ignore this.
-* `%stage2_bootstrap%`: A runfiles-relative path to the stage 2 bootstrap.
-* `%python_binary%`: The path to the target Python interpreter. There are three
-  types of paths:
-  * An absolute path to a system interpreter (e.g. begins with `/`).
-  * A runfiles-relative path to an interpreter (e.g. `somerepo/bin/python3`)
-  * A program to search for on PATH, i.e. a word without spaces, e.g. `python3`.
-* `%workspace_name%`: The name of the workspace the target belongs to.
-* `%is_zipfile%`: The string `1` if this template is prepended to a zipfile to
-  create a self-executable zip file. The string `0` otherwise.
-
-For the other substitution definitions, see the {obj}`stage2_bootstrap_template`
-docs.
-
-:::{versionchanged} 0.33.0
-The set of substitutions depends on {obj}`--bootstrap_impl`
-:::
-""",
-        "coverage_files": """
-:type: depset[File] | None
-
-The files required at runtime for using `coverage_tool`. Will be `None` if no
-`coverage_tool` was provided.
-""",
-        "coverage_tool": """
-:type: File | None
-
-If set, this field is a `File` representing tool used for collecting code
-coverage information from python tests. Otherwise, this is `None`.
-""",
-        "files": """
-:type: depset[File] | None
-
-If this is an in-build runtime, this field is a `depset` of `File`s that need to
-be added to the runfiles of an executable target that uses this runtime (in
-particular, files needed by `interpreter`). The value of `interpreter` need not
-be included in this field. If this is a platform runtime then this field is
-`None`.
-""",
-        "implementation_name": """
-:type: str | None
-
-The Python implementation name (`sys.implementation.name`)
-""",
-        "interpreter": """
-:type: File | None
-
-If this is an in-build runtime, this field is a `File` representing the
-interpreter. Otherwise, this is `None`. Note that an in-build runtime can use
-either a prebuilt, checked-in interpreter or an interpreter built from source.
-""",
-        "interpreter_path": """
-:type: str | None
-
-If this is a platform runtime, this field is the absolute filesystem path to the
-interpreter on the target platform. Otherwise, this is `None`.
-""",
-        "interpreter_version_info": """
-:type: struct
-
-Version information about the interpreter this runtime provides.
-It should match the format given by `sys.version_info`, however
-for simplicity, the micro, releaselevel, and serial values are
-optional.
-A struct with the following fields:
-* `major`: {type}`int`, the major version number
-* `minor`: {type}`int`, the minor version number
-* `micro`: {type}`int | None`, the micro version number
-* `releaselevel`: {type}`str | None`, the release level
-* `serial`: {type}`int | None`, the serial number of the release
-""",
-        "pyc_tag": """
-:type: str | None
-
-The tag portion of a pyc filename, e.g. the `cpython-39` infix
-of `foo.cpython-39.pyc`. See PEP 3147. If not specified, it will be computed
-from {obj}`implementation_name` and {obj}`interpreter_version_info`. If no
-pyc_tag is available, then only source-less pyc generation will function
-correctly.
-""",
-        "python_version": """
-:type: str
-
-Indicates whether this runtime uses Python major version 2 or 3. Valid values
-are (only) `"PY2"` and `"PY3"`.
-""",
-        "stage2_bootstrap_template": """
-:type: File
-
-A template of Python code that runs under the desired interpreter and is
-responsible for orchestrating calling the program's actual main code. This
-bootstrap is responsible for affecting the current runtime's state, such as
-import paths or enabling coverage, so that, when it runs the program's actual
-main code, it works properly under Bazel.
-
-The following substitutions are made during template expansion:
-* `%main%`: A runfiles-relative path to the program's actual main file. This
-  can be a `.py` or `.pyc` file, depending on precompile settings.
-* `%coverage_tool%`: Runfiles-relative path to the coverage library's entry point.
-  If coverage is not enabled or available, an empty string.
-* `%import_all%`: The string `True` if all repositories in the runfiles should
-  be added to sys.path. The string `False` otherwise.
-* `%imports%`: A colon-delimited string of runfiles-relative paths to add to
-  sys.path.
-* `%target%`: The name of the target this is for.
-* `%workspace_name%`: The name of the workspace the target belongs to.
-
-:::{versionadded} 0.33.0
-:::
-""",
-        "stub_shebang": """
-:type: str
-
-"Shebang" expression prepended to the bootstrapping Python stub
-script used when executing {obj}`py_binary` targets.  Does not
-apply to Windows.
-""",
-        "zip_main_template": """
-:type: File
-
-A template of Python code that becomes a zip file's top-level `__main__.py`
-file. The top-level `__main__.py` file is used when the zip file is explicitly
-passed to a Python interpreter. See PEP 441 for more information about zipapp
-support. Note that py_binary-generated zip files are self-executing and
-skip calling `__main__.py`.
-
-The following substitutions are made during template expansion:
-* `%stage2_bootstrap%`: A runfiles-relative string to the stage 2 bootstrap file.
-* `%python_binary%`: The path to the target Python interpreter. There are three
-  types of paths:
-  * An absolute path to a system interpreter (e.g. begins with `/`).
-  * A runfiles-relative path to an interpreter (e.g. `somerepo/bin/python3`)
-  * A program to search for on PATH, i.e. a word without spaces, e.g. `python3`.
-* `%workspace_name%`: The name of the workspace for the built target.
-
-:::{versionadded} 0.33.0
-:::
-""",
-    },
-)
-
 def _PyCcLinkParamsInfo_init(cc_info):
     return {
         "cc_info": CcInfo(linking_context = cc_info.linking_context),

python/private/pypi/dependency_resolver/dependency_resolver.py

@@ -170,19 +170,26 @@ def main(
 
     if UPDATE:
         print("Updating " + requirements_file_relative)
+
+        # Make sure the output file for pip_compile exists. It won't if we are on Windows and --enable_runfiles is not set.
+        if not os.path.exists(requirements_file_relative):
+            os.makedirs(os.path.dirname(requirements_file_relative), exist_ok=True)
+            shutil.copy(resolved_requirements_file, requirements_file_relative)
+
         if "BUILD_WORKSPACE_DIRECTORY" in os.environ:
             workspace = os.environ["BUILD_WORKSPACE_DIRECTORY"]
             requirements_file_tree = os.path.join(workspace, requirements_file_relative)
+            absolute_output_file = Path(requirements_file_relative).absolute()
             # In most cases, requirements_file will be a symlink to the real file in the source tree.
             # If symlinks are not enabled (e.g. on Windows), then requirements_file will be a copy,
             # and we should copy the updated requirements back to the source tree.
-            if not os.path.samefile(resolved_requirements_file, requirements_file_tree):
+            if not absolute_output_file.samefile(requirements_file_tree):
                 atexit.register(
                     lambda: shutil.copy(
-                        resolved_requirements_file, requirements_file_tree
+                        absolute_output_file, requirements_file_tree
                     )
                 )
-        cli(argv)
+        cli(argv, standalone_mode = False)
         requirements_file_relative_path = Path(requirements_file_relative)
         content = requirements_file_relative_path.read_text()
         content = content.replace(absolute_path_prefix, "")

tests/cc/current_py_cc_libs/BUILD.bazel

@@ -20,14 +20,6 @@ current_py_cc_libs_test_suite(name = "current_py_cc_libs_tests")
 cc_test(
     name = "python_libs_linking_test",
     srcs = ["python_libs_linking_test.cc"],
-    # Windows fails with linking errors, but its not clear why; someone
-    # with more C + Windows experience will have to figure it out.
-    # - rickeylev@
-    target_compatible_with = select({
-        "@platforms//os:linux": [],
-        "@platforms//os:osx": [],
-        "//conditions:default": ["@platforms//:incompatible"],
-    }),
     deps = [
         "@rules_python//python/cc:current_py_cc_headers",
         "@rules_python//python/cc:current_py_cc_libs",
@@ -41,10 +33,9 @@ cc_test(
 # for libs/python3.lib.
 # buildifier: disable=native-cc
 cc_test(
-    name = "python_libs_linking_windows_test",
+    name = "python_abi3_libs_linking_windows_test",
     srcs = ["python_libs_linking_test.cc"],
     defines = ["Py_LIMITED_API=0x030A0000"],
-    env = {"HELLO": "world"},
     target_compatible_with = ["@platforms//os:windows"],
     deps = [
         "@rules_python//python/cc:current_py_cc_headers",