internal(pystar): Make py_runtime_pair and autodetecting toolchain mostly loadable. (#1439)

They aren't quite usable yet. This mostly fixes references and adds
surrounding infrastructure to make this mostly loadable and somewhat
runnable.

Note that the "autodetecting" name is misleading: it doesn't really
autodetect anything. But it's the established name and part of the
public API.

The autodetecting toolchain is trimmed down from what Bazel does. The
Bazel version has a Python 2 variant and a "non strict" mode to support
that. With Python 2 no longer supported, that code is removed.

* Also alphabetically sorts the bzl_libraries in //python/private

Work towards #1069

Author: rickeylev

python/private/BUILD.bazel

@@ -44,13 +44,12 @@ filegroup(
 )
 
 bzl_library(
-    name = "reexports_bzl",
-    srcs = ["reexports.bzl"],
-    visibility = [
-        "//docs:__pkg__",
-        "//python:__pkg__",
+    name = "autodetecting_toolchain_bzl",
+    srcs = ["autodetecting_toolchain.bzl"],
+    deps = [
+        "//python:py_runtime_bzl",
+        "//python:py_runtime_pair_bzl",
     ],
-    deps = [":bazel_tools_bzl"],
 )
 
 bzl_library(
@@ -63,15 +62,6 @@ bzl_library(
     deps = ["@bazel_skylib//lib:types"],
 )
 
-bzl_library(
-    name = "which_bzl",
-    srcs = ["which.bzl"],
-    visibility = [
-        "//docs:__subpackages__",
-        "//python:__subpackages__",
-    ],
-)
-
 bzl_library(
     name = "py_cc_toolchain_bzl",
     srcs = [
@@ -112,6 +102,21 @@ bzl_library(
     visibility = ["//:__subpackages__"],
 )
 
+bzl_library(
+    name = "py_runtime_pair_macro_bzl",
+    srcs = ["py_runtime_pair_rule.bzl"],
+    deps = [":py_runtime_pair_rule_bzl"],
+)
+
+bzl_library(
+    name = "py_runtime_pair_rule_bzl",
+    srcs = ["py_runtime_pair_rule.bzl"],
+    deps = [
+        "//python:py_runtime_bzl",
+        "//python:py_runtime_info_bzl",
+    ],
+)
+
 bzl_library(
     name = "py_wheel_bzl",
     srcs = ["py_wheel.bzl"],
@@ -122,12 +127,31 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "reexports_bzl",
+    srcs = ["reexports.bzl"],
+    visibility = [
+        "//docs:__pkg__",
+        "//python:__pkg__",
+    ],
+    deps = [":bazel_tools_bzl"],
+)
+
 bzl_library(
     name = "stamp_bzl",
     srcs = ["stamp.bzl"],
     visibility = ["//:__subpackages__"],
 )
 
+bzl_library(
+    name = "which_bzl",
+    srcs = ["which.bzl"],
+    visibility = [
+        "//docs:__subpackages__",
+        "//python:__subpackages__",
+    ],
+)
+
 # @bazel_tools can't define bzl_library itself, so we just put a wrapper around it.
 bzl_library(
     name = "bazel_tools_bzl",

python/private/autodetecting_toolchain.bzl

@@ -14,51 +14,21 @@
 
 """Definitions related to the Python toolchain."""
 
-load(":utils.bzl", "expand_pyversion_template")
+load("//python:py_runtime.bzl", "py_runtime")
+load("//python:py_runtime_pair.bzl", "py_runtime_pair")
 
-def define_autodetecting_toolchain(
-        name,
-        pywrapper_template,
-        windows_config_setting):
+def define_autodetecting_toolchain(name):
     """Defines the autodetecting Python toolchain.
 
-    This includes both strict and non-strict variants.
-
-    For use only by @bazel_tools//tools/python:BUILD; see the documentation
-    comment there.
-
     Args:
         name: The name of the toolchain to introduce. Must have value
             "autodetecting_toolchain". This param is present only to make the
             BUILD file more readable.
-        pywrapper_template: The label of the pywrapper_template.txt file.
-        windows_config_setting: The label of a config_setting that matches when
-            the platform is windows, in which case the toolchain is configured
-            in a way that triggers a workaround for #7844.
     """
-    if native.package_name() != "tools/python":
-        fail("define_autodetecting_toolchain() is private to " +
-             "@bazel_tools//tools/python")
     if name != "autodetecting_toolchain":
         fail("Python autodetecting toolchain must be named " +
              "'autodetecting_toolchain'")
 
-    expand_pyversion_template(
-        name = "_generate_wrappers",
-        template = pywrapper_template,
-        out2 = ":py2wrapper.sh",
-        out3 = ":py3wrapper.sh",
-        out2_nonstrict = ":py2wrapper_nonstrict.sh",
-        out3_nonstrict = ":py3wrapper_nonstrict.sh",
-        visibility = ["//visibility:private"],
-    )
-
-    # Note that the pywrapper script is a .sh file, not a sh_binary target. If
-    # we needed to make it a proper shell target, e.g. because it needed to
-    # access runfiles and needed to depend on the runfiles library, then we'd
-    # have to use a workaround to allow it to be depended on by py_runtime. See
-    # https://github.com/bazelbuild/bazel/issues/4286#issuecomment-475661317.
-
     # buildifier: disable=native-py
     py_runtime(
         name = "_autodetecting_py3_runtime",
@@ -68,15 +38,6 @@ def define_autodetecting_toolchain(
         visibility = ["//visibility:private"],
     )
 
-    # buildifier: disable=native-py
-    py_runtime(
-        name = "_autodetecting_py3_runtime_nonstrict",
-        interpreter = ":py3wrapper_nonstrict.sh",
-        python_version = "PY3",
-        stub_shebang = "#!/usr/bin/env python3",
-        visibility = ["//visibility:private"],
-    )
-
     # This is a dummy runtime whose interpreter_path triggers the native rule
     # logic to use the legacy behavior on Windows.
     # TODO(#7844): Remove this target.
@@ -95,33 +56,15 @@ def define_autodetecting_toolchain(
             # that we attempted to use the autodetecting toolchain and need to
             # switch back to legacy behavior.
             # TODO(#7844): Remove this hack.
-            windows_config_setting: ":_magic_sentinel_runtime",
+            "@platforms//os:windows": ":_magic_sentinel_runtime",
             "//conditions:default": ":_autodetecting_py3_runtime",
         }),
         visibility = ["//visibility:public"],
     )
 
-    py_runtime_pair(
-        name = "_autodetecting_py_runtime_pair_nonstrict",
-        py3_runtime = select({
-            # Same hack as above.
-            # TODO(#7844): Remove this hack.
-            windows_config_setting: ":_magic_sentinel_runtime",
-            "//conditions:default": ":_autodetecting_py3_runtime_nonstrict",
-        }),
-        visibility = ["//visibility:public"],
-    )
-
     native.toolchain(
         name = name,
         toolchain = ":_autodetecting_py_runtime_pair",
         toolchain_type = ":toolchain_type",
         visibility = ["//visibility:public"],
     )
-
-    native.toolchain(
-        name = name + "_nonstrict",
-        toolchain = ":_autodetecting_py_runtime_pair_nonstrict",
-        toolchain_type = ":toolchain_type",
-        visibility = ["//visibility:public"],
-    )

python/private/autodetecting_toolchain_interpreter.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+# Don't set -e because we don't have robust trapping and printing of errors.
+set -u
+
+# We use /bin/sh rather than /bin/bash for portability. See discussion here:
+# https://groups.google.com/forum/?nomobile=true#!topic/bazel-dev/4Ql_7eDcLC0
+# We do lose the ability to set -o pipefail.
+
+FAILURE_HEADER="\
+Error occurred while attempting to use the default Python toolchain \
+(@rules_python//python:autodetecting_toolchain)."
+
+die() {
+  echo "$FAILURE_HEADER" 1>&2
+  echo "$1" 1>&2
+  exit 1
+}
+
+# We use `which` to locate the Python interpreter command on PATH. `command -v`
+# is another option, but it doesn't check whether the file it finds has the
+# executable bit.
+#
+# A tricky situation happens when this wrapper is invoked as part of running a
+# tool, e.g. passing a py_binary target to `ctx.actions.run()`. Bazel will unset
+# the PATH variable. Then the shell will see there's no PATH and initialize its
+# own, sometimes without exporting it. This causes `which` to fail and this
+# script to think there's no Python interpreter installed. To avoid this we
+# explicitly pass PATH to each `which` invocation. We can't just export PATH
+# because that would modify the environment seen by the final user Python
+# program.
+#
+# See also:
+#
+#     https://github.com/bazelbuild/continuous-integration/issues/578
+#     https://github.com/bazelbuild/bazel/issues/8414
+#     https://github.com/bazelbuild/bazel/issues/8415
+
+# Try the "python3" command name first, then fall back on "python".
+PYTHON_BIN="$(PATH="$PATH" which python3 2> /dev/null)"
+if [ -z "${PYTHON_BIN:-}" ]; then
+  PYTHON_BIN="$(PATH="$PATH" which python 2>/dev/null)"
+fi
+if [ -z "${PYTHON_BIN:-}" ]; then
+  die "Neither 'python3' nor 'python' were found on the target \
+platform's PATH, which is:
+
+$PATH
+
+Please ensure an interpreter is available on this platform (and marked \
+executable), or else register an appropriate Python toolchain as per the \
+documentation for py_runtime_pair \
+(https://github.com/bazelbuild/rules_python/blob/master/docs/python.md#py_runtime_pair)."
+fi
+
+exec "$PYTHON_BIN" "$@"
+

python/private/py_runtime_pair_macro.bzl

@@ -0,0 +1,27 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of py_runtime_pair macro portion."""
+
+load(":py_runtime_pair_rule.bzl", _py_runtime_pair = "py_runtime_pair")
+
+# A fronting macro is used because macros have user-observable behavior;
+# using one from the onset avoids introducing those changes in the future.
+def py_runtime_pair(**kwargs):
+    """Creates a py_runtime_pair target.
+
+    Args:
+        **kwargs: Keyword args to pass onto underlying rule.
+    """
+    _py_runtime_pair(**kwargs)

python/private/py_runtime_pair.bzl python/private/py_runtime_pair_rule.bzlpython/private/py_runtime_pair.bzl renamed to python/private/py_runtime_pair_rule.bzl

@@ -14,9 +14,7 @@
 
 """Implementation of py_runtime_pair."""
 
-# TODO: move py_runtime_pair into rules_python (and the rest of @bazel_tools//python)
-# py_runtime should be loaded from rules_python, but this creates a circular dep, because py_runtime_pair is imported there.
-py_runtime = native.py_runtime
+load("//python:py_runtime_info.bzl", "PyRuntimeInfo")
 
 def _py_runtime_pair_impl(ctx):
     if ctx.attr.py2_runtime != None:
@@ -47,9 +45,9 @@ def _py_runtime_pair_impl(ctx):
 
 # buildifier: disable=unused-variable
 def _is_py2_disabled(ctx):
-    # In Google, this file isn't bundled with Bazel, so we have to conditionally
+    # Because this file isn't bundled with Bazel, so we have to conditionally
     # check for this flag.
-    # TODO: Remove this once a build with the flag is released in Google
+    # TODO: Remove this once all supported Balze versions have this flag.
     if not hasattr(ctx.fragments.py, "disable_py"):
         return False
     return ctx.fragments.py.disable_py2