fix(pypi): use python -B for all invocations

Before this change we would just invoke the Python interpreter. This
means that in the `rules_python` directory there would be `__pycache__`
folders created in the source tree and the same `__pycache__` folders
would be created in the python interpreter repository rules if the
directories were writable.

This change ensures that we are executing `python` with `-B` in those
contexts and reduces any likelihood of us doing the wrong thing.

Work towards #1169.

Author: aignas

CHANGELOG.md

@@ -109,6 +109,9 @@ Unreleased changes template.
   {obj}`--venvs_use_declare_symlink=no` to have it not create symlinks at
   build time (they will be created at runtime instead).
   (Fixes [#2489](https://github.com/bazelbuild/rules_python/issues/2489))
+* (pypi) From now on `python` invocations in repository and module extension
+  evaluation contexts will invoke Python interpreter with `-B` to avoid
+  creating `.pyc` files.
 
 {#v1-2-0-added}
 ### Added

python/private/pypi/evaluate_markers.bzl

@@ -55,12 +55,12 @@ def evaluate_markers(mrctx, *, requirements, python_interpreter, python_interpre
     pypi_repo_utils.execute_checked(
         mrctx,
         op = "ResolveRequirementEnvMarkers({})".format(in_file),
+        python = pypi_repo_utils.resolve_python_interpreter(
+            mrctx,
+            python_interpreter = python_interpreter,
+            python_interpreter_target = python_interpreter_target,
+        ),
         arguments = [
-            pypi_repo_utils.resolve_python_interpreter(
-                mrctx,
-                python_interpreter = python_interpreter,
-                python_interpreter_target = python_interpreter_target,
-            ),
             "-m",
             "python.private.pypi.requirements_parser.resolve_target_platforms",
             in_file,

python/private/pypi/patch_whl.bzl

@@ -27,8 +27,8 @@ other patches ensures that the users have overview on exactly what has changed
 within the wheel.
 """
 
-load("//python/private:repo_utils.bzl", "repo_utils")
 load(":parse_whl_name.bzl", "parse_whl_name")
+load(":pypi_repo_utils.bzl", "pypi_repo_utils")
 
 _rules_python_root = Label("//:BUILD.bazel")
 
@@ -102,10 +102,14 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs):
     record_patch = rctx.path("RECORD.patch")
     whl_patched = patched_whl_name(whl_input.basename)
 
-    repo_utils.execute_checked(
+    pypi_repo_utils.execute_checked(
         rctx,
+        python = python_interpreter,
+        srcs = [
+            "//python/private/pypi:repack_whl.py",
+            "//tools:wheelmaker.py",
+        ],
         arguments = [
-            python_interpreter,
             "-m",
             "python.private.pypi.repack_whl",
             "--record-patch",

python/private/pypi/pypi_repo_utils.bzl

@@ -104,11 +104,27 @@ def _construct_pypath(mrctx, *, entries):
     ])
     return pypath
 
-def _execute_checked(mrctx, *, srcs, **kwargs):
+def _execute_prep(mrctx, *, python, srcs, **kwargs):
+    for src in srcs:
+        # This will ensure that we will re-evaluate the bzlmod extension or
+        # refetch the repository_rule when the srcs change. This should work on
+        # Bazel versions without `mrctx.watch` as well.
+        repo_utils.watch(mrctx, mrctx.path(src))
+
+    environment = kwargs.pop("environment", {})
+    pythonpath = environment.get("PYTHONPATH", "")
+    if pythonpath and not types.is_string(pythonpath):
+        environment["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath)
+    kwargs["environment"] = environment
+    kwargs["arguments"] = [python, "-B"] + kwargs.get("arguments", [])
+    return kwargs
+
+def _execute_checked(mrctx, *, python, srcs, **kwargs):
     """Helper function to run a python script and modify the PYTHONPATH to include external deps.
 
     Args:
         mrctx: Handle to the module_ctx or repository_ctx.
+        python: The python interpreter to use.
         srcs: The src files that the script depends on. This is important to
             ensure that the Bazel repository cache or the bzlmod lock file gets
             invalidated when any one file changes. It is advisable to use
@@ -118,26 +134,34 @@ def _execute_checked(mrctx, *, srcs, **kwargs):
             the `environment` has a value `PYTHONPATH` and it is a list, then
             it will be passed to `construct_pythonpath` function.
     """
+    return repo_utils.execute_checked(
+        mrctx,
+        **_execute_prep(mrctx, python = python, srcs = srcs, **kwargs)
+    )
 
-    for src in srcs:
-        # This will ensure that we will re-evaluate the bzlmod extension or
-        # refetch the repository_rule when the srcs change. This should work on
-        # Bazel versions without `mrctx.watch` as well.
-        repo_utils.watch(mrctx, mrctx.path(src))
-
-    env = kwargs.pop("environment", {})
-    pythonpath = env.get("PYTHONPATH", "")
-    if pythonpath and not types.is_string(pythonpath):
-        env["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath)
+def _execute_checked_stdout(mrctx, *, python, srcs, **kwargs):
+    """Helper function to run a python script and modify the PYTHONPATH to include external deps.
 
-    return repo_utils.execute_checked(
+    Args:
+        mrctx: Handle to the module_ctx or repository_ctx.
+        python: The python interpreter to use.
+        srcs: The src files that the script depends on. This is important to
+            ensure that the Bazel repository cache or the bzlmod lock file gets
+            invalidated when any one file changes. It is advisable to use
+            `RECORD` files for external deps and the list of srcs from the
+            rules_python repo for any scripts.
+        **kwargs: Arguments forwarded to `repo_utils.execute_checked`. If
+            the `environment` has a value `PYTHONPATH` and it is a list, then
+            it will be passed to `construct_pythonpath` function.
+    """
+    return repo_utils.execute_checked_stdout(
         mrctx,
-        environment = env,
-        **kwargs
+        **_execute_prep(mrctx, python = python, srcs = srcs, **kwargs)
     )
 
 pypi_repo_utils = struct(
     construct_pythonpath = _construct_pypath,
     execute_checked = _execute_checked,
+    execute_checked_stdout = _execute_checked_stdout,
     resolve_python_interpreter = _resolve_python_interpreter,
 )

python/private/pypi/whl_library.bzl

@@ -75,14 +75,15 @@ def _get_toolchain_unix_cflags(rctx, python_interpreter, logger = None):
     if not is_standalone_interpreter(rctx, python_interpreter, logger = logger):
         return []
 
-    stdout = repo_utils.execute_checked_stdout(
+    stdout = pypi_repo_utils.execute_checked_stdout(
         rctx,
         op = "GetPythonVersionForUnixCflags",
+        python = python_interpreter,
         arguments = [
-            python_interpreter,
             "-c",
             "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')",
         ],
+        srcs = [],
     )
     _python_version = stdout
     include_path = "{}/include/python{}".format(
@@ -181,7 +182,6 @@ def _whl_library_impl(rctx):
         python_interpreter_target = rctx.attr.python_interpreter_target,
     )
     args = [
-        python_interpreter,
         "-m",
         "python.private.pypi.whl_installer.wheel_installer",
         "--requirement",
@@ -247,6 +247,7 @@ def _whl_library_impl(rctx):
             # truncate the requirement value when logging it / reporting
             # progress since it may contain several ' --hash=sha256:...
             # --hash=sha256:...' substrings that fill up the console
+            python = python_interpreter,
             op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]),
             arguments = args,
             environment = environment,
@@ -295,6 +296,7 @@ def _whl_library_impl(rctx):
     pypi_repo_utils.execute_checked(
         rctx,
         op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
+        python = python_interpreter,
         arguments = args + [
             "--whl-file",
             whl_path,