fix: add flag to use runtime venv creation when using bootstrap=script

CHANGELOG.md

@@ -77,6 +77,10 @@ Unreleased changes template.
   The related issue is [#908](https://github.com/bazelbuild/rules_python/issue/908).
 * (sphinxdocs) Do not crash when `tag_class` does not have a populated `doc` value.
   Fixes ([#2579](https://github.com/bazelbuild/rules_python/issues/2579)).
+* (binaries/tests) Fix packaging when using `--bootstrap_impl=script`: set
+  {obj}`--venvs_use_declare_symlink=no` to have it avoid creating symlinks at
+  build time.
+  Fixes ([#2489](https://github.com/bazelbuild/rules_python/issues/2489)
 
 {#v0-0-0-added}
 ### Added

MODULE.bazel

@@ -84,6 +84,7 @@ bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True)
 bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True)
 bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True)
 bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
+bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True)
 
 # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests.
 # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides.

docs/api/rules_python/python/config_settings/index.md

@@ -212,6 +212,28 @@ Values:
 :::
 ::::
 
+::::{bzl:flag} venvs_use_declare_symlink
+
+Determines if relative symlinks are created using `declare_symlink()` at build
+time.
+
+This is only intended to work around
+[#2489](https://github.com/bazelbuild/rules_python/issues/2489), where some
+packaging rules don't support `declare_symlink()` artifacts.
+
+Values:
+* `yes`: Use `declare_symlink()` and create relative symlinks at build time.
+* `no`: Do not use `declare_symlink()`. Instead, the venv will be created at
+  runtime.
+
+:::{seealso}
+{envvar}`RULES_PYTHON_EXTRACT_ROOT` for customizing where the runtime venv
+is created.
+:::
+
+:::{versionadded} VERSION_NEXT_PATCH
+:::
+
 ::::{bzl:flag} bootstrap_impl
 Determine how programs implement their startup process.
 

docs/environment-variables.md

@@ -1,28 +1,53 @@
 # Environment Variables
 
-:::{envvar} RULES_PYTHON_REPO_DEBUG
+:::{envvar} RULES_PYTHON_BOOTSTRAP_VERBOSE
 
-When `1`, repository rules will print debug information about what they're
+When `1`, debug information about bootstrapping of a program is printed to
+stderr.
+:::
+
+:::{envvar} RULES_PYTHON_BZLMOD_DEBUG
+
+When `1`, bzlmod extensions will print debug information about what they're
 doing. This is mostly useful for development to debug errors.
 :::
 
-:::{envvar} RULES_PYTHON_REPO_DEBUG_VERBOSITY
+:::{envvar} RULES_PYTHON_DEPRECATION_WARNINGS
 
-Determines the verbosity of logging output for repo rules. Valid values:
+When `1`, the rules_python will warn users about deprecated functionality that will
+be removed in a subsequent major `rules_python` version. Defaults to `0` if unset.
+:::
 
-* `DEBUG`
-* `INFO`
-* `TRACE`
+:::{envvar} RULES_PYTHON_ENABLE_PYSTAR
+
+When `1`, the rules_python Starlark implementation of the core rules is used
+instead of the Bazel-builtin rules. Note this requires Bazel 7+.
 :::
 
-:::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH
+:::{envvar} RULES_PYTHON_EXTRACT_ROOT
 
-Determines the python interpreter platform to be used for a particular
-interpreter `(version, os, arch)` triple to be used in repository rules.
-Replace the `VERSION_OS_ARCH` part with actual values when using, e.g.
-`3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the
-os, arch values are the same as the ones mentioned in the
-`//python:versions.bzl` file.
+Directory to use as the root for creating files necessary for bootstrapping so
+that a binary can run.
+
+Only applicable when {obj}`--venvs_use_declare_symlink=no` is used.
+
+When set, a binary will attempt to find a unique, reusable, location within this
+directory for the files it needs to create to aid startup. The files may not be
+deleted upon program exit; it is the responsibility of the caller to ensure
+cleanup.
+
+Manually specifying the directory is useful to lower the overhead of
+extracting/creating files on every program execution. By using a location
+outside /tmp, longer lived programs don't have to worry about files in /tmp
+being cleaned up by the OS.
+
+If not set, then a temporary directory will be created and deleted upon program
+exit.
+:::
+
+:::{envvar} RULES_PYTHON_GAZELLE_VERBOSE
+
+When `1`, debug information from gazelle is printed to stderr.
 :::
 
 :::{envvar} RULES_PYTHON_PIP_ISOLATED
@@ -34,37 +59,32 @@ Valid values:
 * Other non-empty values mean to use isolated mode.
 :::
 
-:::{envvar} RULES_PYTHON_BZLMOD_DEBUG
+:::{envvar} RULES_PYTHON_REPO_DEBUG
 
-When `1`, bzlmod extensions will print debug information about what they're
+When `1`, repository rules will print debug information about what they're
 doing. This is mostly useful for development to debug errors.
 :::
 
-:::{envvar} RULES_PYTHON_DEPRECATION_WARNINGS
-
-When `1`, the rules_python will warn users about deprecated functionality that will
-be removed in a subsequent major `rules_python` version. Defaults to `0` if unset.
-:::
+:::{envvar} RULES_PYTHON_REPO_DEBUG_VERBOSITY
 
-:::{envvar} RULES_PYTHON_ENABLE_PYSTAR
+Determines the verbosity of logging output for repo rules. Valid values:
 
-When `1`, the rules_python Starlark implementation of the core rules is used
-instead of the Bazel-builtin rules. Note this requires Bazel 7+.
+* `DEBUG`
+* `INFO`
+* `TRACE`
 :::
 
-:::{envvar} RULES_PYTHON_BOOTSTRAP_VERBOSE
+:::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH
 
-When `1`, debug information about bootstrapping of a program is printed to
-stderr.
+Determines the python interpreter platform to be used for a particular
+interpreter `(version, os, arch)` triple to be used in repository rules.
+Replace the `VERSION_OS_ARCH` part with actual values when using, e.g.
+`3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the
+os, arch values are the same as the ones mentioned in the
+`//python:versions.bzl` file.
 :::
 
 :::{envvar} VERBOSE_COVERAGE
 
 When `1`, debug information about coverage behavior is printed to stderr.
 :::
-
-
-:::{envvar} RULES_PYTHON_GAZELLE_VERBOSE
-
-When `1`, debug information from gazelle is printed to stderr.
-:::

python/config_settings/BUILD.bazel

@@ -9,6 +9,7 @@ load(
     "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
+    "VenvsUseDeclareSymlinkFlag",
 )
 load(
     "//python/private/pypi:flags.bzl",
@@ -121,6 +122,13 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+string_flag(
+    name = "venvs_use_declare_symlink",
+    build_setting_default = VenvsUseDeclareSymlinkFlag.YES,
+    values = VenvsUseDeclareSymlinkFlag.flag_values(),
+    visibility = ["//visibility:public"],
+)
+
 # pip.parse related flags
 
 string_flag(

python/private/flags.bzl

@@ -123,6 +123,21 @@ PrecompileSourceRetentionFlag = enum(
     get_effective_value = _precompile_source_retention_flag_get_effective_value,
 )
 
+def _venvs_use_declare_symlink_flag_get_value(ctx):
+    return ctx.attr._venvs_use_declare_symlink_flag[BuildSettingInfo].value
+
+# Decides if the venv created by bootstrap=script uses declare_file() to
+# create relative symlinks. Workaround for #2489 (packaging rules not supporting
+# declare_link() files).
+# buildifier: disable=name-conventions
+VenvsUseDeclareSymlinkFlag = FlagEnum(
+    # Use declare_file() and relative symlinks in the venv
+    YES = "yes",
+    # Do not use declare_file() and relative symlinks in the venv
+    NO = "no",
+    get_value = _venvs_use_declare_symlink_flag_get_value,
+)
+
 # Used for matching freethreaded toolchains and would have to be used in wheels
 # as well.
 # buildifier: disable=name-conventions

python/private/py_executable.bzl

@@ -51,7 +51,7 @@ load(
     "target_platform_has_any_constraint",
     "union_attrs",
 )
-load(":flags.bzl", "BootstrapImplFlag")
+load(":flags.bzl", "BootstrapImplFlag", "VenvsUseDeclareSymlinkFlag")
 load(":precompile.bzl", "maybe_precompile")
 load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo")
 load(":py_executable_info.bzl", "PyExecutableInfo")
@@ -195,6 +195,10 @@ accepting arbitrary Python versions.
         "_python_version_flag": attr.label(
             default = "//python/config_settings:python_version",
         ),
+        "_venvs_use_declare_symlink_flag": attr.label(
+            default = "//python/config_settings:venvs_use_declare_symlink",
+            providers = [BuildSettingInfo],
+        ),
         "_windows_constraints": attr.label_list(
             default = [
                 "@platforms//os:windows",
@@ -512,7 +516,25 @@ def _create_venv(ctx, output_prefix, imports, runtime_details):
     ctx.actions.write(pyvenv_cfg, "")
 
     runtime = runtime_details.effective_runtime
-    if runtime.interpreter:
+    venvs_use_declare_symlink_enabled = (
+        VenvsUseDeclareSymlinkFlag.get_value(ctx) == VenvsUseDeclareSymlinkFlag.YES
+    )
+
+    if not venvs_use_declare_symlink_enabled:
+        if runtime.interpreter:
+            interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path)
+        else:
+            interpreter_actual_path = runtime.interpreter_path
+
+        py_exe_basename = paths.basename(interpreter_actual_path)
+
+        # When the venv symlinks are disabled, the $venv/bin/python3 file isn't
+        # needed or used at runtime. However, the zip code uses the interpreter
+        # File object to figure out some paths.
+        interpreter = ctx.actions.declare_file("{}/bin/{}".format(venv, py_exe_basename))
+        ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path))
+
+    elif runtime.interpreter:
         py_exe_basename = paths.basename(runtime.interpreter.short_path)
 
         # Even though ctx.actions.symlink() is used, using
@@ -571,6 +593,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details):
 
     return struct(
         interpreter = interpreter,
+        recreate_venv_at_runtime = not venvs_use_declare_symlink_enabled,
         # Runfiles root relative path or absolute path
         interpreter_actual_path = interpreter_actual_path,
         files_without_interpreter = [pyvenv_cfg, pth, site_init],
@@ -657,15 +680,13 @@ def _create_stage1_bootstrap(
     else:
         python_binary_path = runtime_details.executable_interpreter_path
 
-    if is_for_zip and venv:
-        python_binary_actual = venv.interpreter_actual_path
-    else:
-        python_binary_actual = ""
+    python_binary_actual = venv.interpreter_actual_path if venv else ""
 
     subs = {
         "%is_zipfile%": "1" if is_for_zip else "0",
         "%python_binary%": python_binary_path,
         "%python_binary_actual%": python_binary_actual,
+        "%recreate_venv_at_runtime%": str(int(venv.recreate_venv_at_runtime)) if venv else "0",
         "%target%": str(ctx.label),
         "%workspace_name%": ctx.workspace_name,
     }

python/private/stage1_bootstrap_template.sh

@@ -9,15 +9,17 @@ fi
 # runfiles-relative path
 STAGE2_BOOTSTRAP="%stage2_bootstrap%"
 
-# runfiles-relative path
+# runfiles-relative path to python interpreter to use
 PYTHON_BINARY='%python_binary%'
 # The path that PYTHON_BINARY should symlink to.
 # runfiles-relative path, absolute path, or single word.
-# Only applicable for zip files.
+# Only applicable for zip files or when venv is recreated at runtime.
 PYTHON_BINARY_ACTUAL="%python_binary_actual%"
 
 # 0 or 1
 IS_ZIPFILE="%is_zipfile%"
+# 0 or 1
+RECREATE_VENV_AT_RUNTIME="%recreate_venv_at_runtime%"
 
 if [[ "$IS_ZIPFILE" == "1" ]]; then
   # NOTE: Macs have an old version of mktemp, so we must use only the
@@ -104,6 +106,7 @@ python_exe=$(find_python_interpreter $RUNFILES_DIR $PYTHON_BINARY)
 # Zip files have to re-create the venv bin/python3 symlink because they
 # don't contain it already.
 if [[ "$IS_ZIPFILE" == "1" ]]; then
+  use_exec=0
   # It should always be under runfiles, but double check this. We don't
   # want to accidentally create symlinks elsewhere.
   if [[ "$python_exe" != $RUNFILES_DIR/* ]]; then
@@ -121,13 +124,60 @@ if [[ "$IS_ZIPFILE" == "1" ]]; then
     symlink_to=$(which $PYTHON_BINARY_ACTUAL)
     # Guard against trying to symlink to an empty value
     if [[ $? -ne 0 ]]; then
-      echo >&2 "ERROR: Python to use found on PATH: $PYTHON_BINARY_ACTUAL"
+      echo >&2 "ERROR: Python to use not found on PATH: $PYTHON_BINARY_ACTUAL"
       exit 1
     fi
   fi
   # The bin/ directory may not exist if it is empty.
   mkdir -p "$(dirname $python_exe)"
   ln -s "$symlink_to" "$python_exe"
+elif [[ "$RECREATE_VENV_AT_RUNTIME" == "1" ]]; then
+  if [[ -n "$RULES_PYTHON_EXTRACT_ROOTT" ]]; then
+    use_exec=1
+    # Use our runfiles path as a unique, reusable, location for the
+    # binary-specific venv being created.
+    venv="$RULES_PYTHON_EXTRACT_ROOTT/$(dirname $(dirname $PYTHON_BINARY))"
+    mkdir -p $RULES_PYTHON_EXTRACT_ROOTT
+  else
+    # Re-exec'ing can't be used because we have to clean up the temporary
+    # venv directory that is created.
+    use_exec=0
+    venv=$(mktemp -d)
+    if [[ -n "$venv" && -z "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then
+      trap 'rm -fr "$venv"' EXIT
+    fi
+  fi
+
+  if [[ "$PYTHON_BINARY_ACTUAL" == /* ]]; then
+    # An absolute path, i.e. platform runtime, e.g. /usr/bin/python3
+    symlink_to=$PYTHON_BINARY_ACTUAL
+  elif [[ "$PYTHON_BINARY_ACTUAL" == */* ]]; then
+    # A runfiles-relative path
+    symlink_to="$RUNFILES_DIR/$PYTHON_BINARY_ACTUAL"
+  else
+    # A plain word, e.g. "python3". Symlink to where PATH leads
+    symlink_to=$(which $PYTHON_BINARY_ACTUAL)
+    # Guard against trying to symlink to an empty value
+    if [[ $? -ne 0 ]]; then
+      echo >&2 "ERROR: Python to use not found on PATH: $PYTHON_BINARY_ACTUAL"
+      exit 1
+    fi
+  fi
+  mkdir -p "$venv/bin"
+  # Match the basename; some tools, e.g. pyvenv key off the executable name
+  python_exe="$venv/bin/$(basename $PYTHON_BINARY_ACTUAL)"
+  if [[ ! -e "$python_exe" ]]; then
+    ln -s "$symlink_to" "$python_exe"
+  fi
+  runfiles_venv="$RUNFILES_DIR/$(dirname $(dirname $PYTHON_BINARY))"
+  if [[ ! -e "$venv/pyvenv.cfg" ]]; then
+    ln -s "$runfiles_venv/pyvenv.cfg" "$venv/pyvenv.cfg"
+  fi
+  if [[ ! -e "$venv/lib" ]]; then
+    ln -s "$runfiles_venv/lib" "$venv/lib"
+  fi
+else
+  use_exec=1
 fi
 
 # At this point, we should have a valid reference to the interpreter.
@@ -165,7 +215,6 @@ if [[ "$IS_ZIPFILE" == "1" ]]; then
   interpreter_args+=("-XRULES_PYTHON_ZIP_DIR=$zip_dir")
 fi
 
-
 export RUNFILES_DIR
 
 command=(
@@ -184,9 +233,10 @@ command=(
 # See https://github.com/bazelbuild/rules_python/issues/2043#issuecomment-2215469971
 # for more information.
 #
-# However, when running a zip file, we need to clean up the workspace after the
-# process finishes so control must return here.
-if [[ "$IS_ZIPFILE" == "1" ]]; then
+# However, we can't use exec when there is cleanup to do afterwards. Control
+# must return to this process so it can run the trap handlers. Such cases
+# occur when zip mode or recreate_venv_at_runtime creates temporary files.
+if [[ "$use_exec" == "0" ]]; then
   "${command[@]}"
   exit $?
 else