fix(py_venv): Partially resolve PYTHONHOME (#668)

arrdem · web-flow · commit 8a557da5f8f8 · 2025-10-14T15:35:52.000-06:00
Use the partially resolved interpreter path (eg. the first partially resolved path for the interpreter binary which is under `.runfiles/`) as the basis from which to identify the `pyvenv.cfg` file and thus as the `sys.prefix`. This fixes an issue which would allow the interpreter to observe an un-resolved `sys.prefix` relative to which `_aspect.pth` source imports would be invalid. Fixes #666. ### Changes are visible to end-users: yes - Searched for relevant documentation and updated as needed: yes - Breaking change (forces users to change their own code or config): no - Suggested release notes appear below: yes Fixed a bug which would prevent 1stparty and other relative imports from working as desired when venvs are consumed outside of Bazel. ### Test plan - Manual testing; please provide instructions so we can reproduce: Using `chicagotrading/shorty` at `venv-bug-repro`, ``` ❯ git diff | cat diff --git a/MODULE.bazel b/MODULE.bazel index 5f63c5f..b8d3dea 100755 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -20,3 +20,8 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( python_version = "3.12", ) + +local_path_override( + module_name = "aspect_rules_py", + path = "/Users/arrdem/Documents/work/aspect/rules_py", +) ❯ bazel run //app:app_bin.venv INFO: Analyzed target //app:app_bin.venv (1 packages loaded, 876 targets configured). INFO: Found 1 target... Target //app:app_bin.venv up-to-date: bazel-bin/app/app_bin.venv INFO: Elapsed time: 1.056s, Critical Path: 0.02s INFO: 1 process: 577 action cache hit, 1 internal. INFO: Build completed successfully, 1 total action INFO: Running command line: bazel-bin/app/app_bin.venv final "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/bin/python" [aspect] venv root "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv" venv.cfg "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/pyvenv.cfg" [aspect] PyCfg { root: "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv", cfg: "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/pyvenv.cfg", version_info: "3.12.0", interpreter: Runfiles { rpath: "rules_python++python+python_3_12_aarch64-apple-darwin/bin/python3", repo: "_main" }, user_site: false } [aspect] "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/bin/python3" [aspect] Attempting to execute: "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/external/rules_python++python+python_3_12_aarch64-apple-darwin/bin/python3" with argv[0] as "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/bin/python3" and args as ["-B", "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/aspect_rules_py+/py/private/py_venv/link.py"] Linking: /private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv -> /Users/arrdem/Documents/work/aspect/clients/chicago-trading/shorty/.app+app_bin.venv Link is up to date! To configure the virtualenv in your IDE, configure an interpreter with the homedir /Users/arrdem/Documents/work/aspect/clients/chicago-trading/shorty/.app+app_bin.venv Please note that you may encounter issues if your editor doesn't evaluate the `activate` script. If you do please file an issue at https://github.com/aspect-build/rules_py/issues/new?template=BUG-REPORT.yaml To activate the virtualenv in your shell run source /Users/arrdem/Documents/work/aspect/clients/chicago-trading/shorty/.app+app_bin.venv/bin/activate virtualenvwrapper users may further want to $ ln -s /Users/arrdem/Documents/work/aspect/clients/chicago-trading/shorty/.app+app_bin.venv $WORKON_HOME/.app+app_bin.venv ❯ source .app+app_bin.venv/bin/activate ❯ python3 -m app final "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/bin/python3" [aspect] venv root "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv" venv.cfg "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/pyvenv.cfg" [aspect] PyCfg { root: "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv", cfg: "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/pyvenv.cfg", version_info: "3.12.0", interpreter: Runfiles { rpath: "rules_python++python+python_3_12_aarch64-apple-darwin/bin/python3", repo: "_main" }, user_site: false } [aspect] "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/bin/python3" [aspect] Attempting to execute: "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/external/rules_python++python+python_3_12_aarch64-apple-darwin/bin/python3" with argv[0] as "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/bin/python3" and args as ["-m", "app"] hi from foo_lib Hi from __main__ ❯ python3 -P -m app final "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/bin/python3" [aspect] venv root "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv" venv.cfg "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/pyvenv.cfg" [aspect] PyCfg { root: "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv", cfg: "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/pyvenv.cfg", version_info: "3.12.0", interpreter: Runfiles { rpath: "rules_python++python+python_3_12_aarch64-apple-darwin/bin/python3", repo: "_main" }, user_site: false } [aspect] "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/bin/python3" [aspect] Attempting to execute: "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/external/rules_python++python+python_3_12_aarch64-apple-darwin/bin/python3" with argv[0] as "/private/var/tmp/_bazel_arrdem/6292cdb66616265d0039b3e456edaa70/execroot/_main/bazel-out/darwin_arm64-fastbuild/bin/app/app_bin.venv.runfiles/_main/app/.app_bin.venv/bin/python3" and args as ["-P", "-m", "app"] hi from foo_lib Hi from __main__ ```
diff --git a/py/defs.bzl b/py/defs.bzl
@@ -93,6 +93,7 @@ def _py_binary_or_test(name, rule, srcs, main, data = [], deps = [], resolutions
 
     _py_venv_link(
         name = "{}.venv".format(name),
+        srcs = srcs,
         data = data,
         deps = deps,
         imports = kwargs.get("imports"),
diff --git a/py/tools/venv_shim/src/main.rs b/py/tools/venv_shim/src/main.rs
@@ -12,16 +12,6 @@ use std::process::Command;
 const PYVENV: &str = "pyvenv.cfg";
 
 fn find_venv_root(exec_name: impl AsRef<Path>) -> miette::Result<(PathBuf, PathBuf)> {
-    if let Ok(it) = env::var("VIRTUAL_ENV") {
-        let root = PathBuf::from(it);
-        let cfg = root.join(PYVENV);
-        if cfg.exists() {
-            return Ok((root, cfg));
-        } else {
-            eprintln!("Warning: $VIRTUAL_ENV is set but seems to be invalid; ignoring")
-        }
-    }
-
     let exec_name = exec_name.as_ref().to_owned();
     if let Some(parent) = exec_name.parent().and_then(|it| it.parent()) {
         let cfg = parent.join(PYVENV);
@@ -216,13 +206,6 @@ fn main() -> miette::Result<()> {
         miette::bail!("could not discover an execution command-line");
     };
 
-    let (venv_root, venv_cfg) = find_venv_root(exec_name)?;
-
-    let venv_config = parse_venv_cfg(&venv_root, &venv_cfg)?;
-
-    // The logical path of the interpreter
-    let venv_interpreter = venv_root.join("bin/python3");
-
     // Alright this is a bit of a mess.
     //
     // There is a std::env::current_exe(), but it has platform dependent
@@ -305,7 +288,24 @@ fn main() -> miette::Result<()> {
 
     #[cfg(feature = "debug")]
     eprintln!("final  {:?}", executable);
-    let actual_interpreter = find_actual_interpreter(executable, &venv_config)?;
+
+    // Now that we've identified where the .runfiles venv really is, we want to
+    // use that as the basis for configuring our virtualenv and setting
+    // everything else up.
+    let (venv_root, venv_cfg) = find_venv_root(&executable)?;
+    #[cfg(feature = "debug")]
+    eprintln!("[aspect] venv root {:?} venv.cfg {:?}", venv_root, venv_cfg);
+
+    let venv_config = parse_venv_cfg(&venv_root, &venv_cfg)?;
+    #[cfg(feature = "debug")]
+    eprintln!("[aspect] {:?}", venv_config);
+
+    // The logical path of the interpreter
+    let venv_interpreter = venv_root.join("bin/python3");
+    #[cfg(feature = "debug")]
+    eprintln!("[aspect] {:?}", venv_interpreter);
+
+    let actual_interpreter = find_actual_interpreter(&executable, &venv_config)?;
 
     #[cfg(feature = "debug")]
     eprintln!(
@@ -347,6 +347,9 @@ fn main() -> miette::Result<()> {
     // Set the executable pointer for MacOS, but we do it consistently
     cmd.env("PYTHONEXECUTABLE", &venv_interpreter);
 
+    // Clobber VIRTUAL_ENV which may have been set by activate to an unresolved path
+    cmd.env("VIRTUAL_ENV", &venv_root);
+
     // Similar to `-s` but this avoids us having to muck with the argv in ways
     // that could be visible to the called program.
     if !venv_config.user_site {
