gh-127440: Try using base_executable to detect prefix and exec_prefix

Lib/test/test_embed.py

@@ -1635,7 +1635,8 @@ def test_init_pyvenv_cfg(self):
                 print("home = %s" % pyvenv_home, file=fp)
                 print("include-system-site-packages = false", file=fp)
 
-            paths = self.module_search_paths()
+            paths = self.module_search_paths(prefix=tmpdir,
+                                             exec_prefix=tmpdir)
             if not MS_WINDOWS:
                 paths[-1] = lib_dynload
             else:
@@ -1649,7 +1650,7 @@ def test_init_pyvenv_cfg(self):
             base_executable = os.path.join(pyvenv_home, os.path.basename(executable))
             exec_prefix = pyvenv_home
             config = {
-                'base_prefix': sysconfig.get_config_var("prefix"),
+                'base_prefix': tmpdir,
                 'base_exec_prefix': exec_prefix,
                 'exec_prefix': tmpdir,
                 'prefix': tmpdir,

Lib/test/test_getpath.py

@@ -386,6 +386,43 @@ def test_venv_changed_name_posix(self):
         actual = getpath(ns, expected)
         self.assertEqual(expected, actual)
 
+    def test_venv_with_invalid_home_in_pyvenv_posix(self):
+        ns = MockPosixNamespace(
+            argv0="/venv/bin/python",
+            PREFIX="/usr",
+            ENV_PATH="/venv/bin:/usr/bin",
+        )
+        ns.add_known_xfile("/path/to/copy_dir/bin/python")
+        ns.add_known_xfile("/path/to/python-link")
+        ns.add_known_link("/path/to/python-link",
+                          "/path/to/copy_dir/bin/python")
+
+        ns.add_known_xfile("/venv/bin/python")
+        ns.add_known_link("/venv/bin/python",
+                          "/path/to/python-link")
+
+        ns.add_known_file("/path/to/copy_dir/lib/python9.8/os.py")
+        ns.add_known_dir("/path/to/copy_dir/lib/python9.8/lib-dynload")
+        ns.add_known_file("/venv/pyvenv.cfg", [
+            r"home = /"
+        ])
+        expected = dict(
+            executable="/venv/bin/python",
+            prefix="/venv",
+            exec_prefix="/venv",
+            base_executable="/path/to/copy_dir/bin/python",
+            base_prefix="/path/to/copy_dir",
+            base_exec_prefix="/path/to/copy_dir",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/path/to/copy_dir/lib/python98.zip",
+                "/path/to/copy_dir/lib/python9.8",
+                "/path/to/copy_dir/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
     def test_venv_non_installed_zip_path_posix(self):
         "Test a venv created from non-installed python has correct zip path."""
         ns = MockPosixNamespace(

Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst

@@ -0,0 +1 @@
+Fix calculation of ``base_prefix``  and ``base_exec_prefix`` when the ``home`` in ``pyvenv.cfg`` is inaccurate and the installation directory (``PREFIX``) does not exist. Calculation will try to use ``base_executable`` now. Patch by Charlie Zhao.

Modules/getpath.py

@@ -236,6 +236,7 @@ def search_up(prefix, *landmarks, test=isfile):
 
 real_executable_dir = None
 platstdlib_dir = None
+pyvenvcfg_home = None
 
 # ******************************************************************************
 # CALCULATE program_name
@@ -377,7 +378,7 @@ def search_up(prefix, *landmarks, test=isfile):
             # Override executable_dir/real_executable_dir with the value from 'home'.
             # These values may be later used to calculate prefix/base_prefix, if a more
             # reliable source — like the runtime library (libpython) path — isn't available.
-            executable_dir = real_executable_dir = value.strip()
+            pyvenvcfg_home = executable_dir = real_executable_dir = value.strip()
             # If base_executable — which points to the Python interpreted from
             # the base installation — isn't set (eg. when embedded), try to find
             # it in 'home'.
@@ -608,10 +609,16 @@ def search_up(prefix, *landmarks, test=isfile):
         if prefix and not stdlib_dir:
             stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
 
-    if PREFIX and not prefix:
+    if PREFIX and STDLIB_LANDMARKS and not prefix and any(isfile(joinpath(PREFIX, f)) for f in STDLIB_LANDMARKS):
         prefix = PREFIX
-        if not any(isfile(joinpath(prefix, f)) for f in STDLIB_LANDMARKS):
-            warn('Could not find platform independent libraries <prefix>')
+
+    # Note: the `home` variable in pyvenv.cfg is not always accurate.
+    # Detect prefix by searching from *real* executable location for the stdlib_dir.
+    # See details: https://github.com/python/cpython/issues/127440
+    if pyvenvcfg_home and STDLIB_SUBDIR and STDLIB_LANDMARKS and base_executable and not prefix:
+        prefix = search_up(base_executable, *STDLIB_LANDMARKS)
+        if prefix and not stdlib_dir:
+            stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
 
     if not prefix:
         prefix = abspath('')
@@ -627,6 +634,12 @@ def search_up(prefix, *landmarks, test=isfile):
             exec_prefix = prefix
         if not exec_prefix and executable_dir:
             exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir)
+
+        # See details: https://github.com/python/cpython/issues/127440
+        if pyvenvcfg_home and not exec_prefix and base_executable:
+            base_executable_dir = dirname(base_executable)
+            exec_prefix = search_up(base_executable_dir, PLATSTDLIB_LANDMARK, test=isdir)
+
         if not exec_prefix and EXEC_PREFIX:
             exec_prefix = EXEC_PREFIX
         if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)):