Harden __main__.py against non-hermetic stdlib. (#3117)

Fixes #3116

Author: jsirois

CHANGES.md

@@ -1,5 +1,13 @@
 # Release Notes
 
+## 2.91.2
+
+This release fixes hermeticity of the Pex boot code against the Python stdlib itself. In some corner
+cases stdlib modules could be loaded from the PYTHONPATH and lead to failure of the Pex boot code
+before reaching its `sys.path` scrubbing code.
+
+* Harden `__main__.py` against non-hermetic stdlib. (#3117)
+
 ## 2.91.1
 
 This release partially fixes an interpreter caching bug for CPython interpreters that have the same

pex/bin/pex.py

@@ -344,8 +344,8 @@ def configure_clp_pex_options(parser):
         default=True,
         help=(
             "If --venv is specified, don't rewrite Python script shebangs in the venv to pass "
-            "`-sE` to the interpreter; for example, to enable running the venv PEX itself or its "
-            "Python scripts with a custom `PYTHONPATH`."
+            "`-I` (or `-sE` for older Pythons) to the interpreter; for example, to enable running "
+            "the venv PEX itself or its Python scripts with a custom `PYTHONPATH`."
         ),
     )
 

pex/pex.py

@@ -569,8 +569,8 @@ def path(self):
         """Return the path this PEX was built at."""
         return self._pex
 
-    def execute(self):
-        # type: () -> Any
+    def execute(self, python_args=()):
+        # type: (Sequence[str]) -> Any
         """Execute the PEX.
 
         This function makes assumptions that it is the last function called by the interpreter.
@@ -607,14 +607,14 @@ def execute(self):
                     V=3,
                 )
 
-        result = self._wrap_coverage(self._wrap_profiling, self._execute)
+        result = self._wrap_coverage(self._wrap_profiling, self._execute, python_args)
         if "PYTHONINSPECT" not in os.environ:
             sys.exit(0 if isinstance(result, Globals) else result)
         else:
             return result
 
-    def _execute(self):
-        # type: () -> Any
+    def _execute(self, python_args):
+        # type: (Sequence[str]) -> Any
         force_interpreter = self._vars.PEX_INTERPRETER
 
         self._clean_environment(strip_pex_env=self._pex_info.strip_pex_env)
@@ -629,7 +629,7 @@ def _execute(self):
 
         if force_interpreter:
             TRACER.log("PEX_INTERPRETER specified, dropping into interpreter")
-            return self.execute_interpreter()
+            return self.execute_interpreter(python_args)
 
         if not any(
             (
@@ -640,7 +640,7 @@ def _execute(self):
             )
         ):
             TRACER.log("No entry point specified, dropping into interpreter")
-            return self.execute_interpreter()
+            return self.execute_interpreter(python_args)
 
         if self._pex_info_overrides.script and self._pex_info_overrides.entry_point:
             return "Cannot specify both script and entry_point for a PEX!"
@@ -663,32 +663,35 @@ def _execute(self):
             assert self._pex_info.entry_point
             return self.execute_entry(parse_entry_point(self._pex_info.entry_point))
 
-    def execute_interpreter(self):
-        # type: () -> Any
+    def execute_interpreter(self, python_args):
+        # type: (Sequence[str]) -> Any
 
         # A Python interpreter always inserts the CWD at the head of the sys.path.
         # See https://docs.python.org/3/library/sys.html#sys.path
         sys.path.insert(0, "")
 
         args = sys.argv[1:]
-        python_options = []
+        python_options = list(python_args)
+        called_with_python_options = False
         for index, arg in enumerate(args):
             # Check if the arg is an expected startup arg.
             if arg.startswith("-") and arg not in ("-", "-c", "-m"):
-                python_options.append(arg)
+                # N.B.: In the face of short options that can be combined, this is a naive check.
+                if arg not in python_options:
+                    python_options.append(arg)
+                    called_with_python_options = True
             else:
                 args = args[index:]
                 break
         else:
             # All the args were python options
             args = []
 
-        # The pex was called with Python interpreter options
-        if python_options:
-            return self.execute_with_options(python_options, args)
+        if called_with_python_options:
+            return self.re_execute_with_options(python_options, args)
 
         if args:
-            # NB: We take care here to setup sys.argv to match how CPython does it for each case.
+            # NB: We take care here to set up sys.argv to match how CPython does it for each case.
             arg = args[0]
             if arg == "-c":
                 content = args[1]
@@ -734,7 +737,7 @@ def execute_interpreter(self):
             return Globals(pex_repl())
 
     @staticmethod
-    def execute_with_options(
+    def re_execute_with_options(
         python_options,  # type: List[str]
         args,  # List[str]
     ):
@@ -749,7 +752,7 @@ def execute_with_options(
             return "Unable to resolve PEX __main__ module file: {}".format(main)
 
         python = sys.executable
-        cmdline = [python] + python_options + [main.__file__] + args
+        cmdline = [python] + python_options + [os.path.dirname(main.__file__)] + args
         TRACER.log(
             "Re-executing with Python interpreter options: cmdline={cmdline!r}".format(
                 cmdline=" ".join(cmdline)

pex/pex_boot.py

@@ -13,7 +13,7 @@
 if sys.version_info >= (3, 10):
 
     def orig_argv():
-        # type: () -> List[str]
+        # type: () -> Optional[List[str]]
         return sys.orig_argv
 
 else:
@@ -24,7 +24,7 @@ def orig_argv():
         from ctypes import pythonapi
 
         def orig_argv():
-            # type: () -> List[str]
+            # type: () -> Optional[List[str]]
 
             # Under MyPy for Python 3.5, ctypes.POINTER is incorrectly typed. This code is tested
             # to work correctly in practice on all Pythons Pex supports.
@@ -42,8 +42,8 @@ def orig_argv():
     except ImportError:
         # N.B.: This handles the older PyPy case.
         def orig_argv():
-            # type: () -> List[str]
-            return []
+            # type: () -> Optional[List[str]]
+            return None
 
 
 def __re_exec__(
@@ -147,6 +147,7 @@ def boot(
     bootstrap_dir,  # type: str
     pex_root,  # type: str
     pex_hash,  # type: str
+    hermetic_boot,  # type: bool
     has_interpreter_constraints,  # type: bool
     pex_path,  # type: Tuple[str, ...]
     is_venv,  # type: bool
@@ -175,11 +176,49 @@ def boot(
 
     python_args = list(inject_python_args)  # type: List[str]
     orig_args = orig_argv()
-    if orig_args:
+    if orig_args is not None:
+        orig_python_args = []  # type: List[str]
         for index, arg in enumerate(orig_args[1:], start=1):
             if os.path.exists(arg) and os.path.samefile(entry_point, arg):
-                python_args.extend(orig_args[1:index])
+                orig_python_args = orig_args[1:index]
                 break
+        python_args.extend(orig_python_args)
+
+        # N.B.: This re-exec is shown to cost ~3% of runtime (~30ms) for small, quick zipapp PEXes
+        # on a cold boot; so we avoid the re-exec unless PYTHONPATH is set, which should be the
+        # only common way to pollute the stdlib in a manner that doesn't permanently break the
+        # underlying Python environment in the first place.
+        if (
+            "PYTHONPATH" in os.environ
+            and __SHOULD_EXECUTE__
+            and hermetic_boot
+            and os.environ.get("PEX_INHERIT_PATH", "false") == "false"
+        ):
+            re_exec = False
+            if sys.version_info[:2] >= (3, 4):
+                if "-I" not in orig_python_args:
+                    python_args.append("-I")
+                    re_exec = True
+            else:
+                has_hermetic_args = (
+                    ("-s" in orig_python_args and "-E" in orig_python_args)
+                    or "-sE" in orig_python_args
+                    or "-Es" in orig_python_args
+                )
+                if not has_hermetic_args:
+                    python_args.append("-sE")
+                    re_exec = True
+            if re_exec:
+                args = [sys.executable]
+                args.extend(python_args)
+                args.append(entry_point)
+                args.extend(sys.argv[1:])
+                if os.name == "nt":
+                    import subprocess
+
+                    sys.exit(subprocess.call(args=args))
+                else:
+                    os.execv(args[0], args)
 
     installed_from = os.environ.pop(__INSTALLED_FROM__, None)
     if installed_from:

pex/pex_bootstrapper.py

@@ -700,7 +700,7 @@ def bootstrap_pex(
             from . import pex
 
             try:
-                return pex.PEX(entry_point).execute()
+                return pex.PEX(entry_point).execute(python_args=python_args)
             except pex.PEX.Error as e:
                 return e
 

pex/pex_builder.py

@@ -33,6 +33,7 @@
 from pex.executables import chmod_plus_x, create_sh_python_redirector_shebang
 from pex.finders import get_entry_point_from_console_script, get_script_from_distributions
 from pex.fs import safe_rename, safe_symlink
+from pex.inherit_path import InheritPath
 from pex.installed_wheel import InstalledWheel
 from pex.interpreter import PythonInterpreter
 from pex.layout import Layout
@@ -483,12 +484,18 @@ def _prepare_code(self):
         with open(os.path.join(_ABS_PEX_PACKAGE_DIR, "pex_boot.py")) as fp:
             pex_boot = fp.read()
 
+        is_venv = self._pex_info.venv
+        hermetic_boot = (is_venv and self._pex_info.venv_hermetic_scripts) or (
+            not is_venv and self._pex_info.inherit_path is InheritPath.FALSE
+        )
+
         pex_main = dedent(
             """
             result, should_exit, is_globals = boot(
                 bootstrap_dir={bootstrap_dir!r},
                 pex_root={pex_root!r},
                 pex_hash={pex_hash!r},
+                hermetic_boot={hermetic_boot!r},
                 has_interpreter_constraints={has_interpreter_constraints!r},
                 pex_path={pex_path!r},
                 is_venv={is_venv!r},
@@ -503,9 +510,10 @@ def _prepare_code(self):
             bootstrap_dir=self._pex_info.bootstrap,
             pex_root=self._pex_info.raw_pex_root,
             pex_hash=self._pex_info.pex_hash,
+            hermetic_boot=hermetic_boot,
             has_interpreter_constraints=bool(self._pex_info.interpreter_constraints),
             pex_path=self._pex_info.pex_path,
-            is_venv=self._pex_info.venv,
+            is_venv=is_venv,
             inject_python_args=self._pex_info.inject_python_args,
         )
         bootstrap = pex_boot + "\n" + pex_main

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.91.1"
+__version__ = "2.91.2"

tests/integration/scie/test_pex_scie.py

@@ -951,7 +951,9 @@ def assert_output(
             in output_lines
         )
         stderr_import_logging = any(line.startswith("import ") for line in output_lines)
-        assert expect_python_verbose is stderr_import_logging, "\n".join(output_lines)
+        assert expect_python_verbose is stderr_import_logging, "\n".join(
+            output_lines[:5] + ["..."] + output_lines[-5:]
+        )
 
     assert_output(
         args=[busybox, "foo-script-ad-hoc"],

tests/integration/test_issue_3116.py

@@ -0,0 +1,87 @@
+# Copyright 2026 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import os.path
+import subprocess
+import sys
+
+import pytest
+
+from pex.common import safe_rmtree
+from pex.interpreter import PythonInterpreter
+from pex.layout import Layout
+from testing import IS_PYPY, make_env, run_pex_command
+from testing.cli import run_pex3
+from testing.pytest_utils.tmp import Tempdir
+
+
+@pytest.mark.parametrize(
+    "layout", [pytest.param(layout, id=layout.value) for layout in Layout.values()]
+)
+def test_enum_backport_injection_foiled(
+    tmpdir,  # type: Tempdir
+    current_interpreter,  # type: PythonInterpreter
+    layout,  # type: Layout.Value
+):
+    # type: (...) -> None
+
+    if IS_PYPY and layout is Layout.ZIPAPP:
+        pytest.skip(
+            reason=(
+                "PyPy imports site when loading a zipapp __main__.py and blows up before we can do "
+                "anything."
+            )
+        )
+
+    pythonpath = tmpdir.join("pythonpath")
+    run_pex3("venv", "create", "--layout", "flat", "-d", pythonpath, "enum34").assert_success()
+
+    pex_root = tmpdir.join("pex-root")
+    pex = tmpdir.join("cowsay.pex")
+    pex_main = pex if layout is Layout.ZIPAPP else os.path.join(pex, "pex")
+
+    run_pex_command(
+        args=[
+            "--runtime-pex-root",
+            pex_root,
+            "cowsay<6",
+            "-c",
+            "cowsay",
+            "-o",
+            pex,
+            "--layout",
+            layout.value,
+        ]
+    ).assert_success()
+
+    assert b"| Moo! |" in subprocess.check_output(args=[pex_main, "Moo!"])
+
+    safe_rmtree(pex_root)
+    assert b"| Boo! |" in subprocess.check_output(args=[sys.executable, pex_main, "Boo!"])
+
+    interpreter = sys.executable
+    extra_env = {}
+    if 0 != subprocess.call(args=[sys.executable, "-c", ""], env=make_env(PYTHONPATH=pythonpath)):
+        # N.B.: Injecting `enum34` on the `sys.path` is so insidious that it can foil Python startup
+        # on its own. In particular, when the python interpreter sits in a venv with modern pep-660
+        # editables (the Pex test infra case for newer Pythons), those editables `.pth` will fail
+        # to load early on interpreter startup. Pex is not in the backtrace there, and we can't help
+        # users in that case - Python itself is poisoned. To avoid that case in tests though, we
+        # resolve out of our venv interpreter to get the underlying system interpreter with no such
+        # editable installs / `.pth` files.
+        interpreter = current_interpreter.resolve_base_interpreter().binary
+        extra_env = dict(
+            PATH=os.pathsep.join((os.path.dirname(interpreter), os.environ.get("PATH", os.defpath)))
+        )
+
+    safe_rmtree(pex_root)
+    assert b"| Foo! |" in subprocess.check_output(
+        args=[pex_main, "Foo!"], env=make_env(PYTHONPATH=pythonpath, **extra_env)
+    )
+
+    safe_rmtree(pex_root)
+    assert b"| Bar! |" in subprocess.check_output(
+        args=[interpreter, pex_main, "Bar!"], env=make_env(PYTHONPATH=pythonpath)
+    )

tests/test_pex.py

@@ -752,7 +752,7 @@ def test_execute_interpreter_dashm_module_with_python_options():
         )
         stdout, stderr = process.communicate()
 
-        assert b"" == stderr
+        assert b"" == stderr, stderr
         assert "{} one two\n".format(
             os.path.realpath(os.path.join(pex_chroot, "foo/bar.py"))
         ) == stdout.decode("utf-8")