Re-apply Pipenv --system bug workaround

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Re-apply a workaround for a Pipenv bug when using `--system`, that causes packages to not be installed correctly if they are also a dependency of Pipenv (such as `certifi` or `packaging`). ([#2011](https://github.com/heroku/heroku-buildpack-python/pull/2011))
 
 ## [v329] - 2026-01-08
 
@@ -197,7 +198,7 @@
 
 ## [v293] - 2025-07-23
 
-- Work around a Pipenv bug when using `--system`, that causes packages to not be installed correctly if they are also a dependency of Pipenv (such as `certifi` ). ([#1842](https://github.com/heroku/heroku-buildpack-python/pull/1842))
+- Work around a Pipenv bug when using `--system`, that causes packages to not be installed correctly if they are also a dependency of Pipenv (such as `certifi`). ([#1842](https://github.com/heroku/heroku-buildpack-python/pull/1842))
 
 ## [v292] - 2025-07-23
 

lib/pipenv.sh

@@ -81,17 +81,31 @@ function pipenv::install_pipenv() {
 	export PATH="${pipenv_bin_dir}:${PATH}"
 	# Force Pipenv to manage the system Python site-packages instead of using venvs.
 	export PIPENV_SYSTEM="1"
+	# Hide Pipenv's notice about finding/using an existing virtual environment.
+	export PIPENV_VERBOSITY="-1"
+	# Work around a Pipenv bug when using `--system`, whereby it doesn't correctly install dependencies
+	# that happen to also be a dependency of Pipenv (such as `certifi` and `packaging`). In general
+	# Pipenv's support for its `--system` mode is very buggy. Longer term we should explore moving
+	# to venvs, however, that will need to be coordinated across all package managers and will also
+	# change paths for Python which could break other use cases. Be careful removing this even if the
+	# `pip_basic` test that installs certifi/packaging still passes, since the repro seems to depend
+	# on specific package version combinations / other factors and so the testcase is very fragile.
+	export VIRTUAL_ENV="${python_home}"
 
 	# Set the same env vars in the environment used by later buildpacks.
 	cat >>"${export_file}" <<-EOF
 		export PATH="${pipenv_bin_dir}:\${PATH}"
 		export PIPENV_SYSTEM="1"
+		export PIPENV_VERBOSITY="-1"
+		export VIRTUAL_ENV="${python_home}"
 	EOF
 
 	# And the environment used at app run-time.
 	cat >>"${profile_d_file}" <<-EOF
 		export PATH="${pipenv_bin_dir}:\${PATH}"
 		export PIPENV_SYSTEM="1"
+		export PIPENV_VERBOSITY="-1"
+		export VIRTUAL_ENV="${python_home}"
 	EOF
 }
 

spec/fixtures/pipenv_basic/Pipfile

@@ -4,8 +4,11 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-certifi = "*"
 typing-extensions = "*"
+# Test that dependencies that happen to also be vendored dependencies of Pipenv are correctly installed,
+# since Pipenv's --system mode is buggy and requires a workaround to ensure they aren't skipped.
+certifi = "*"
+packaging = "*"
 
 [dev-packages]
 

spec/fixtures/pipenv_basic/bin/compile

@@ -23,6 +23,10 @@ pip list --disable-pip-version-check --exclude pip
 echo
 
 python -c 'import typing_extensions; print(typing_extensions)'
+# Test that dependencies that happen to also be vendored dependencies of Pipenv are correctly installed,
+# since Pipenv's --system mode is buggy and requires a workaround to ensure they aren't skipped.
+python -c 'import certifi; print(certifi)'
+python -c 'import packaging; print(packaging)'
 echo
 
 jq --sort-keys '.' "${CACHE_DIR}/build-data/python.json"

spec/hatchet/ci_spec.rb

@@ -105,8 +105,6 @@
           -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION}
           -----> Installing Pipenv #{PIPENV_VERSION}
           -----> Installing dependencies using 'pipenv install --deploy --dev'
-                 Installing dependencies from Pipfile.lock \\(.+\\)...
-                 Installing dependencies from Pipfile.lock \\(.+\\)...
           -----> Skipping Django collectstatic since the env var DISABLE_COLLECTSTATIC is set.
           -----> Running bin/post_compile hook
                  BUILD_DIR=/app
@@ -123,8 +121,10 @@
                  LIBRARY_PATH=/app/.heroku/python/lib
                  PATH=/app/.heroku/python/pipenv/bin:/app/.heroku/python/bin:/usr/local/bin:/usr/bin:/bin:/app/.sprettur/bin/
                  PIPENV_SYSTEM=1
+                 PIPENV_VERBOSITY=-1
                  PKG_CONFIG_PATH=/app/.heroku/python/lib/pkg-config
                  PYTHONUNBUFFERED=1
+                 VIRTUAL_ENV=/app/.heroku/python
           -----> Saving cache
 
            !     Note: We recently added support for the package manager uv:
@@ -143,9 +143,11 @@
           LIBRARY_PATH=/app/.heroku/python/lib
           PATH=/app/.heroku/python/bin:/app/.heroku/python/pipenv/bin:/usr/local/bin:/usr/bin:/bin:/app/.sprettur/bin/
           PIPENV_SYSTEM=1
+          PIPENV_VERBOSITY=-1
           PYTHONHOME=/app/.heroku/python
           PYTHONPATH=/app
           PYTHONUNBUFFERED=true
+          VIRTUAL_ENV=/app/.heroku/python
           -----> No test-setup command provided. Skipping.
           -----> Running test command `./bin/print-env-vars.sh && pytest --version`...
           CI=true
@@ -157,9 +159,11 @@
           LIBRARY_PATH=/app/.heroku/python/lib
           PATH=/app/.heroku/python/bin:/app/.heroku/python/pipenv/bin:/usr/local/bin:/usr/bin:/bin:/app/.sprettur/bin/:/app/.sprettur/bin/
           PIPENV_SYSTEM=1
+          PIPENV_VERBOSITY=-1
           PYTHONHOME=/app/.heroku/python
           PYTHONPATH=/app
           PYTHONUNBUFFERED=true
+          VIRTUAL_ENV=/app/.heroku/python
           WEB_CONCURRENCY=5
           pytest .+
           -----> test command `./bin/print-env-vars.sh && pytest --version` completed successfully
@@ -173,8 +177,6 @@
           -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION}
           -----> Using cached Pipenv #{PIPENV_VERSION}
           -----> Installing dependencies using 'pipenv install --deploy --dev'
-                 Installing dependencies from Pipfile.lock \\(.+\\)...
-                 Installing dependencies from Pipfile.lock \\(.+\\)...
           -----> Skipping Django collectstatic since the env var DISABLE_COLLECTSTATIC is set.
           -----> Running bin/post_compile hook
         REGEX

spec/hatchet/pipenv_spec.rb

@@ -15,7 +15,6 @@
           remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION}
           remote: -----> Installing Pipenv #{PIPENV_VERSION}
           remote: -----> Installing dependencies using 'pipenv install --deploy'
-          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Running bin/post_compile hook
           remote:        BUILD_DIR=/tmp/build_.+
           remote:        CACHE_DIR=/tmp/codon/tmp/cache
@@ -27,8 +26,10 @@
           remote:        LIBRARY_PATH=/app/.heroku/python/lib
           remote:        PATH=/app/.heroku/python/pipenv/bin:/app/.heroku/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
           remote:        PIPENV_SYSTEM=1
+          remote:        PIPENV_VERBOSITY=-1
           remote:        PKG_CONFIG_PATH=/app/.heroku/python/lib/pkg-config
           remote:        PYTHONUNBUFFERED=1
+          remote:        VIRTUAL_ENV=/app/.heroku/python
           remote: -----> Saving cache
           remote: 
           remote:  !     Note: We recently added support for the package manager uv:
@@ -47,9 +48,11 @@
           remote: LIBRARY_PATH=/app/.heroku/python/lib
           remote: PATH=/app/.heroku/python/bin:/app/.heroku/python/pipenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
           remote: PIPENV_SYSTEM=1
+          remote: PIPENV_VERBOSITY=-1
           remote: PYTHONHOME=/app/.heroku/python
           remote: PYTHONPATH=/app
           remote: PYTHONUNBUFFERED=true
+          remote: VIRTUAL_ENV=/app/.heroku/python
           remote: 
           remote: \\['',
           remote:  '/app',
@@ -61,10 +64,13 @@
           remote: pipenv, version #{PIPENV_VERSION}
           remote: Package           Version
           remote: ----------------- -+
-          remote: certifi           2025.11.12
+          remote: certifi           2026.1.4
+          remote: packaging         25.0
           remote: typing_extensions 4.15.0
           remote: 
           remote: <module 'typing_extensions' from '/app/.heroku/python/lib/python3.14/site-packages/typing_extensions.py'>
+          remote: <module 'certifi' from '/app/.heroku/python/lib/python3.14/site-packages/certifi/__init__.py'>
+          remote: <module 'packaging' from '/app/.heroku/python/lib/python3.14/site-packages/packaging/__init__.py'>
           remote: 
           remote: \\{
           remote:   "cache_restore_duration": [0-9.]+,
@@ -99,7 +105,6 @@
           remote: -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION}
           remote: -----> Using cached Pipenv #{PIPENV_VERSION}
           remote: -----> Installing dependencies using 'pipenv install --deploy'
-          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Running bin/post_compile hook
           remote:        .+
           remote: -----> Saving cache
@@ -115,9 +120,11 @@
           LIBRARY_PATH=/app/.heroku/python/lib
           PATH=/app/.heroku/python/bin:/app/.heroku/python/pipenv/bin:/usr/local/bin:/usr/bin:/bin
           PIPENV_SYSTEM=1
+          PIPENV_VERBOSITY=-1
           PYTHONHOME=/app/.heroku/python
           PYTHONPATH=/app
           PYTHONUNBUFFERED=true
+          VIRTUAL_ENV=/app/.heroku/python
           WEB_CONCURRENCY=2
           pipenv, version #{PIPENV_VERSION}
         OUTPUT
@@ -143,7 +150,6 @@
           remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION}
           remote: -----> Installing Pipenv #{PIPENV_VERSION}
           remote: -----> Installing dependencies using 'pipenv install --deploy'
-          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Saving cache
         REGEX
       end
@@ -198,7 +204,6 @@
           remote: -----> Installing Python 3.10.0
           remote: -----> Installing Pipenv #{PIPENV_VERSION}
           remote: -----> Installing dependencies using 'pipenv install --deploy'
-          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Saving cache
         REGEX
       end
@@ -216,7 +221,6 @@
           remote: -----> Installing Python #{LATEST_PYTHON_3_13}
           remote: -----> Installing Pipenv #{PIPENV_VERSION}
           remote: -----> Installing dependencies using 'pipenv install --deploy'
-          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Saving cache
         REGEX
       end
@@ -275,7 +279,6 @@
           remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION}
           remote: -----> Installing Pipenv #{PIPENV_VERSION}
           remote: -----> Installing dependencies using 'pipenv install --deploy'
-          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Saving cache
         REGEX
       end
@@ -468,7 +471,6 @@
           remote: -----> Installing Python #{LATEST_PYTHON_3_14}
           remote: -----> Installing Pipenv #{PIPENV_VERSION}
           remote: -----> Installing dependencies using 'pipenv install --deploy'
-          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Saving cache
         REGEX
       end
@@ -483,7 +485,6 @@
       app.deploy do |app|
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE))
           remote: -----> Installing dependencies using 'pipenv install --deploy'
-          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Running bin/post_compile hook
           remote:        __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'}
           remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
@@ -526,7 +527,6 @@
         app.push!
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE))
           remote: -----> Installing dependencies using 'pipenv install --deploy'
-          remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Running bin/post_compile hook
           remote:        __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'}
           remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}