Make Python version pinning pin to the major version only (#1714)

For apps that do not specify an explicit Python version (e.g.: via a
`.python-version` or `runtime.txt` file), the buildpack uses a curated
default version for the first build of the app.

Then for subsequent builds of the app, the buildpack selects a Python
version based on the version found in the build cache, so that the
version used for the app doesn't change in a breaking way over time as
the buildpack's own default version changes. This feature is referred to
as "version pinning" and/or "sticky versions".

The existing implementation of this feature pinned the version to the
full Python version (e.g. `3.13.0`), meaning that the app would always use
that exact Python version, even when newer backwards-compatible patch
releases (such as `3.13.1`) became available over time.

Now that we have Python major version -> latest patch version resolution
support (as of #1658) and improved build output around cache
invalidation reasons (as of #1679), we can switch to instead only
pinning to the major Python version (e.g. `3.13`). This allows apps that
do not specify a Python version to pick up any bug and security fixes
for their major Python version the next time the app is built, whilst
still keeping the compatibility properties of version pinning.

Longer term, the plan is to deprecate/sunset version pinning entirely
(since it leads to confusing UX / lack of parity between multiple apps
deployed from the same codebase at different times, e.g. review apps), and
the Python CNB has already dropped support for it. However, that will
be a breaking change for the classic buildpack, so out of scope for now.

GUS-W-17384879.

Author: edmorley

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Changed Python version pinning behaviour for apps that do not specify a Python version. Repeat builds are now pinned to the major Python version only (`3.X`) instead of the full Python version (`3.X.Y`), so that they always use the latest patch version. ([#1714](https://github.com/heroku/heroku-buildpack-python/pull/1714))
 
 ## [v269] - 2024-12-04
 

bin/compile

@@ -106,11 +106,11 @@ hooks::run_hook "pre_compile"
 package_manager="$(package_manager::determine_package_manager "${BUILD_DIR}")"
 meta_set "package_manager" "${package_manager}"
 
-cached_python_version="$(cache::cached_python_version "${CACHE_DIR}")"
+cached_python_full_version="$(cache::cached_python_full_version "${CACHE_DIR}")"
 
 # We use the Bash 4.3+ `nameref` feature to pass back multiple values from this function
 # without having to hardcode globals. See: https://stackoverflow.com/a/38997681
-python_version::read_requested_python_version "${BUILD_DIR}" "${package_manager}" "${cached_python_version}" requested_python_version python_version_origin
+python_version::read_requested_python_version "${BUILD_DIR}" "${package_manager}" "${cached_python_full_version}" requested_python_version python_version_origin
 meta_set "python_version_reason" "${python_version_origin}"
 
 # TODO: More strongly recommend specifying a Python version (eg switch the messaging to
@@ -122,7 +122,7 @@ case "${python_version_origin}" in
 		echo "       To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes"
 		;;
 	cached)
-		output::step "No Python version was specified. Using the same version as the last build: Python ${requested_python_version}"
+		output::step "No Python version was specified. Using the same major version as the last build: Python ${requested_python_version}"
 		echo "       To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes"
 		;;
 	*)
@@ -135,7 +135,7 @@ python_major_version="${python_full_version%.*}"
 meta_set "python_version" "${python_full_version}"
 meta_set "python_version_major" "${python_major_version}"
 
-cache::restore "${BUILD_DIR}" "${CACHE_DIR}" "${STACK:?}" "${cached_python_version}" "${python_full_version}" "${package_manager}"
+cache::restore "${BUILD_DIR}" "${CACHE_DIR}" "${STACK:?}" "${cached_python_full_version}" "${python_full_version}" "${package_manager}"
 
 # The directory for the .profile.d scripts.
 mkdir -p "$(dirname "$PROFILE_PATH")"

lib/cache.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 
 # Read the full Python version of the Python install in the cache, or the empty string
 # if the cache is empty or doesn't contain a Python version metadata file.
-function cache::cached_python_version() {
+function cache::cached_python_full_version() {
 	local cache_dir="${1}"
 
 	if [[ -f "${cache_dir}/.heroku/python-version" ]]; then
@@ -29,7 +29,7 @@ function cache::restore() {
 	local build_dir="${1}"
 	local cache_dir="${2}"
 	local stack="${3}"
-	local cached_python_version="${4}"
+	local cached_python_full_version="${4}"
 	local python_full_version="${5}"
 	local package_manager="${6}"
 
@@ -48,8 +48,8 @@ function cache::restore() {
 		cache_invalidation_reasons+=("The stack has changed from ${cached_stack:-"unknown"} to ${stack}")
 	fi
 
-	if [[ "${cached_python_version}" != "${python_full_version}" ]]; then
-		cache_invalidation_reasons+=("The Python version has changed from ${cached_python_version:-"unknown"} to ${python_full_version}")
+	if [[ "${cached_python_full_version}" != "${python_full_version}" ]]; then
+		cache_invalidation_reasons+=("The Python version has changed from ${cached_python_full_version:-"unknown"} to ${python_full_version}")
 	fi
 
 	# The metadata store only exists in caches created in v252+ of the buildpack (released 2024-06-17),

lib/python_version.sh

@@ -18,6 +18,8 @@ DEFAULT_PYTHON_MAJOR_VERSION="${DEFAULT_PYTHON_FULL_VERSION%.*}"
 INT_REGEX="(0|[1-9][0-9]*)"
 # Versions of form N.N or N.N.N.
 PYTHON_VERSION_REGEX="${INT_REGEX}\.${INT_REGEX}(\.${INT_REGEX})?"
+# Versions of form N.N.N only.
+PYTHON_FULL_VERSION_REGEX="${INT_REGEX}\.${INT_REGEX}\.${INT_REGEX}"
 
 # Determine what Python version has been requested for the project.
 #
@@ -33,16 +35,13 @@ PYTHON_VERSION_REGEX="${INT_REGEX}\.${INT_REGEX}(\.${INT_REGEX})?"
 #
 # If a version wasn't specified by the app, then new apps/those with an empty cache will use
 # a buildpack default version for the first build, and then subsequent cached builds will use
-# the same Python full version in perpetuity (aka sticky versions). Sticky versioning leads to
+# the same Python major version in perpetuity (aka sticky versions). Sticky versioning leads to
 # confusing UX so is something we want to deprecate/sunset in the future (and have already done
 # so in the Python CNB).
-# TODO: Change the sticky versioning implementation so it's only sticky to the major version
-# rather than the full version, so apps that don't specify a Python version at least get
-# security patch updates.
 function python_version::read_requested_python_version() {
 	local build_dir="${1}"
 	local package_manager="${2}"
-	local cached_python_version="${3}"
+	local cached_python_full_version="${3}"
 	# We use the Bash 4.3+ `nameref` feature to pass back multiple values from this function
 	# without having to hardcode globals. See: https://stackoverflow.com/a/38997681
 	declare -n version="${4}"
@@ -75,8 +74,9 @@ function python_version::read_requested_python_version() {
 	fi
 
 	# Protect against unsupported (eg PyPy) or invalid versions being found in the cache metadata.
-	if [[ "${cached_python_version}" =~ ^${PYTHON_VERSION_REGEX}$ ]]; then
-		version="${cached_python_version}"
+	if [[ "${cached_python_full_version}" =~ ^${PYTHON_FULL_VERSION_REGEX}$ ]]; then
+		local cached_python_major_version="${cached_python_full_version%.*}"
+		version="${cached_python_major_version}"
 		origin="cached"
 	else
 		version="${DEFAULT_PYTHON_MAJOR_VERSION}"

spec/hatchet/pip_spec.rb

@@ -79,7 +79,7 @@
       app.deploy do |app|
         # The test fixture's requirements.txt is a symlink to a requirements file in a subdirectory in
         # order to test that symlinked requirements files work in general and with cache invalidation.
-        File.write('requirements/prod.txt', 'six', mode: 'a')
+        File.write('requirements/prod.txt', 'six==1.17.0', mode: 'a')
         app.commit!
         app.push!
         expect(clean_output(app.output)).to include(<<~OUTPUT)
@@ -93,12 +93,12 @@
           remote: -----> Installing dependencies using 'pip install -r requirements.txt'
           remote:        Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 5))
           remote:          Downloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB)
-          remote:        Collecting six (from -r requirements.txt (line 6))
-          remote:          Downloading six-1.16.0-py2.py3-none-any.whl.metadata (1.8 kB)
+          remote:        Collecting six==1.17.0 (from -r requirements.txt (line 6))
+          remote:          Downloading six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB)
           remote:        Downloading typing_extensions-4.12.2-py3-none-any.whl (37 kB)
-          remote:        Downloading six-1.16.0-py2.py3-none-any.whl (11 kB)
+          remote:        Downloading six-1.17.0-py2.py3-none-any.whl (11 kB)
           remote:        Installing collected packages: typing-extensions, six
-          remote:        Successfully installed six-1.16.0 typing-extensions-4.12.2
+          remote:        Successfully installed six-1.17.0 typing-extensions-4.12.2
         OUTPUT
       end
     end

spec/hatchet/python_version_spec.rb

@@ -61,20 +61,15 @@
           app.push!
           expect(clean_output(app.output)).to include(<<~OUTPUT)
             remote: -----> Python app detected
-            remote: -----> No Python version was specified. Using the same version as the last build: Python 3.12.6
+            remote: -----> No Python version was specified. Using the same major version as the last build: Python 3.12
             remote:        To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes
             remote: -----> Discarding cache since:
+            remote:        - The Python version has changed from 3.12.6 to #{LATEST_PYTHON_3_12}
             remote:        - The pip version has changed from 24.0 to #{PIP_VERSION}
-            remote: -----> Installing Python 3.12.6
-            remote: 
-            remote:  !     Warning: A Python security update is available!
-            remote:  !     
-            remote:  !     Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_12}
-            remote:  !     See: https://devcenter.heroku.com/articles/python-runtimes
-            remote: 
+            remote: -----> Installing Python #{LATEST_PYTHON_3_12}
             remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION}
           OUTPUT
-          expect(app.run('python -V')).to include('Python 3.12.6')
+          expect(app.run('python -V')).to include("Python #{LATEST_PYTHON_3_12}")
         end
       end
     end

spec/hatchet/stack_spec.rb

@@ -21,18 +21,13 @@
         app.push!
         expect(clean_output(app.output)).to include(<<~OUTPUT)
           remote: -----> Python app detected
-          remote: -----> No Python version was specified. Using the same version as the last build: Python 3.12.3
+          remote: -----> No Python version was specified. Using the same major version as the last build: Python 3.12
           remote:        To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes
           remote: -----> Discarding cache since:
           remote:        - The stack has changed from heroku-22 to heroku-24
+          remote:        - The Python version has changed from 3.12.3 to #{LATEST_PYTHON_3_12}
           remote:        - The pip version has changed
-          remote: -----> Installing Python 3.12.3
-          remote: 
-          remote:  !     Warning: A Python security update is available!
-          remote:  !     
-          remote:  !     Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_12}
-          remote:  !     See: https://devcenter.heroku.com/articles/python-runtimes
-          remote: 
+          remote: -----> Installing Python #{LATEST_PYTHON_3_12}
           remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION}
           remote: -----> Installing SQLite3
           remote: -----> Installing dependencies using 'pip install -r requirements.txt'
@@ -53,7 +48,7 @@
         app.push!
         expect(clean_output(app.output)).to include(<<~OUTPUT)
           remote: -----> Python app detected
-          remote: -----> No Python version was specified. Using the same version as the last build: Python #{DEFAULT_PYTHON_FULL_VERSION}
+          remote: -----> No Python version was specified. Using the same major version as the last build: Python #{DEFAULT_PYTHON_MAJOR_VERSION}
           remote:        To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes
           remote: -----> Discarding cache since:
           remote:        - The stack has changed from heroku-24 to heroku-22