Refactor S3 asset URL handling (#1085)

Previously the buildpack's S3 bucket was defined in two places - once
in `VENDOR_URL` and again during the pip installation step. This
duplication was necessary since `VENDOR_URL` also contained the stack's
name, whereas the pip use-case used a non-stack-specific S3 key prefix.

In order to:
* reduce this duplication
* simplify this buildpack's S3 bucket migration (where we'll soon be
  needing the vary the bucket name and wouldn't want to have to
  duplicate that logic in multiple places)
* allow overriding of the URL for the pip use-case

...the `VENDOR_URL` variable has been replaced with `S3_BASE_URL` which
no longer contains the stack name.

The user-configurable override has similarly been renamed from
`BUILDPACK_VENDOR_URL` to `BUILDPACK_S3_BASE_URL`. Note: As before,
this override cannot be set via standard app variables (see #989).

The unused `USE_STAGING_BINARIES` environment variable has been
removed, since it's a leftover from the project to stand up a staging S3 bucket.
It's redundant given the `BUILDPACK_S3_BASE_URL` variable.

Closes @W-8142401@.

Author: edmorley

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Replace `BUILDPACK_VENDOR_URL` and `USE_STAGING_BINARIES` with `BUILDPACK_S3_BASE_URL` (#1085).
 
 ## v181 (2020-09-29)
 

bin/compile

@@ -37,17 +37,15 @@ ENV_DIR=$3
 # Export Path variables, for use in sub-scripts.
 export BUILD_DIR CACHE_DIR ENV_DIR
 
-# Set the Buildpack's internet target for downloading Python distributions.
-# The user can provide BUILDPACK_VENDOR_URL to specify a custom target.
+# Set the base URL for downloading buildpack assets like Python runtimes.
+# The user can provide BUILDPACK_S3_BASE_URL to specify a custom target.
 # Note: this is designed for non-Heroku use, as it does not use the user-provided
 # environment variable mechanism (the ENV_DIR).
-VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK"
-if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then
-    VENDOR_URL="$BUILDPACK_VENDOR_URL"
-elif [[ -n ${USE_STAGING_BINARIES} ]]; then
-    VENDOR_URL="$USE_STAGING_BINARIES/$STACK"
-fi
-export VENDOR_URL
+DEFAULT_S3_BASE_URL='https://lang-python.s3.amazonaws.com'
+S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-${DEFAULT_S3_BASE_URL}}"
+# This has to be exported since it's used by the geo-libs step which is run in a subshell.
+# TODO: Stop exporting once the geo-libs step is removed or no longer uses `sub_env`.
+export S3_BASE_URL
 
 # Default Python Versions
 # shellcheck source=bin/default_pythons

bin/steps/gdal

@@ -10,7 +10,7 @@
 # This script is invoked by [`bin/compile`](/).
 
 # The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
+VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz"
 
 PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 

bin/steps/geo-libs

@@ -10,9 +10,9 @@
 # This script is invoked by [`bin/compile`](/).
 
 # The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
-VENDORED_GEOS="${VENDOR_URL}/libraries/vendor/geos.tar.gz"
-VENDORED_PROJ="${VENDOR_URL}/libraries/vendor/proj.tar.gz"
+VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz"
+VENDORED_GEOS="${S3_BASE_URL}/${STACK}/libraries/vendor/geos.tar.gz"
+VENDORED_PROJ="${S3_BASE_URL}/${STACK}/libraries/vendor/proj.tar.gz"
 
 PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 

bin/steps/pylibmc

@@ -15,7 +15,7 @@ if [[ "$STACK" != "cedar-14" ]]; then
 fi
 
 # The location of the pre-compiled libmemcached binary.
-VENDORED_MEMCACHED="${VENDOR_URL}/libraries/vendor/libmemcache.tar.gz"
+VENDORED_MEMCACHED="${S3_BASE_URL}/${STACK}/libraries/vendor/libmemcache.tar.gz"
 
 # Syntax sugar.
 # shellcheck source=bin/utils

bin/steps/python

@@ -5,7 +5,7 @@ runtime-fixer runtime.txt
 PYTHON_VERSION=$(cat runtime.txt)
 
 # The location of the pre-compiled python binary.
-VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
+VENDORED_PYTHON="${S3_BASE_URL}/${STACK}/runtimes/${PYTHON_VERSION}.tar.gz"
 
 SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
 SECURITY_UPDATE_PYPY="The PyPy project has released a security update! Please consider upgrading to"
@@ -160,7 +160,7 @@ fi
 # Instead, we use the pip wheel to install itself, using the method described here:
 # https://github.com/pypa/pip/issues/2351#issuecomment-69994524
 PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
-PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL_URL="${S3_BASE_URL}/common/${PIP_WHEEL_FILENAME}"
 PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"
 
 if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then

test/run-features

@@ -113,10 +113,10 @@ testHooks() {
     PROFILE_PATH
     PWD
     PYTHONUNBUFFERED
+    S3_BASE_URL
     SHLVL
     SOME_APP_CONFIG_VAR
     STACK
-    VENDOR_URL
   )
   if [[ "${STACK}" == "cedar-14" || "${STACK}" == "heroku-16" ]]; then
     # Remove "OLDPWD" from expected_env_vars since for bash <4.4 it's not exported to subshells: