Add support for Heroku-20 (#968)

edmorley · web-flow · commit fcf696b835d7 · 2020-10-07T19:44:33.000+01:00
This adds support for the upcoming Heroku-20 stack. The Heroku-20 Dockerfile is identical to that for Heroku-18, other than the base image, and stack-related env var changes. The initial Python versions made available will be those in: https://devcenter.heroku.com/articles/python-support#supported-runtimes https://devcenter.heroku.com/articles/python-support#supported-pypy-runtimes ...minus CPython 2.7, since it's EOL. Which are: * `python-3.6.12` * `python-3.7.9` * `python-3.8.6` * `python-3.9.0` * `pypy2.7-7.3.2` * `pypy3.6-7.3.2` Note: Unlike CPython 2.7, the PyPy 2.7 branch is still supported: https://doc.pypy.org/en/latest/faq.html#how-long-will-pypy-support-python2 In addition, I've generated binaries for each patch release immediately prior to the latest versions (with the exception of 3.9.0, since there isn't one), otherwise it's not possible to run the "out of date Python" warning tests. The binaries were generated using the process here: https://github.com/heroku/heroku-buildpack-python/blob/main/builds/README.md Specifically: ``` make deploy-runtimes STACKS='heroku-20' \ RUNTIMES='python-3.6.11 python-3.6.12 python-3.7.8 python-3.7.9 python-3.8.5 python-3.8.6 python-3.9.0 pypy2.7-7.3.1 pypy2.7-7.3.2 pypy3.6-7.3.1 pypy3.6-7.3.2' \ ENV_FILE=... ``` Binaries for the GDAL/GEOS/PROJ feature have not been generated, since it's deprecated and due for removal shortly: https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-now-deprecated Note: Like the Python 3.9.0 release, this uses the new S3 bucket, so apps will need to be using a recent version of the buildpack in order to build on Heroku-20: https://devcenter.heroku.com/articles/python-support#checking-the-python-buildpack-version Closes @W-7485877@.
diff --git a/.travis.yml b/.travis.yml
@@ -34,6 +34,10 @@ env:
     - STACK=heroku-18 TEST_CMD=test/run-deps
     - STACK=heroku-18 TEST_CMD=test/run-versions
     - STACK=heroku-18 TEST_CMD=test/run-features
+
+    - STACK=heroku-20 TEST_CMD=test/run-deps
+    - STACK=heroku-20 TEST_CMD=test/run-versions
+    - STACK=heroku-20 TEST_CMD=test/run-features
   global:
     - HATCHET_RETRIES=3
     - IS_RUNNING_ON_CI=true
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add support for Heroku-20 (#968).
 
 ## v182 (2020-10-06)
 
diff --git a/bin/steps/gdal b/bin/steps/gdal
@@ -21,8 +21,11 @@ source "$BIN_DIR/utils"
 # If GDAL exists within requirements, use vendored gdal.
 if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
 
-  if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
+  if [[ ! -f ".heroku/vendor/bin/gdalserver" && "${STACK}" == 'heroku-20' ]]; then
+    puts-warn "The buildpack's built-in GDAL functonality is not supported on Heroku-20."
+    puts-warn "Please use this buildpack instead: https://github.com/heroku/heroku-geo-buildpack"
 
+  elif [[ ! -f ".heroku/vendor/bin/gdalserver" ]]; then
     puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated."
     puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack"
 
diff --git a/bin/steps/geo-libs b/bin/steps/geo-libs
@@ -20,8 +20,11 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
-# If GDAL exists within requirements, use vendored gdal.
-if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
+if [[ "$BUILD_WITH_GEO_LIBRARIES" && "${STACK}" == 'heroku-20' ]]; then
+  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20."
+  puts-warn "Please use this buildpack for GDAL, GEOS and PROJ: https://github.com/heroku/heroku-geo-buildpack"
+  puts-warn "To hide this message, unset the BUILD_WITH_GEO_LIBRARIES variable using: heroku config:unset BUILD_WITH_GEO_LIBRARIES"
+elif [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
   mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"
 
   puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
diff --git a/builds/heroku-20.Dockerfile b/builds/heroku-20.Dockerfile
@@ -0,0 +1,20 @@
+FROM heroku/heroku:20-build
+
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="heroku-buildpack-python" \
+    S3_PREFIX="heroku-20/" \
+    STACK="heroku-20"
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt /app/
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+
+COPY . /app
diff --git a/test/fixtures/pipenv-full-version/Pipfile b/test/fixtures/pipenv-full-version/Pipfile
@@ -6,4 +6,4 @@ verify_ssl = true
 requests = "*"
 
 [requires]
-python_full_version = "3.6.3"
+python_full_version = "3.7.8"
diff --git a/test/fixtures/pipenv-full-version/Pipfile.lock b/test/fixtures/pipenv-full-version/Pipfile.lock
diff --git a/test/fixtures/python3_6_warn/runtime.txt b/test/fixtures/python3_6_warn/runtime.txt
@@ -1 +1 @@
-python-3.6.7
+python-3.6.11
diff --git a/test/fixtures/python3_7_warn/runtime.txt b/test/fixtures/python3_7_warn/runtime.txt
@@ -1 +1 @@
-python-3.7.1
+python-3.7.8
diff --git a/test/fixtures/python3_8_warn/runtime.txt b/test/fixtures/python3_8_warn/runtime.txt
@@ -1 +1 @@
-python-3.8.0
+python-3.8.5
diff --git a/test/run-deps b/test/run-deps
@@ -21,16 +21,13 @@ testGEOS() {
   local env_dir="$(mktmpdir)"
   echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
   compile 'geos' '' "${env_dir}"
-  assertCaptured "geos"
-  assertCapturedSuccess
-}
-
-testGEOSDeprecation() {
-  local env_dir="$(mktmpdir)"
-  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
-  compile 'geos' '' "${env_dir}"
-  assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated.
- !     An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
+  if [[ $STACK == "heroku-20" ]]; then
+    assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20."
+  else
+    assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
+  fi
+  # This should assertCapturedError on Heroku-20, but the test doesn't actually
+  # install anything that uses GEOS so succeeds (see W-8145375)
   assertCapturedSuccess
 }
 
@@ -55,6 +52,11 @@ testPsycopg2() {
 }
 
 testPysqlite() {
+  # pysqlite does not support Python 3 (since the sqlite3 stdlib can be used there),
+  # so we have to test with Python 2, which we've not made available for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "pysqlite"
   assertCaptured "pysqlite"
   assertCapturedSuccess
diff --git a/test/run-features b/test/run-features
@@ -68,6 +68,10 @@ testPipenvPythonVersion3_6() {
 }
 
 testPipenvPythonVersion2_7() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "pipenv-version2"
   assertCaptured "Installing ${LATEST_27}"
   # Can't use `assertCapturedSuccess` since stderr contains:
@@ -76,8 +80,12 @@ testPipenvPythonVersion2_7() {
 }
 
 testPipenvPythonFullVersion() {
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
+  fi
   compile "pipenv-full-version"
-  assertCaptured "3.6.3"
+  assertCaptured "3.7.8"
   # Can't use `assertCapturedSuccess` since stderr contains:
   # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
   assertCapturedSuccessWithStdErr
diff --git a/test/run-versions b/test/run-versions
@@ -14,6 +14,10 @@ testPythonVersionUnspecified() {
 }
 
 testPython2_7() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python2"
   assertCaptured "Installing ${LATEST_27}"
   assertCaptured "python-2-7-eol-faq";
@@ -24,6 +28,10 @@ testPython2_7() {
 }
 
 testPython2_7_warn() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python2_warn"
   assertCaptured "Installing python-2.7.15"
   assertCaptured "python-2-7-eol-faq";
@@ -33,6 +41,10 @@ testPython2_7_warn() {
 }
 
 testPython3_4() {
+  # Python 3.4 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_4"
   assertCaptured "Installing ${LATEST_34}"
   assertNotCaptured "security update"
@@ -44,6 +56,10 @@ testPython3_4() {
 }
 
 testPython3_4_warn() {
+  # Python 3.4 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_4_warn"
   assertCaptured "Installing python-3.4.9"
   assertCaptured "security update!"
@@ -54,6 +70,10 @@ testPython3_4_warn() {
 }
 
 testPython3_5() {
+  # Python 3.5 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_5"
   assertCaptured "Installing ${LATEST_35}"
   assertNotCaptured "security update"
@@ -63,6 +83,10 @@ testPython3_5() {
 }
 
 testPython3_5_warn() {
+  # Python 3.5 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_5_warn"
   assertCaptured "Installing python-3.5.6"
   assertCaptured "security update!"
@@ -81,7 +105,7 @@ testPython3_6() {
 
 testPython3_6_warn() {
   compile "python3_6_warn"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
   assertCaptured "security update!"
   assertCaptured "${LATEST_36}"
   assertCapturedSuccess
@@ -106,7 +130,7 @@ testPython3_7_warn() {
     return
   fi
   compile "python3_7_warn"
-  assertCaptured "Installing python-3.7.1"
+  assertCaptured "Installing python-3.7.8"
   assertCaptured "security update!"
   assertCaptured "${LATEST_37}"
   assertCapturedSuccess
@@ -131,7 +155,7 @@ testPython3_8_warn() {
     return
   fi
   compile "python3_8_warn"
-  assertCaptured "Installing python-3.8.0"
+  assertCaptured "Installing python-3.8.5"
   assertCaptured "security update!"
   assertCaptured "${LATEST_38}"
   assertCapturedSuccess
@@ -208,7 +232,7 @@ testPypy2_7_warn() {
 testStickyPythonVersion() {
   local cache_dir="$(mktmpdir)"
   compile "python3_6_warn" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
   assertCapturedSuccess
   compile "python_version_unspecified" "$cache_dir"
   assertNotCaptured "Installing python"
@@ -217,16 +241,16 @@ testStickyPythonVersion() {
   # Whilst this file seems like an implementation detail (so something that should
   # not be tested), we must guarantee the filename remains consistent for backwards
   # compatibility across buildpack versions for already-built apps.
-  assertFile "python-3.6.7" ".heroku/python-version"
+  assertFile "python-3.6.11" ".heroku/python-version"
 }
 
 testPythonVersionChange() {
   local cache_dir="$(mktmpdir)"
   compile "python3_6_warn" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
   assertCapturedSuccess
   compile "python3_6" "$cache_dir"
-  assertCaptured "Found python-3.6.7, removing"
+  assertCaptured "Found python-3.6.11, removing"
   assertCapturedSuccess
 }
 

Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,7 @@`
`2`	`2`
`3`	`3`	`## Unreleased`
`4`	`4`
	`5`	`+- Add support for Heroku-20 (#968).`
`5`	`6`
`6`	`7`	`## v182 (2020-10-06)`
`7`	`8`