Merge branch 'master' into patch-1

Author: CaseyFaist

.gitignore

@@ -7,3 +7,6 @@ repos/*
 
 #Venv
 buildpack/*
+
+builds/dockerenv.staging*
+builds/dockerenv.production

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
 repos:
--   repo: git://github.com/detailyang/pre-commit-shell
-    rev: 1.0.4
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
     hooks:
-    - id: shell-lint
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace

.travis.yml

@@ -1,35 +1,39 @@
 language: ruby
 dist: trusty
 sudo: required
+
 rvm:
 - 2.4.4
 before_script:
   - gem install bundler -v 1.16.2
   - bundle exec hatchet ci:setup
+
+script:
+  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
+  - docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER
+  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
+  - docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER
+  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
+  - docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER
+
 jobs:
   include:
   - stage: Bash linting (shellcheck)
     sudo: false
     script: make check
-  - stage: Stack Unit Tests
-    services: docker
-    env: STACK=heroku-18
-    script: "./tests.sh"
-  - stage: Stack Unit Tests
-    services: docker
-    env: STACK=heroku-16
-    script: "./tests.sh"
-  - stage: Stack Unit Tests
-    services: docker
-    env: STACK=cedar-14
-    script: "./tests.sh"
   - stage: Hatchet Integration
+    if: branch = master
+    name: Run Hatchet
     script: "bundle exec rspec"
 env:
+  matrix:
+    - TESTFOLDER=test/run-deps
+    - TESTFOLDER=test/run-versions
+    - TESTFOLDER=test/run-features
   global:
-  - HATCHET_RETRIES=3
-  - IS_RUNNING_ON_CI=true
-  - HATCHET_APP_LIMIT=5
-  - HATCHET_DEPLOY_STRATEGY=git
-  - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
-  - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=
+    - HATCHET_RETRIES=3
+    - IS_RUNNING_ON_CI=true
+    - HATCHET_APP_LIMIT=5
+    - HATCHET_DEPLOY_STRATEGY=git
+    - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
+    - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Python Buildpack Changelog
 
+# Master
+
+- Bump Hatchet tests to point at new default python version.
+
+- Python 3.6.9 and 3.7.4 now available.
+
+- Move get-pip utility to S3
+
+- Build utility and documentation updates
+
+--------------------------------------------------------------------------------
+
+# 155 (2019-08-22)
+
+add docs and make target for heroku-18 bob builds
+
+# 154 (2019-07-17)
+
+Fix python 3.5.7 formula actually building 3.7.2
+
+# 153 (2019-06-21)
+
+Hotfix for broken heroku-16 deploys
+
 # 152 (2019-04-04)
 
 Python 3.7.3 now available.

Makefile

@@ -1,7 +1,7 @@
 # These targets are not files
 .PHONY: tests
 
-test: test-heroku-18 test-heroku-16
+test: test-heroku-18 test-heroku-16 test-cedar-14
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
@@ -10,23 +10,23 @@ check:
 
 test-cedar-14:
 	@echo "Running tests in docker (cedar-14)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 test-heroku-16:
 	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 test-heroku-18:
 	@echo "Running tests in docker (heroku-18)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 buildenv-heroku-16:
 	@echo "Creating build environment (heroku-16)..."
 	@echo
-	@docker build --pull -t python-buildenv-heroku-16 .
+	@docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 .
 	@echo
 	@echo "Usage..."
 	@echo
@@ -36,6 +36,19 @@ buildenv-heroku-16:
 	@echo
 	@docker run -it --rm python-buildenv-heroku-16
 
+buildenv-heroku-18:
+	@echo "Creating build environment (heroku-18)..."
+	@echo
+	@docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 .
+	@echo
+	@echo "Usage..."
+	@echo
+	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
+	@echo "  $$ bob build runtimes/python-2.7.13"
+	@echo "  $$ bob deploy runtimes/python-2.7.13"
+	@echo
+	@docker run -it --rm python-buildenv-heroku-18
+
 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/

bin/default_pythons

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-DEFAULT_PYTHON_VERSION="python-3.6.8"
-LATEST_36="python-3.6.8"
-LATEST_37="python-3.7.3"
+DEFAULT_PYTHON_VERSION="python-3.6.9"
+LATEST_36="python-3.6.9"
+LATEST_37="python-3.7.4"
 LATEST_35="python-3.5.7"
 LATEST_34="python-3.4.10"
 LATEST_27="python-2.7.16"

bin/steps/python

@@ -97,6 +97,15 @@ if [ ! "$SKIP_INSTALL" ]; then
   hash -r
 fi
 
+# Heroku uses the get-pip utility maintained by the Python community to vendor Pip.
+# https://github.com/pypa/get-pip
+GETPIP="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
+
+if ! curl "${GETPIP}" -o "$ROOT_DIR/get-pip.py"; then
+    mcount "failure.python.get-pip"
+    echo "Failed to pull down get-pip"
+    exit 1
+fi
 
 # If Pip isn't up to date:
 if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
@@ -107,7 +116,7 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
   rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
   rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
 
-  /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
+  /app/.heroku/python/bin/python "$ROOT_DIR/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
   /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
 fi
 

bin/steps/sqlite3

@@ -6,6 +6,8 @@ source "$BIN_DIR/utils"
 sqlite3_version() {
   if [ "$STACK" = "cedar-14" ]; then
     SQLITE3_VERSION="3.8.2-1ubuntu2.2"
+  elif [ "$STACK" = "heroku-16" ]; then
+    SQLITE3_VERSION="3.11.0-1ubuntu1.2"
   else
     SQLITE3_VERSION=${SQLITE3_VERSION:-$(dpkg -s libsqlite3-0 | grep Version | sed 's/Version: //')}
   fi

builds/README.md

@@ -1,41 +1,57 @@
 # Python Buildpack Binaries
 
-For Cedar-14 stack
-------------------
+## Building the Docker Images
 
-To get started with it, create an app on Heroku inside a clone of this repository, and set your S3 config vars:
+**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack:
 
-    $ heroku create --buildpack https://github.com/heroku/heroku-buildpack-python#not-heroku
-    $ heroku config:set WORKSPACE_DIR=builds
-    $ heroku config:set AWS_ACCESS_KEY_ID=<your_aws_key>
-    $ heroku config:set AWS_SECRET_ACCESS_KEY=<your_aws_secret>
-    $ heroku config:set S3_BUCKET=<your_s3_bucket_name>
+    $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
+    $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
+    $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
 
+## Using the Image
 
-Then, shell into an instance and run a build by giving the name of the formula inside `builds`:
+You can e.g. `bash` into each of the images you built using their tag:
 
-    $ heroku run bash
-    Running `bash` attached to terminal... up, run.6880
-    ~ $ bob build runtimes/python-2.7.6
+    docker run --rm -ti heroku-python-build-cedar-14 bash
+    docker run --rm -ti heroku-python-build-heroku-16 bash
+    docker run --rm -ti heroku-python-build-heroku-18 bash
 
-    Fetching dependencies... found 2:
-      - libraries/sqlite
+You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`:
 
-    Building formula runtimes/python-2.7.6:
-        === Building Python 2.7.6
-        Fetching Python v2.7.6 source...
-        Compiling...
+    docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15
 
-If this works, run `bob deploy` instead of `bob build` to have the result uploaded to S3 for you.
+In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details.
 
-To speed things up drastically, it'll usually be a good idea to `heroku run bash --size PX` instead.
+## Configuration
 
-For Heroku-16 stack
--------------------
+File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`.
 
-1. Ensure GNU Make and Docker are installed.
-2. From the root of the buildpack repository, run: `make buildenv-heroku-16`
-3. Follow the instructions displayed!
+Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials.
 
+### Passing AWS credentials to the container
 
-Enjoy :)
+If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file.
+
+#### Passing credentials explicitly
+
+    docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash
+
+#### Passing credentials through  the environment
+
+The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in:
+
+    AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+
+or
+
+    export AWS_ACCESS_KEY_ID=...
+    export AWS_SECRET_ACCESS_KEY=...
+    docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+
+#### Passing credentials through a separate env file
+
+This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values.
+
+For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`.
+
+    docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash

Dockerfile.heroku-18 renamed to builds/cedar-14.Dockerfile

@@ -1,13 +1,15 @@
-FROM heroku/heroku:18-build
+FROM heroku/cedar:14
 
 WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
     S3_BUCKET="lang-python" \
-    S3_PREFIX="heroku-18/"
+    S3_PREFIX="cedar-14/" \
+    DEBIAN_FRONTEND=noninteractive \
+    STACK="cedar-14"
 
 RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
 
 COPY requirements.txt /app/
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+RUN pip install -r /app/requirements.txt
 
 COPY . /app