Merge pull request heroku#799 from heroku/build-cleanup

Build cleanup

Author: CaseyFaist

.gitignore

@@ -7,3 +7,6 @@ repos/*
 
 #Venv
 buildpack/*
+
+builds/dockerenv.staging*
+builds/dockerenv.production

.travis.yml

@@ -9,10 +9,12 @@ before_script:
   - bundle exec hatchet ci:setup
 
 script:
-  - docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/Dockerfile .
-  - docker run --rm -e "STACK=heroku-16" heroku-python-build-heroku-16 bash $TESTFOLDER
-  - docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/Dockerfile.heroku-18 .
-  - docker run --rm -e "STACK=heroku-18" heroku-python-build-heroku-18 bash $TESTFOLDER
+  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
+  - docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER
+  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
+  - docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER
+  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
+  - docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER
 
 jobs:
   include:

Makefile

@@ -8,6 +8,11 @@ check:
 	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 
+test-cedar-14:
+	@echo "Running tests in docker (cedar-14)..."
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar-14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@echo ""
+
 test-heroku-16:
 	@echo "Running tests in docker (heroku-16)..."
 	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
@@ -21,7 +26,7 @@ test-heroku-18:
 buildenv-heroku-16:
 	@echo "Creating build environment (heroku-16)..."
 	@echo
-	@docker build --pull -t python-buildenv-heroku-16 .
+	@docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 .
 	@echo
 	@echo "Usage..."
 	@echo
@@ -34,7 +39,7 @@ buildenv-heroku-16:
 buildenv-heroku-18:
 	@echo "Creating build environment (heroku-18)..."
 	@echo
-	@docker build --pull -f Dockerfile.heroku-18 -t python-buildenv-heroku-18 .
+	@docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 .
 	@echo
 	@echo "Usage..."
 	@echo

builds/README.md

@@ -1,47 +1,57 @@
 # Python Buildpack Binaries
 
-For Cedar-14 stack
-------------------
+## Building the Docker Images
 
-To get started with it, create an app on Heroku inside a clone of this repository, and set your S3 config vars:
+**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack:
 
-    $ heroku create --buildpack https://github.com/heroku/heroku-buildpack-python#not-heroku
-    $ heroku config:set WORKSPACE_DIR=builds
-    $ heroku config:set AWS_ACCESS_KEY_ID=<your_aws_key>
-    $ heroku config:set AWS_SECRET_ACCESS_KEY=<your_aws_secret>
-    $ heroku config:set S3_BUCKET=<your_s3_bucket_name>
+    $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
+    $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
+    $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
 
+## Using the Image
 
-Then, shell into an instance and run a build by giving the name of the formula inside `builds`:
+You can e.g. `bash` into each of the images you built using their tag:
 
-    $ heroku run bash
-    Running `bash` attached to terminal... up, run.6880
-    ~ $ bob build runtimes/python-2.7.6
+    docker run --rm -ti heroku-python-build-cedar-14 bash
+    docker run --rm -ti heroku-python-build-heroku-16 bash
+    docker run --rm -ti heroku-python-build-heroku-18 bash
 
-    Fetching dependencies... found 2:
-      - libraries/sqlite
+You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`:
 
-    Building formula runtimes/python-2.7.6:
-        === Building Python 2.7.6
-        Fetching Python v2.7.6 source...
-        Compiling...
+    docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15
 
-If this works, run `bob deploy` instead of `bob build` to have the result uploaded to S3 for you.
+In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details.
 
-To speed things up drastically, it'll usually be a good idea to `heroku run bash --size PX` instead.
+## Configuration
 
-For Heroku-16 stack
--------------------
+File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`.
 
-1. Ensure GNU Make and Docker are installed.
-2. From the root of the buildpack repository, run: `make buildenv-heroku-16`
-3. Follow the instructions displayed!
+Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials.
 
-For Heroku-18 stack
--------------------
+### Passing AWS credentials to the container
 
-1. Ensure GNU Make and Docker are installed.
-2. From the root of the buildpack repository, run: `make buildenv-heroku-18`
-3. Follow the instructions displayed!
+If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file.
 
-Enjoy :)
+#### Passing credentials explicitly
+
+    docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash
+
+#### Passing credentials through  the environment
+
+The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in:
+
+    AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+
+or
+
+    export AWS_ACCESS_KEY_ID=...
+    export AWS_SECRET_ACCESS_KEY=...
+    docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+
+#### Passing credentials through a separate env file
+
+This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values.
+
+For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`.
+
+    docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash

builds/cedar-14.Dockerfile

@@ -0,0 +1,15 @@
+FROM heroku/cedar:14
+
+WORKDIR /app
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="lang-python" \
+    S3_PREFIX="cedar-14/" \
+    DEBIAN_FRONTEND=noninteractive \
+    STACK="cedar-14"
+
+RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+RUN pip install -r /app/requirements.txt
+
+COPY . /app

builds/dockerenv.default

@@ -0,0 +1,5 @@
+AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY
+S3_BUCKET
+S3_PREFIX
+S3_REGION

Dockerfile renamed to builds/heroku-16.Dockerfile

@@ -4,7 +4,8 @@ WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
     S3_BUCKET="lang-python" \
     S3_PREFIX="heroku-16/" \
-    DEBIAN_FRONTEND=noninteractive
+    DEBIAN_FRONTEND=noninteractive \
+    STACK="heroku-16"
 
 RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
 

Dockerfile.heroku-18 renamed to builds/heroku-18.Dockerfile

@@ -4,7 +4,8 @@ WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
     S3_BUCKET="lang-python" \
     S3_PREFIX="heroku-18/" \
-    DEBIAN_FRONTEND=noninteractive
+    DEBIAN_FRONTEND=noninteractive \
+    STACK="heroku-18"
 
 RUN apt-get update && apt-get install --no-install-recommends -y python-pip-whl=9.0.1-2 python-pip=9.0.1-2 python-setuptools python-wheel && rm -rf /var/lib/apt/lists/*
 

builds/runtimes/pypy

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# fail hard
+set -o pipefail
+# fail harder
+set -eu
+
+OUT_PREFIX=$1
+
+dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
+dep_name=$(basename $BASH_SOURCE) # this is us
+dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name
+dep_package=${dep_name}2-v${dep_version} # it's always "pypy2-…"
+dep_dirname=${dep_package}-linux64
+dep_archive_name=${dep_dirname}.tar.bz2
+dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+
+echo "Building PyPy…"
+
+curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+
+ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python"

builds/runtimes/pypy-5.3.1

@@ -2,13 +2,4 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-# NOTICE: This formula only works for the cedar-14 stack, not cedar.
-
-OUT_PREFIX=$1
-
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy2-v5.3.1-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python
+source $(dirname $0)/pypy