lots of comments

Signed-off-by: Kenneth Reitz <[email protected]>

Author: kennethreitz

bin/compile

@@ -15,70 +15,96 @@
 # Fail fast and fail hard.
 set -eo pipefail
 
-# Standard Library.
+# Boostrap the Buildpack Standard Library.
 export BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
 
 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
 
-# Prepend proper path for virtualenv hackery. This will be deprecated soon.
+# Prepend proper path for old-school virtualenv hackery.
+# This may not be neccessary.
 export PATH=:/usr/local/bin:$PATH
 
-# Paths.
+# Setup Path variables, for later use in the Buildpack.
 BIN_DIR=$(cd "$(dirname "$0")"; pwd) # absolute path
 ROOT_DIR=$(dirname "$BIN_DIR")
 BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
 
+# Export Path variables, for use in sub-scripts.
 export BUILD_DIR CACHE_DIR ENV_DIR
 
+# Set the Buildpack's internet target for downloading Python distributions.
+# The user can provide BUILDPACK_VENDOR_URL to specify a custom target.
+# Note: this is designed for non-Heroku use, as it does not use the user-provided
+# environment variable mechanism (the ENV_DIR).
 VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK"
 if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then
     VENDOR_URL="$BUILDPACK_VENDOR_URL"
 fi
 export VENDOR_URL
 
-# Python defaults
-DEFAULT_PYTHON_VERSION="python-3.6.5"
-LATEST_3="python-3.6.5"
+# Which versions of Python are we using?
+# These variables are used to specify which versions of Python to install by default,
+# as well as prompt the user to upgrade if they are using an un–supported version.
+# Note: When 3.7 lands, I recommend switing to LATEST_36 and LATEST_37.
+DEFAULT_PYTHON_VERSION="python-3.6.4"
+LATEST_3="python-3.6.4"
 LATEST_2="python-2.7.15"
 
+# Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
+# If pip doesn't match this version (the version we install), run the installer.
 PIP_UPDATE="9.0.2"
 
 export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_3
 
-# Common Problem Warnings
+# Common Problem Warnings:
+# This section creates a temporary file in which to stick the output of `pip install`.
+# The `warnings` subscript then greps through this for common problems and guides
+# the user towards resolution of known issues.
 WARNINGS_LOG=$(mktemp)
 export WARNINGS_LOG
 export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
 
-# Setup vendored tools and pip-pop (pip-diff)
+# The buildpack ships with a few executable tools (e.g. pip-grep, etc).
+# This installs them into the path, so we can execute them directly.
 export PATH=$PATH:$ROOT_DIR/vendor/:$ROOT_DIR/vendor/pip-pop
 
-# Support Anvil Build_IDs
+# Set environment variables if they weren't set by the platform.
+# Note: this is legacy, for a deprecated build system known as Anvil.
+# This can likely be removed, with caution.
 [ ! "$SLUG_ID" ] && SLUG_ID="defaultslug"
 [ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID
 [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK
 
-# Sanitizing environment variables.
+# Sanitize externally-provided environment variables:
+# The following environment variables are either problematic or simply unneccessary
+# for the buildpack to have knowledge of, so we unset them, to keep the environment
+# as clean and pristine as possible.
 unset GIT_DIR PYTHONHOME PYTHONPATH
 unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN
 unset CYTOKINE_LOG_FILE GEM_PATH
 
-# Syntax sugar.
+# Import the utils script, which contains helper functions used throughout the buildpack.
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
-# Import collection of warnings.
+# Import the warnings script, which contains the `pip install` user warning mechanisms
+# (mentioned and explained above)
 # shellcheck source=bin/warnings
 source "$BIN_DIR/warnings"
 
-# we need to put a bunch of symlinks in there later
+# Make the directory in which we will create symlinks from the temporary build directory
+# to `/app`.
+# Symlinks are required, since Python is not a portable installation.
+# More on this topic later.
 mkdir -p /app/.heroku
 
-# Set up outputs under new context
+# This buildpack programatically generates (or simply copies) a number of files for
+# buildpack machinery: an export script, and a number of `.profile.d` scripts. This
+# section declares the locations of those files and targets.
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
 EXPORT_PATH="$BIN_DIR/../export"
 GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
@@ -87,43 +113,75 @@ WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh"
 # We'll need to send these statics to other scripts we `source`.
 export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
 
-# Prepend proper environment variables for Python use.
+# Python Environment Variables
+# Set Python-specific environment variables, for running Python within the buildpack.
+# Notes on each variable included.
+
+# PATH is relatively obvious, we need to be able to execute 'python'.
 export PATH=/app/.heroku/python/bin:/app/.heroku/vendor/bin:$PATH
+# Tell Python to not buffer it's stdin/stdout.
 export PYTHONUNBUFFERED=1
+# Set the locale to a well-known and expected standard.
 export LANG=en_US.UTF-8
+# `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known
+# C dependencies (e.g. libmemcached on cedar-14). This section configures Python (GCC, more specifically)
+# and pip to automatically include these paths when building binaries.
 export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
 export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
 export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
 export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
 export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
 
+# The Application Code
+# --------------------
+
 # Switch to the repo's context.
 cd "$BUILD_DIR"
 
-# Prepare the cache.
+# The Cache
+# ---------
+
+# The workflow for the Python Buildpack's cache is as follows:
+#
+# - `~/.heroku/{known-paths}` are copied from the cache into the slug.
+# - The build is executed, modifying `~/.heroku/{known-paths}`.
+# - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache.
+
+# Create the cache directory, if it doesn't exist.
 mkdir -p "$CACHE_DIR"
 
 # Restore old artifacts from the cache.
 mkdir -p .heroku
 
+# The Python installation.
 cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true
+# A plain text file which contains the current stack being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
+# A plain text file which contains the current python version being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
+# Any pre-compiled binaries, provided by the buildpack.
 cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
+# "editable" installations of code repositories, via pip or pipenv.
 if [[ -d "$CACHE_DIR/.heroku/src" ]]; then
   cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true
 fi
 
-# Experimental pre_compile hook.
+# The pre_compile hook. Customers rely on this. Don't remove it.
+# This part of the code is used to allow users to customize their build experience
+# without forking the buildpack by providing a `bin/pre_compile` script, which gets
+# run inline with the buildpack automatically.
+
 # shellcheck source=bin/steps/hooks/pre_compile
 source "$BIN_DIR/steps/hooks/pre_compile"
 
-# Sticky runtimes.
+# Sticky runtimes. If there was a previous build, and it used a given version of Python,
+# continue to use that version of Python in perpituity (warnings will be raised if
+# they are out–of–date).
 if [ -f "$CACHE_DIR/.heroku/python-version" ]; then
   DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
 fi
 
-# Stack fallback for non-declared caches.
+# We didn't always record the stack version. This code is in place because of that.
 if [ -f "$CACHE_DIR/.heroku/python-stack" ]; then
   CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack")
 else
@@ -133,17 +191,26 @@ fi
 export CACHED_PYTHON_STACK
 
 # Pipenv Python version support.
+# Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
+# Convert it to a runtime.txt file.
+
 # shellcheck source=bin/steps/pipenv-python-version
 source "$BIN_DIR/steps/pipenv-python-version"
 
-# If no runtime given, assume default version.
+# If no runtime was provided by the user, assume the default Python runtime version.
 if [ ! -f runtime.txt ]; then
   echo "$DEFAULT_PYTHON_VERSION" > runtime.txt
 fi
 
+# Create the directory for .profile.d, if it doesn't exist.
 mkdir -p "$(dirname "$PROFILE_PATH")"
+# Create the directory for editable source code installation, if it doesn't exist.
 mkdir -p /app/.heroku/src
 
+# On Heroku CI, builds happen in `/app`. Otherwise, on the Heroku platform,
+# they occur in a temp directory. Beacuse Python is not portable, we must create
+# symlinks to emulate that we are operating in `/app` during the build process.
+# This is (hopefully obviously) because apps end up running from `/app` in production.
 if [[ $BUILD_DIR != '/app' ]]; then
     # python expects to reside in /app, so set up symlinks
     # we will not remove these later so subsequent buildpacks can still invoke it
@@ -152,60 +219,79 @@ if [[ $BUILD_DIR != '/app' ]]; then
     # Note: .heroku/src is copied in later.
 fi
 
-# Install Python.
+# Download / Install Python, from pre-build binaries available on Amazon S3.
+# This step also bootstraps pip / setuptools.
 let start=$(nowms)
 # shellcheck source=bin/steps/python
 source "$BIN_DIR/steps/python"
 mtime "python.install.time" "${start}"
 
-# Pipenv support.
+# Install Pipenv dependencies, if a Pipfile was provided.
 # shellcheck source=bin/steps/pipenv
 source "$BIN_DIR/steps/pipenv"
 
 # Uninstall removed dependencies with Pip.
+# The buildpack will automatically remove any declared dependencies (in requirements.txt)
+# that were explicitly removed. This machinery is a bit complex, but it is not complicated.
 let start=$(nowms)
 # shellcheck source=bin/steps/pip-uninstall
 source "$BIN_DIR/steps/pip-uninstall"
 mtime "pip.uninstall.time" "${start}"
 
-
 # If no requirements.txt file given, assume `setup.py develop` is intended.
+# This allows for people to ship a setup.py application to Heroku
+# (which is rare, but I vouch that it should work!)
+
 if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
   echo "-e ." > requirements.txt
 fi
 
 # Fix egg-links.
+# Because we're installing things into a different path than we're running them (temp dir vs app dir),
+# We must re-write all of Python's eggpath links to target the proper directory.
 # shellcheck source=bin/steps/eggpath-fix
 source "$BIN_DIR/steps/eggpath-fix"
 
 # Mercurial support.
+# If a customer appears to be using mercurial for dependency resolution, we install it first.
+# Note: this only applies to pip, not pipenv. This can likely be removed, over time. Measure it first.
 # shellcheck source=bin/steps/mercurial
 source "$BIN_DIR/steps/mercurial"
 
 # Pylibmc support.
+# On cedar-14, libmemcached was not available. The buildpack provides its own version, instead.
 # shellcheck source=bin/steps/pylibmc
 source "$BIN_DIR/steps/pylibmc"
 
-# Support for Geo libraries.
+# Support for Geo libraries. This is deprecated functionality, only functional on cedar-14.
+# It is undocumented.
 # shellcheck source=bin/steps/geo-libs
 sub_env "$BIN_DIR/steps/geo-libs"
 
 # GDAL support.
+# This is part of the Geo support.
 # shellcheck source=bin/steps/gdal
 source "$BIN_DIR/steps/gdal"
 
-# Install dependencies with Pip (where the magic happens).
+# pip install
+# -----------
+
+# Install dependencies with pip (where the magic happens).
 let start=$(nowms)
 # shellcheck source=bin/steps/pip-install
 source "$BIN_DIR/steps/pip-install"
 mtime "pip.install.time" "${start}"
 
 # Support for NLTK corpora.
+# Note: this may only work on Python 2.7. I don't think many customers use this functionality,
+# and it should probably be undocumented.
+# (there's an import error on 3.6 that should hopefully be fixed upstream at some point)
 let start=$(nowms)
 sub_env "$BIN_DIR/steps/nltk"
 mtime "nltk.download.time" "${start}"
 
-# Support for pip install -e.
+# Support for editable installations. Here, we are copying pip–created src directory,
+# and copying it into the proper place (the logical place to do this was early, but it must be done here).
 # In CI, $BUILD_DIR is /app.
 if [[ ! "$BUILD_DIR" == "/app" ]]; then
   rm -fr "$BUILD_DIR/.heroku/src"
@@ -214,24 +300,35 @@ fi
 
 
 # Django collectstatic support.
+# The buildpack automatically runs collectstatic for Django applications.
+# This is the cause for the majority of build failures on the Python platform.
+# These failures are intentional — if collectstatic (which can be tricky, at times) fails,
+# your build fails.
 let start=$(nowms)
 sub_env "$BIN_DIR/steps/collectstatic"
 mtime "collectstatic.time" "${start}"
 
-# Create .profile script for application runtime environment variables.
+
+# Progamatically create .profile.d script for application runtime environment variables.
+
+# Set the PATH to include Python / pip / pipenv / etc.
 set_env PATH "\$HOME/.heroku/python/bin:\$PATH"
+# Tell Python to run in unbuffered mode.
 set_env PYTHONUNBUFFERED true
+# Tell Python where it lives.
 set_env PYTHONHOME "\$HOME/.heroku/python"
-
+# Set variables for C libraries.
 set_env LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LIBRARY_PATH"
 set_env LD_LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LD_LIBRARY_PATH"
-
+# Locale.
 set_default_env LANG en_US.UTF-8
+# The Python hash seed is set to random.
 set_default_env PYTHONHASHSEED random
+# Tell Python to look for Python modules in the /app dir. Don't change this.
 set_default_env PYTHONPATH "\$HOME"
 
-# python expects to be in /app, if at runtime, it is not, set
-# up symlinks... this can occur when the subdir buildpack is used
+# Python expects to be in /app, if at runtime, it is not, set
+# up symlinks… this can occur when the subdir buildpack is used.
 cat <<EOT >> "$PROFILE_PATH"
 if [[ \$HOME != "/app" ]]; then
     mkdir -p /app/.heroku
@@ -244,17 +341,15 @@ EOT
 cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH"
 cp "$ROOT_DIR/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH"
 
-
-# Experimental post_compile hook.
+# Experimental post_compile hook. Don't remove this.
 # shellcheck source=bin/steps/hooks/post_compile
 source "$BIN_DIR/steps/hooks/post_compile"
 
 # Fix egg-links, again.
 # shellcheck source=bin/steps/eggpath-fix2
 source "$BIN_DIR/steps/eggpath-fix2"
 
-# Store new artifacts in cache.
-
+# Store new artifacts in the cache.
 rm -rf "$CACHE_DIR/.heroku/python"
 rm -rf "$CACHE_DIR/.heroku/python-version"
 rm -rf "$CACHE_DIR/.heroku/python-stack"