Skip to content

Commit 9b1a69a

Browse files
authored
Vendor buildpack-stdlib instead of fetching from S3 (#1100)
Since fetching buildpack-stdlib from S3 has a number of disadvantages: - it's not possible to grep the repo when trying to work out what something from the stdlib is doing - shellcheck can't fully scan the code, since it similarly doesn't have the source - another compile-time HTTP request that can fail due to transient network issues and so reduce reliability - dependency on the security/release-process of an additional bucket Since the stdlib is small, doesn't often change, and is not a binary, it's a great fit for just vendoring in the repo. The `BIN_DIR` calculation added to `bin/utils` is based on the approach mentioned here: https://www.ostricher.com/2014/10/the-right-way-to-get-the-directory-of-a-bash-script/ ...rather than copying the version in `bin/compile`, since the latter uses `$0` so doesn't work with sourced scripts (such as `bin/utils`). Closes W-8094463.
1 parent 58dd638 commit 9b1a69a

File tree

4 files changed

+204
-9
lines changed

4 files changed

+204
-9
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
## v183 (2020-10-12)
77

88
- Add support for Heroku-20 (#968).
9+
- Vendor buildpack-stdlib instead of fetching from S3 (#1100).
910
- Fix metric names for metrics emitted within `sub_env` (#1099).
1011

1112
## v182 (2020-10-06)

bin/compile

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,7 @@
1515
# Fail fast and fail hard.
1616
set -eo pipefail
1717

18-
# Boostrap the Buildpack Standard Library.
19-
# Disable unused env var warning since shellcheck doesn't know about the stdlib.
20-
# shellcheck disable=2034
18+
# Used by buildpack-stdlib's metrics features.
2119
export BPLOG_PREFIX="buildpack.python"
2220
export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
2321

bin/utils

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@
22
shopt -s extglob
33
shopt -s nullglob
44

5-
# The standard library.
6-
if [[ ! -f /tmp/stdlib.sh ]]; then
7-
curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v8/stdlib.sh > /tmp/stdlib.sh
8-
fi
9-
# shellcheck source=/dev/null
10-
source /tmp/stdlib.sh
5+
# This is necessary since this script is sometimes sourced from
6+
# subshells that don't have the variables from bin/compile.
7+
# Remove this once we no longer wrap all the things in `sub_env`.
8+
BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
9+
ROOT_DIR=$(dirname "${BIN_DIR}")
10+
# shellcheck source=vendor/buildpack-stdlib_v8.sh
11+
source "${ROOT_DIR}/vendor/buildpack-stdlib_v8.sh"
1112

1213
if [ "$(uname)" == Darwin ]; then
1314
sed() { command sed -l "$@"; }

vendor/buildpack-stdlib_v8.sh

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
#!/usr/bin/env bash
2+
3+
# From:
4+
# https://raw.githubusercontent.com/heroku/buildpack-stdlib/v8/stdlib.sh
5+
6+
# Buildpack defaults
7+
# ---------------
8+
9+
export BUILDPACK_LOG_FILE="${BUILDPACK_LOG_FILE:-/dev/null}"
10+
11+
# Standard Output
12+
# ---------------
13+
14+
# Buildpack Steps.
15+
puts_step() {
16+
if [[ "$*" == "-" ]]; then
17+
read -r output
18+
else
19+
output=$*
20+
fi
21+
echo -e "\\e[1m\\e[36m=== $output\\e[0m"
22+
unset output
23+
}
24+
25+
# Buildpack Error.
26+
puts_error() {
27+
if [[ "$*" == "-" ]]; then
28+
read -r output
29+
else
30+
output=$*
31+
fi
32+
echo -e "\\e[1m\\e[31m=!= $output\\e[0m"
33+
}
34+
35+
# Buildpack Warning.
36+
puts_warn() {
37+
if [[ "$*" == "-" ]]; then
38+
read -r output
39+
else
40+
output=$*
41+
fi
42+
echo -e "\\e[1m\\e[33m=!= $output\\e[0m"
43+
}
44+
45+
# Is verbose set?
46+
is_verbose() {
47+
if [[ -n $BUILDPACK_VERBOSE ]]; then
48+
return 0
49+
else
50+
return 1
51+
fi
52+
}
53+
54+
# Buildpack Verbose.
55+
puts_verbose() {
56+
if is_verbose; then
57+
if [[ "$*" == "-" ]]; then
58+
read -r output
59+
else
60+
output=$*
61+
fi
62+
echo "$output"
63+
unset output
64+
fi
65+
}
66+
67+
# Buildpack Utilities
68+
# -------------------
69+
70+
# Usage: $ set-env key value
71+
# NOTICE: Expects PROFILE_PATH & EXPORT_PATH to be set!
72+
set_env() {
73+
# TODO: automatically create profile path directory if it doesn't exist.
74+
echo "export $1=$2" >> "$PROFILE_PATH"
75+
echo "export $1=$2" >> "$EXPORT_PATH"
76+
}
77+
78+
# Usage: $ set-default-env key value
79+
# NOTICE: Expects PROFILE_PATH & EXPORT_PATH to be set!
80+
set_default_env() {
81+
echo "export $1=\${$1:-$2}" >> "$PROFILE_PATH"
82+
echo "export $1=\${$1:-$2}" >> "$EXPORT_PATH"
83+
}
84+
85+
# Usage: $ un-set-env key
86+
# NOTICE: Expects PROFILE_PATH to be set!
87+
un_set_env() {
88+
echo "unset $1" >> "$PROFILE_PATH"
89+
}
90+
91+
# Usage: $ _env-blacklist pattern
92+
# Outputs a regex of default blacklist env vars.
93+
_env_blacklist() {
94+
local regex=${1:-''}
95+
if [ -n "$regex" ]; then
96+
regex="|$regex"
97+
fi
98+
echo "^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH$regex)$"
99+
}
100+
101+
# Usage: $ export-env ENV_DIR WHITELIST BLACKLIST
102+
# Exports the environment variables defined in the given directory.
103+
export_env() {
104+
local env_dir=${1:-$ENV_DIR}
105+
local whitelist=${2:-''}
106+
local blacklist
107+
blacklist="$(_env_blacklist "$3")"
108+
if [ -d "$env_dir" ]; then
109+
# Environment variable names won't contain characters affected by:
110+
# shellcheck disable=SC2045
111+
for e in $(ls "$env_dir"); do
112+
echo "$e" | grep -E "$whitelist" | grep -qvE "$blacklist" &&
113+
export "$e=$(cat "$env_dir/$e")"
114+
:
115+
done
116+
fi
117+
}
118+
119+
# Usage: $ sub-env command
120+
# Runs a subshell of specified command with user-provided config.
121+
# NOTICE: Expects ENV_DIR to be set. WHITELIST & BLACKLIST are optional.
122+
# Examples:
123+
# WHITELIST=${2:-''}
124+
# BLACKLIST=${3:-'^(GIT_DIR|PYTHONHOME|LD_LIBRARY_PATH|LIBRARY_PATH|PATH)$'}
125+
sub_env() {
126+
(
127+
# TODO: Fix https://github.com/heroku/buildpack-stdlib/issues/37
128+
# shellcheck disable=SC2153
129+
export_env "$ENV_DIR" "$WHITELIST" "$BLACKLIST"
130+
131+
"$@"
132+
)
133+
}
134+
135+
# Logging
136+
# -------
137+
138+
# Notice: These functions expect BPLOG_PREFIX and BUILDPACK_LOG_FILE to be defined (BUILDPACK_LOG_FILE can point to /dev/null if not provided by the buildpack).
139+
# Example: BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}; BPLOG_PREFIX="buildpack.go"
140+
141+
# Returns now, in milleseconds. Useful for logging.
142+
# Example: $ let start=$(nowms); sleep 30; mtime "glide.install.time" "${start}"
143+
nowms() {
144+
date +%s%3N
145+
}
146+
147+
# Log arbitrary data to the logfile (e.g. a packaging file).
148+
# Usage: $ bplog "$(<${vendorJSON})
149+
bplog() {
150+
echo -n "${@}" | awk 'BEGIN {printf "msg=\""; f="%s"} {gsub(/"/, "\\\"", $0); printf f, $0} {if (NR == 1) f="\\n%s" } END { print "\"" }' >> "${BUILDPACK_LOG_FILE}"
151+
}
152+
153+
# Measures time elapsed for a specific build step.
154+
# Usage: $ let start=$(nowms); mtime "glide.install.time" "${start}"
155+
# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#distributions-measure
156+
mtime() {
157+
local key="${BPLOG_PREFIX}.${1}"
158+
local start="${2}"
159+
local end="${3:-$(nowms)}"
160+
echo "${key} ${start} ${end}" | awk '{ printf "measure#%s=%.3f\n", $1, ($3 - $2)/1000 }' >> "${BUILDPACK_LOG_FILE}"
161+
}
162+
163+
# Logs a count for a specific built step.
164+
# Usage: $ mcount "tool.govendor"
165+
# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#counting-count
166+
mcount() {
167+
local k="${BPLOG_PREFIX}.${1}"
168+
local v="${2:-1}"
169+
echo "count#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
170+
}
171+
172+
# Logs a measure for a specific build step.
173+
# Usage: $ mmeasure "tool.installed_dependencies" 42
174+
# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#distributions-measure
175+
mmeasure() {
176+
local k="${BPLOG_PREFIX}.${1}"
177+
local v="${2}"
178+
echo "measure#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
179+
}
180+
181+
# Logs a unuique measurement build step.
182+
# Usage: $ munique "versions.count" 2.7.13
183+
# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#uniques-unique
184+
munique() {
185+
local k="${BPLOG_PREFIX}.${1}"
186+
local v="${2}"
187+
echo "unique#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
188+
}
189+
190+
# Measures when an exit path to the buildpack is reached, given a name, then exits 1.
191+
# Usage: $ mcount-exi "binExists"
192+
mcount_exit() {
193+
mcount "error.${1}"
194+
exit 1
195+
}

0 commit comments

Comments
 (0)