Skip to content

Commit f455189

Browse files
h3xcath3xcat
authored
Resolve slow key retrieval when some GPG key servers are not reachable (#1016)
* Reduce timeout to GPG key servers for Python feature * Dynamically find out reachable GPG servers * Apply fix to other features * Fix build issues * Fix shellcheck SC2068 by quoting array expansion * Bump up patch version for affected features * improve readability for keyservers_curl_map
1 parent 09d5632 commit f455189

16 files changed

+354
-50
lines changed

‎src/git-lfs/devcontainer-feature.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"id": "git-lfs",
3-
"version": "1.2.1",
3+
"version": "1.2.2",
44
"name": "Git Large File Support (LFS)",
55
"documentationURL": "https://github.com/devcontainers/features/tree/main/src/git-lfs",
66
"description": "Installs Git Large File Support (Git LFS) along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like git and curl.",

‎src/git-lfs/install.sh

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,6 @@ GIT_LFS_ARCHIVE_GPG_KEY_URI="https://packagecloud.io/github/git-lfs/gpgkey"
1515
GIT_LFS_ARCHIVE_ARCHITECTURES="amd64 arm64"
1616
GIT_LFS_ARCHIVE_VERSION_CODENAMES="stretch buster bullseye bionic focal jammy"
1717
GIT_LFS_CHECKSUM_GPG_KEYS="0x88ace9b29196305ba9947552f1ba225c0223b187 0x86cd3297749375bcf8206715f54fe648088335a9 0xaa3b3450295830d2de6db90caba67be5a5795889"
18-
GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com
19-
keyserver hkp://keyserver.ubuntu.com:80
20-
keyserver hkps://keys.openpgp.org
21-
keyserver hkp://keyserver.pgp.com"
2218

2319
set -e
2420

@@ -64,15 +60,52 @@ find_version_from_git_tags() {
6460
echo "${variable_name}=${!variable_name}"
6561
}
6662

63+
# Get the list of GPG key servers that are reachable
64+
get_gpg_key_servers() {
65+
declare -A keyservers_curl_map=(
66+
["hkp://keyserver.ubuntu.com"]="http://keyserver.ubuntu.com:11371"
67+
["hkp://keyserver.ubuntu.com:80"]="http://keyserver.ubuntu.com"
68+
["hkps://keys.openpgp.org"]="https://keys.openpgp.org"
69+
["hkp://keyserver.pgp.com"]="http://keyserver.pgp.com:11371"
70+
)
71+
72+
local curl_args=""
73+
local keyserver_reachable=false # Flag to indicate if any keyserver is reachable
74+
75+
if [ ! -z "${KEYSERVER_PROXY}" ]; then
76+
curl_args="--proxy ${KEYSERVER_PROXY}"
77+
fi
78+
79+
for keyserver in "${!keyservers_curl_map[@]}"; do
80+
local keyserver_curl_url="${keyservers_curl_map[${keyserver}]}"
81+
if curl -s ${curl_args} --max-time 5 ${keyserver_curl_url} > /dev/null; then
82+
echo "keyserver ${keyserver}"
83+
keyserver_reachable=true
84+
else
85+
echo "(*) Keyserver ${keyserver} is not reachable." >&2
86+
fi
87+
done
88+
89+
if ! $keyserver_reachable; then
90+
echo "(!) No keyserver is reachable." >&2
91+
exit 1
92+
fi
93+
}
94+
6795
# Import the specified key in a variable name passed in as
6896
receive_gpg_keys() {
6997
local keys=${!1}
7098

99+
# Install curl
100+
if ! type curl > /dev/null 2>&1; then
101+
check_packages curl
102+
fi
103+
71104
# Use a temporary location for gpg keys to avoid polluting image
72105
export GNUPGHOME="/tmp/tmp-gnupg"
73106
mkdir -p ${GNUPGHOME}
74107
chmod 700 ${GNUPGHOME}
75-
echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf
108+
echo -e "disable-ipv6\n$(get_gpg_key_servers)" > ${GNUPGHOME}/dirmngr.conf
76109
# GPG key download sometimes fails for some reason and retrying fixes it.
77110
local retry_count=0
78111
local gpg_ok="false"

‎src/git/devcontainer-feature.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"id": "git",
3-
"version": "1.3.0",
3+
"version": "1.3.1",
44
"name": "Git (from source)",
55
"documentationURL": "https://github.com/devcontainers/features/tree/main/src/git",
66
"description": "Install an up-to-date version of Git, built from source as needed. Useful for when you want the latest and greatest features. Auto-detects latest stable version and installs needed dependencies.",

‎src/git/install.sh

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,6 @@ GIT_VERSION=${VERSION} # 'system' checks the base image first, else installs 'la
1111
USE_PPA_IF_AVAILABLE=${PPA}
1212

1313
GIT_CORE_PPA_ARCHIVE_GPG_KEY=E1DD270288B4E6030699E45FA1715D88E1DF1F24
14-
GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com
15-
keyserver hkp://keyserver.ubuntu.com:80
16-
keyserver hkps://keys.openpgp.org
17-
keyserver hkp://keyserver.pgp.com"
1814

1915
if [ "$(id -u)" -ne 0 ]; then
2016
echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.'
@@ -68,6 +64,38 @@ clean_up() {
6864
}
6965
clean_up
7066

67+
# Get the list of GPG key servers that are reachable
68+
get_gpg_key_servers() {
69+
declare -A keyservers_curl_map=(
70+
["hkp://keyserver.ubuntu.com"]="http://keyserver.ubuntu.com:11371"
71+
["hkp://keyserver.ubuntu.com:80"]="http://keyserver.ubuntu.com"
72+
["hkps://keys.openpgp.org"]="https://keys.openpgp.org"
73+
["hkp://keyserver.pgp.com"]="http://keyserver.pgp.com:11371"
74+
)
75+
76+
local curl_args=""
77+
local keyserver_reachable=false # Flag to indicate if any keyserver is reachable
78+
79+
if [ ! -z "${KEYSERVER_PROXY}" ]; then
80+
curl_args="--proxy ${KEYSERVER_PROXY}"
81+
fi
82+
83+
for keyserver in "${!keyservers_curl_map[@]}"; do
84+
local keyserver_curl_url="${keyservers_curl_map[${keyserver}]}"
85+
if curl -s ${curl_args} --max-time 5 ${keyserver_curl_url} > /dev/null; then
86+
echo "keyserver ${keyserver}"
87+
keyserver_reachable=true
88+
else
89+
echo "(*) Keyserver ${keyserver} is not reachable." >&2
90+
fi
91+
done
92+
93+
if ! $keyserver_reachable; then
94+
echo "(!) No keyserver is reachable." >&2
95+
exit 1
96+
fi
97+
}
98+
7199
# Import the specified key in a variable name passed in as
72100
receive_gpg_keys() {
73101
local keys=${!1}
@@ -77,11 +105,16 @@ receive_gpg_keys() {
77105
keyring_args="--no-default-keyring --keyring $2"
78106
fi
79107

108+
# Install curl
109+
if ! type curl > /dev/null 2>&1; then
110+
check_packages curl
111+
fi
112+
80113
# Use a temporary location for gpg keys to avoid polluting image
81114
export GNUPGHOME="/tmp/tmp-gnupg"
82115
mkdir -p ${GNUPGHOME}
83116
chmod 700 ${GNUPGHOME}
84-
echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf
117+
echo -e "disable-ipv6\n$(get_gpg_key_servers)" > ${GNUPGHOME}/dirmngr.conf
85118
# GPG key download sometimes fails for some reason and retrying fixes it.
86119
local retry_count=0
87120
local gpg_ok="false"

‎src/github-cli/devcontainer-feature.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"id": "github-cli",
3-
"version": "1.0.12",
3+
"version": "1.0.13",
44
"name": "GitHub CLI",
55
"documentationURL": "https://github.com/devcontainers/features/tree/main/src/github-cli",
66
"description": "Installs the GitHub CLI. Auto-detects latest version and installs needed dependencies.",

‎src/github-cli/install.sh

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,6 @@ CLI_VERSION=${VERSION:-"latest"}
1111
INSTALL_DIRECTLY_FROM_GITHUB_RELEASE=${INSTALLDIRECTLYFROMGITHUBRELEASE:-"true"}
1212

1313
GITHUB_CLI_ARCHIVE_GPG_KEY=23F3D4EA75716059
14-
GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com
15-
keyserver hkp://keyserver.ubuntu.com:80
16-
keyserver hkps://keys.openpgp.org
17-
keyserver hkp://keyserver.pgp.com"
1814

1915
set -e
2016

@@ -26,6 +22,37 @@ if [ "$(id -u)" -ne 0 ]; then
2622
exit 1
2723
fi
2824

25+
# Get the list of GPG key servers that are reachable
26+
get_gpg_key_servers() {
27+
declare -A keyservers_curl_map=(
28+
["hkp://keyserver.ubuntu.com"]="http://keyserver.ubuntu.com:11371"
29+
["hkp://keyserver.ubuntu.com:80"]="http://keyserver.ubuntu.com"
30+
["hkps://keys.openpgp.org"]="https://keys.openpgp.org"
31+
["hkp://keyserver.pgp.com"]="http://keyserver.pgp.com:11371"
32+
)
33+
34+
local curl_args=""
35+
local keyserver_reachable=false # Flag to indicate if any keyserver is reachable
36+
37+
if [ ! -z "${KEYSERVER_PROXY}" ]; then
38+
curl_args="--proxy ${KEYSERVER_PROXY}"
39+
fi
40+
41+
for keyserver in "${!keyservers_curl_map[@]}"; do
42+
local keyserver_curl_url="${keyservers_curl_map[${keyserver}]}"
43+
if curl -s ${curl_args} --max-time 5 ${keyserver_curl_url} > /dev/null; then
44+
echo "keyserver ${keyserver}"
45+
keyserver_reachable=true
46+
else
47+
echo "(*) Keyserver ${keyserver} is not reachable." >&2
48+
fi
49+
done
50+
51+
if ! $keyserver_reachable; then
52+
echo "(!) No keyserver is reachable." >&2
53+
exit 1
54+
fi
55+
}
2956

3057
# Import the specified key in a variable name passed in as
3158
receive_gpg_keys() {
@@ -35,11 +62,16 @@ receive_gpg_keys() {
3562
keyring_args="--no-default-keyring --keyring $2"
3663
fi
3764

65+
# Install curl
66+
if ! type curl > /dev/null 2>&1; then
67+
check_packages curl
68+
fi
69+
3870
# Use a temporary location for gpg keys to avoid polluting image
3971
export GNUPGHOME="/tmp/tmp-gnupg"
4072
mkdir -p ${GNUPGHOME}
4173
chmod 700 ${GNUPGHOME}
42-
echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf
74+
echo -e "disable-ipv6\n$(get_gpg_key_servers)" > ${GNUPGHOME}/dirmngr.conf
4375
# GPG key download sometimes fails for some reason and retrying fixes it.
4476
local retry_count=0
4577
local gpg_ok="false"

‎src/kubectl-helm-minikube/devcontainer-feature.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"id": "kubectl-helm-minikube",
3-
"version": "1.1.9",
3+
"version": "1.1.10",
44
"name": "Kubectl, Helm, and Minikube",
55
"documentationURL": "https://github.com/devcontainers/features/tree/main/src/kubectl-helm-minikube",
66
"description": "Installs latest version of kubectl, Helm, and optionally minikube. Auto-detects latest versions and installs needed dependencies.",

‎src/kubectl-helm-minikube/install.sh

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,6 @@ MINIKUBE_SHA256="${MINIKUBE_SHA256:-"automatic"}"
2222
USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}"
2323

2424
HELM_GPG_KEYS_URI="https://raw.githubusercontent.com/helm/helm/main/KEYS"
25-
GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com
26-
keyserver hkp://keyserver.ubuntu.com:80
27-
keyserver hkps://keys.openpgp.org
28-
keyserver hkp://keyserver.pgp.com"
2925

3026
if [ "$(id -u)" -ne 0 ]; then
3127
echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.'
@@ -234,6 +230,38 @@ get_helm() {
234230
curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc"
235231
}
236232

233+
# Get the list of GPG key servers that are reachable
234+
get_gpg_key_servers() {
235+
declare -A keyservers_curl_map=(
236+
["hkp://keyserver.ubuntu.com"]="http://keyserver.ubuntu.com:11371"
237+
["hkp://keyserver.ubuntu.com:80"]="http://keyserver.ubuntu.com"
238+
["hkps://keys.openpgp.org"]="https://keys.openpgp.org"
239+
["hkp://keyserver.pgp.com"]="http://keyserver.pgp.com:11371"
240+
)
241+
242+
local curl_args=""
243+
local keyserver_reachable=false # Flag to indicate if any keyserver is reachable
244+
245+
if [ ! -z "${KEYSERVER_PROXY}" ]; then
246+
curl_args="--proxy ${KEYSERVER_PROXY}"
247+
fi
248+
249+
for keyserver in "${!keyservers_curl_map[@]}"; do
250+
local keyserver_curl_url="${keyservers_curl_map[${keyserver}]}"
251+
if curl -s ${curl_args} --max-time 5 ${keyserver_curl_url} > /dev/null; then
252+
echo "keyserver ${keyserver}"
253+
keyserver_reachable=true
254+
else
255+
echo "(*) Keyserver ${keyserver} is not reachable." >&2
256+
fi
257+
done
258+
259+
if ! $keyserver_reachable; then
260+
echo "(!) No keyserver is reachable." >&2
261+
exit 1
262+
fi
263+
}
264+
237265
if [ ${HELM_VERSION} != "none" ]; then
238266
# Install Helm, verify signature and checksum
239267
echo "Downloading Helm..."
@@ -255,7 +283,7 @@ if [ ${HELM_VERSION} != "none" ]; then
255283
mkdir -p "${GNUPGHOME}"
256284
chmod 700 ${GNUPGHOME}
257285
curl -sSL "${HELM_GPG_KEYS_URI}" -o /tmp/helm/KEYS
258-
echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf
286+
echo -e "disable-ipv6\n$(get_gpg_key_servers)" > ${GNUPGHOME}/dirmngr.conf
259287
gpg -q --import "/tmp/helm/KEYS"
260288
if ! gpg --verify "${tmp_helm_filename}.asc" > ${GNUPGHOME}/verify.log 2>&1; then
261289
echo "Verification failed!"

‎src/python/devcontainer-feature.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"id": "python",
3-
"version": "1.6.2",
3+
"version": "1.6.3",
44
"name": "Python",
55
"documentationURL": "https://github.com/devcontainers/features/tree/main/src/python",
66
"description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.",

‎src/python/install.sh

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,7 @@ ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}"
3131
# Comma-separated list of additional tools to be installed via pipx.
3232
IFS="," read -r -a DEFAULT_UTILS <<< "${TOOLSTOINSTALL:-flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest}"
3333

34-
3534
PYTHON_SOURCE_GPG_KEYS="64E628F8D684696D B26995E310250568 2D347EA6AA65421D FB9921286F5E1540 3A5CA953F73C700D 04C367C218ADD4FF 0EDDC5F26A45C816 6AF053F07D9DC8D2 C9BE28DEE6DF025C 126EB563A74B06BF D9866941EA5BBD71 ED9D77D5 A821E680E5FA6305"
36-
GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com
37-
keyserver hkp://keyserver.ubuntu.com:80
38-
keyserver hkps://keys.openpgp.org
39-
keyserver hkp://keyserver.pgp.com"
4035

4136
KEYSERVER_PROXY="${HTTPPROXY:-"${HTTP_PROXY:-""}"}"
4237

@@ -130,6 +125,38 @@ updaterc() {
130125
fi
131126
}
132127

128+
# Get the list of GPG key servers that are reachable
129+
get_gpg_key_servers() {
130+
declare -A keyservers_curl_map=(
131+
["hkp://keyserver.ubuntu.com"]="http://keyserver.ubuntu.com:11371"
132+
["hkp://keyserver.ubuntu.com:80"]="http://keyserver.ubuntu.com"
133+
["hkps://keys.openpgp.org"]="https://keys.openpgp.org"
134+
["hkp://keyserver.pgp.com"]="http://keyserver.pgp.com:11371"
135+
)
136+
137+
local curl_args=""
138+
local keyserver_reachable=false # Flag to indicate if any keyserver is reachable
139+
140+
if [ ! -z "${KEYSERVER_PROXY}" ]; then
141+
curl_args="--proxy ${KEYSERVER_PROXY}"
142+
fi
143+
144+
for keyserver in "${!keyservers_curl_map[@]}"; do
145+
local keyserver_curl_url="${keyservers_curl_map[${keyserver}]}"
146+
if curl -s ${curl_args} --max-time 5 ${keyserver_curl_url} > /dev/null; then
147+
echo "keyserver ${keyserver}"
148+
keyserver_reachable=true
149+
else
150+
echo "(*) Keyserver ${keyserver} is not reachable." >&2
151+
fi
152+
done
153+
154+
if ! $keyserver_reachable; then
155+
echo "(!) No keyserver is reachable." >&2
156+
exit 1
157+
fi
158+
}
159+
133160
# Import the specified key in a variable name passed in as
134161
receive_gpg_keys() {
135162
local keys=${!1}
@@ -143,11 +170,16 @@ receive_gpg_keys() {
143170
keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}"
144171
fi
145172

173+
# Install curl
174+
if ! type curl > /dev/null 2>&1; then
175+
check_packages curl
176+
fi
177+
146178
# Use a temporary location for gpg keys to avoid polluting image
147179
export GNUPGHOME="/tmp/tmp-gnupg"
148180
mkdir -p ${GNUPGHOME}
149181
chmod 700 ${GNUPGHOME}
150-
echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf
182+
echo -e "disable-ipv6\n$(get_gpg_key_servers)" > ${GNUPGHOME}/dirmngr.conf
151183
# GPG key download sometimes fails for some reason and retrying fixes it.
152184
local retry_count=0
153185
local gpg_ok="false"
@@ -182,6 +214,11 @@ receive_gpg_keys_centos7() {
182214
keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}"
183215
fi
184216

217+
# Install curl
218+
if ! type curl > /dev/null 2>&1; then
219+
check_packages curl
220+
fi
221+
185222
# Use a temporary location for gpg keys to avoid polluting image
186223
export GNUPGHOME="/tmp/tmp-gnupg"
187224
mkdir -p ${GNUPGHOME}
@@ -193,7 +230,7 @@ receive_gpg_keys_centos7() {
193230
set +e
194231
echo "(*) Downloading GPG keys..."
195232
until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do
196-
for keyserver in $(echo "${GPG_KEY_SERVERS}" | sed 's/keyserver //'); do
233+
for keyserver in $(echo "$(get_gpg_key_servers)" | sed 's/keyserver //'); do
197234
( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys --keyserver=${keyserver} ) 2>&1
198235
downloaded_keys=$(gpg --list-keys | grep ^pub | wc -l)
199236
if [[ ${num_keys} = ${downloaded_keys} ]]; then

0 commit comments

Comments
 (0)