Skip to content

Commit 3dee250

Browse files
bradjcbradjc
authored
Merge pull request #477 from tock/dev/fetch-script-fail-on-error
lib/fetch-{libc++,newlib}: exit on error
2 parents 3d8c19b + 36433f2 commit 3dee250

File tree

3 files changed

+190
-66
lines changed

3 files changed

+190
-66
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ jobs:
2222
- uses: actions/checkout@v2
2323
with:
2424
submodules: false # LVGL makefile manually installs the submodule
25+
- name: Disable wget progress output
26+
run: |
27+
echo "verbose = off" >> $HOME/.wgetrc
2528
- name: ci-format
2629
run: pushd examples; ./format_all.sh || exit; popd
2730

@@ -41,6 +44,9 @@ jobs:
4144
- run: arm-none-eabi-gcc --version
4245
- name: setup-riscv-toolchain
4346
run: sudo apt-get install -y gcc-riscv64-unknown-elf
47+
- name: Disable wget progress output
48+
run: |
49+
echo "verbose = off" >> $HOME/.wgetrc
4450
- name: ci-build
4551
run: pushd examples; ./build_all.sh || exit; popd
4652
- name: ci-debug-build

‎lib/fetch-libc++.sh‎

Lines changed: 92 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
#!/usr/bin/env bash
22

3+
# Exit on error:
4+
set -Eeuo pipefail
5+
36
GCC_VERSION=$1
47

5-
if [ $GCC_VERSION = "14.1.0" ]; then
8+
if [[ $GCC_VERSION == "14.1.0" ]]; then
69
GCC_SHA="46d50ea6a380f2c977e6aad187da216c35b793b31b3ea6de9646339c2a22f13c"
7-
elif [ $GCC_VERSION = "13.2.0" ]; then
10+
elif [[ $GCC_VERSION == "13.2.0" ]]; then
811
GCC_SHA="9940242a1390f897f8c3fe6e7a8821e863580797f9a6e80a91f52e41dd8086a1"
9-
elif [ $GCC_VERSION = "12.3.0" ]; then
12+
elif [[ $GCC_VERSION == "12.3.0" ]]; then
1013
GCC_SHA="b0686eb1905594bde7b746fc58be97aceac8f802d8b5171adb6a4e84f3906d30"
11-
elif [ $GCC_VERSION = "10.5.0" ]; then
14+
elif [[ $GCC_VERSION == "10.5.0" ]]; then
1215
GCC_SHA="1cd4eef592bcc7b9ec77e2c21b50dabcff9b614b4cd1ec82a9dac238c8789c95"
1316
fi
1417

@@ -21,39 +24,95 @@ MIRRORS=(\
2124
"https://alpha.mirror.svc.schuermann.io/files/tock"\
2225
)
2326

24-
if test -x /usr/bin/shasum; then
25-
CHECK_SHA_CMD="shasum -a 256 -c"
26-
else
27-
CHECK_SHA_CMD="sha256sum -c"
28-
fi
27+
function check_sha256() {
28+
if test -x /usr/bin/shasum; then
29+
echo "$1 $2" | shasum -a 256 -c
30+
return $?
31+
else
32+
sha256sum -c <(echo "$1 $2")
33+
return $?
34+
fi
35+
}
36+
37+
FOUND=0
2938

30-
let FOUND=0
31-
32-
# Try from each mirror until we successfully download a .zip file.
33-
for MIRROR in ${MIRRORS[@]}; do
34-
URL=$MIRROR/$ZIP_FILE
35-
echo "Fetching libc++ from ${MIRROR}..."
36-
echo " Fetching ${URL}..."
37-
# Note: There must be two space characters for `shasum` (sha256sum doesn't care)
38-
wget -O $ZIP_FILE "$URL" && (echo "$GCC_SHA $ZIP_FILE" | $CHECK_SHA_CMD)
39-
if [ $? -ne 0 ]; then
40-
if test -f $ZIP_FILE; then
41-
file $ZIP_FILE
42-
ls -l $ZIP_FILE
43-
shasum -a 256 $ZIP_FILE
39+
function fetch_and_unpack() {
40+
# Try from each mirror until we successfully download a .zip file.
41+
for MIRROR in ${MIRRORS[@]}; do
42+
URL=$MIRROR/$ZIP_FILE
43+
echo "Fetching libc++ from ${MIRROR}..."
44+
echo " Fetching ${URL}..."
45+
# Note: There must be two space characters for `shasum` (sha256sum doesn't care)
46+
wget -O $ZIP_FILE "$URL" && check_sha256 "$GCC_SHA" "$ZIP_FILE"
47+
if [ $? -ne 0 ]; then
48+
if test -f $ZIP_FILE; then
49+
file $ZIP_FILE
50+
ls -l $ZIP_FILE
51+
shasum -a 256 $ZIP_FILE
52+
fi
53+
echo " WARNING: Fetching libc++ from mirror $MIRROR failed!" >&2
54+
else
55+
FOUND=1
56+
break
4457
fi
45-
echo " WARNING: Fetching libc++ from mirror $MIRROR failed!" >&2
58+
done
59+
60+
if [[ $FOUND -ne 0 ]]; then
61+
echo "Unpacking $ZIP_FILE..."
62+
# -n: don't overwrite existing files, -q: quiet mode
63+
unzip -n -q $ZIP_FILE
64+
echo "Done upacking $ZIP_FILE..."
65+
exit 0
4666
else
47-
let FOUND=1
48-
break
67+
echo "ERROR: Unable to find tock-libc++"
68+
exit -1
4969
fi
50-
done
70+
}
5171

52-
if [[ $FOUND -ne 0 ]]; then
53-
echo "Unpacking $ZIP_FILE..."
54-
unzip -q $ZIP_FILE
55-
echo "Done upacking $ZIP_FILE..."
72+
# We must ensure that multiple parallel fetch and unzip operations
73+
# don't trample over each other, which we do by obtaining a write-lock
74+
# on the ZIP file that's being downloaded / extracted.
75+
#
76+
# This will also truncate any already downloaded ZIP file, which is
77+
# fine because we'll overwrite it anyways.
78+
#
79+
# Under Linux platforms, we can expect `flock` to be installed, whereas for
80+
# BSDs and macOS, we use `shlock` instead:
81+
if which flock 2>&1 >/dev/null; then
82+
NONBLOCK_LOCK_ACQ_FAIL=0
83+
# We'd like for bash to choose the file descriptor, but that syntax
84+
# ({fd}<"${ZIP_FILE}") isn't supported on the ancient version of
85+
# Bash shipped on macOS. So we just pick one (200).
86+
while true; do
87+
: >> "${ZIP_FILE}"
88+
{
89+
if [ $NONBLOCK_LOCK_ACQ_FAIL -eq 0 ]; then
90+
flock -n 200 || NONBLOCK_LOCK_ACQ_FAIL=1
91+
if [ $NONBLOCK_LOCK_ACQ_FAIL -ne 0 ]; then
92+
# Try again, blocking this time:
93+
echo "Could not acquire non-blocking lock on ${ZIP_FILE}, waiting for lock to be released..." >&2
94+
continue
95+
fi
96+
else
97+
flock 200
98+
fi
99+
echo "Acquired lock on file ${ZIP_FILE}" >&2
100+
fetch_and_unpack "${ZIP_FILE}"
101+
} 200<"${ZIP_FILE}"
102+
done
103+
elif which shlock 2>&1 >/dev/null; then
104+
while true; do
105+
trap "rm -f ${ZIP_FILE}.lock" EXIT
106+
if shlock -f "${ZIP_FILE}.lock" -p "$BASHPID"; then
107+
break
108+
else
109+
echo "Could not acquire lock on ${ZIP_FILE}.lock, retrying in 10..." >&2
110+
sleep 10
111+
fi
112+
done
113+
echo "Acquired lock on file ${ZIP_FILE}" >&2
114+
fetch_and_unpack "${ZIP_FILE}"
56115
else
57-
echo "ERROR: Unable to find tock-libc++"
58-
exit -1
116+
echo "Neither \"flock\" nor \"shlock\" seem to be installed, cannot proceed!"
117+
exit 1
59118
fi

‎lib/fetch-newlib.sh‎

Lines changed: 92 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
#!/usr/bin/env bash
22

3+
# Exit on error:
4+
set -Eeuo pipefail
5+
36
NEWLIB_VERSION=$1
47

5-
if [ $NEWLIB_VERSION = "4.4.0.20231231" ]; then
8+
if [[ $NEWLIB_VERSION == "4.4.0.20231231" ]]; then
69
NEWLIB_SHA="686af44e1bba625eb24b3cfb1fd2d48a61848c1edebbd49b5dbec554ebf2ea94"
7-
elif [ $NEWLIB_VERSION = "4.3.0.20230120" ]; then
10+
elif [[ $NEWLIB_VERSION == "4.3.0.20230120" ]]; then
811
NEWLIB_SHA="2595f02f7cb2fd2e444f4ddc7955deca4c52deb3f91411c4d28326be8b0d9e0d"
9-
elif [ $NEWLIB_VERSION = "4.2.0.20211231" ]; then
12+
elif [[ $NEWLIB_VERSION == "4.2.0.20211231" ]]; then
1013
NEWLIB_SHA="5916d76f1cc3c0f5487275823c85a9a9954edfa15f5706342ecb254d634ed559"
1114
fi
1215

@@ -17,41 +20,97 @@ ZIP_FILE=libtock-newlib-$NEWLIB_VERSION.zip
1720
MIRRORS=(\
1821
"https://www.cs.virginia.edu/~bjc8c/archive/tock"\
1922
"https://alpha.mirror.svc.schuermann.io/files/tock"\
20-
)
23+
)
2124

22-
if test -x /usr/bin/shasum; then
23-
CHECK_SHA_CMD="shasum -a 256 -c"
24-
else
25-
CHECK_SHA_CMD="sha256sum -c"
26-
fi
25+
function check_sha256() {
26+
if test -x /usr/bin/shasum; then
27+
echo "$1 $2" | shasum -a 256 -c
28+
return $?
29+
else
30+
sha256sum -c <(echo "$1 $2")
31+
return $?
32+
fi
33+
}
34+
35+
FOUND=0
2736

28-
let FOUND=0
29-
30-
# Try from each mirror until we successfully download a .zip file.
31-
for MIRROR in ${MIRRORS[@]}; do
32-
URL=$MIRROR/$ZIP_FILE
33-
echo "Fetching newlib from ${MIRROR}..."
34-
echo " Fetching ${URL}..."
35-
# Note: There must be two space characters for `shasum` (sha256sum doesn't care)
36-
wget -O $ZIP_FILE "$URL" && (echo "$NEWLIB_SHA $ZIP_FILE" | $CHECK_SHA_CMD)
37-
if [ $? -ne 0 ]; then
38-
if test -f $ZIP_FILE; then
39-
file $ZIP_FILE
40-
ls -l $ZIP_FILE
41-
shasum -a 256 $ZIP_FILE
37+
function fetch_and_unpack() {
38+
# Try from each mirror until we successfully download a .zip file.
39+
for MIRROR in ${MIRRORS[@]}; do
40+
URL=$MIRROR/$ZIP_FILE
41+
echo "Fetching newlib from ${MIRROR}..."
42+
echo " Fetching ${URL}..."
43+
# Note: There must be two space characters for `shasum` (sha256sum doesn't care)
44+
wget -O $ZIP_FILE "$URL" && check_sha256 "$NEWLIB_SHA" "$ZIP_FILE"
45+
if [ $? -ne 0 ]; then
46+
if test -f $ZIP_FILE; then
47+
file $ZIP_FILE
48+
ls -l $ZIP_FILE
49+
shasum -a 256 $ZIP_FILE
50+
fi
51+
echo " WARNING: Fetching newlib from mirror $MIRROR failed!" >&2
52+
else
53+
FOUND=1
54+
break
4255
fi
43-
echo " WARNING: Fetching newlib from mirror $MIRROR failed!" >&2
56+
done
57+
58+
if [[ $FOUND -ne 0 ]]; then
59+
echo "Unpacking $ZIP_FILE..."
60+
# -n: don't overwrite existing files, -q: quiet mode
61+
unzip -n -q $ZIP_FILE
62+
echo "Done upacking $ZIP_FILE..."
63+
exit 0
4464
else
45-
let FOUND=1
46-
break
65+
echo "ERROR: Unable to find tock-newlib"
66+
exit -1
4767
fi
48-
done
68+
}
4969

50-
if [[ $FOUND -ne 0 ]]; then
51-
echo "Unpacking $ZIP_FILE..."
52-
unzip -q $ZIP_FILE
53-
echo "Done upacking $ZIP_FILE..."
70+
# We must ensure that multiple parallel fetch and unzip operations
71+
# don't trample over each other, which we do by obtaining a write-lock
72+
# on the ZIP file that's being downloaded / extracted.
73+
#
74+
# This will also truncate any already downloaded ZIP file, which is
75+
# fine because we'll overwrite it anyways.
76+
#
77+
# Under Linux platforms, we can expect `flock` to be installed, whereas for
78+
# BSDs and macOS, we use `shlock` instead:
79+
if which flock 2>&1 >/dev/null; then
80+
NONBLOCK_LOCK_ACQ_FAIL=0
81+
# We'd like for bash to choose the file descriptor, but that syntax
82+
# ({fd}<"${ZIP_FILE}") isn't supported on the ancient version of
83+
# Bash shipped on macOS. So we just pick one (200).
84+
while true; do
85+
: >> "${ZIP_FILE}"
86+
{
87+
if [ $NONBLOCK_LOCK_ACQ_FAIL -eq 0 ]; then
88+
flock -n 200 || NONBLOCK_LOCK_ACQ_FAIL=1
89+
if [ $NONBLOCK_LOCK_ACQ_FAIL -ne 0 ]; then
90+
# Try again, blocking this time:
91+
echo "Could not acquire non-blocking lock on ${ZIP_FILE}, waiting for lock to be released..." >&2
92+
continue
93+
fi
94+
else
95+
flock 200
96+
fi
97+
echo "Acquired lock on file ${ZIP_FILE}" >&2
98+
fetch_and_unpack "${ZIP_FILE}"
99+
} 200<"${ZIP_FILE}"
100+
done
101+
elif which shlock 2>&1 >/dev/null; then
102+
while true; do
103+
trap "rm -f ${ZIP_FILE}.lock" EXIT
104+
if shlock -f "${ZIP_FILE}.lock" -p "$BASHPID"; then
105+
break
106+
else
107+
echo "Could not acquire lock on ${ZIP_FILE}.lock, retrying in 10..." >&2
108+
sleep 10
109+
fi
110+
done
111+
echo "Acquired lock on file ${ZIP_FILE}" >&2
112+
fetch_and_unpack "${ZIP_FILE}"
54113
else
55-
echo "ERROR: Unable to find tock-newlib"
56-
exit -1
114+
echo "Neither \"flock\" nor \"shlock\" seem to be installed, cannot proceed!"
115+
exit 1
57116
fi

0 commit comments

Comments
 (0)