Skip to content

Commit fc21640

Browse files
TheBlueMattTheBlueMatt
authored
Merge pull request lightningdevkit#3020 from tnull/2024-04-drop-electrsd-autodownload-for-good
Drop `electrsd`'s auto-download feature for good
2 parents bc1c026 + 4320f48 commit fc21640

File tree

8 files changed

+109
-86
lines changed

8 files changed

+109
-86
lines changed

‎.github/workflows/build.yml‎

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
runs-on: ${{ matrix.platform }}
2323
steps:
2424
- name: Checkout source code
25-
uses: actions/checkout@v3
25+
uses: actions/checkout@v4
2626
- name: Install Rust ${{ matrix.toolchain }} toolchain
2727
run: |
2828
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ matrix.toolchain }}
@@ -32,14 +32,38 @@ jobs:
3232
run: |
3333
rustup target add thumbv7m-none-eabi
3434
sudo apt-get -y install gcc-arm-none-eabi
35-
- name: shellcheck the CI script
35+
- name: shellcheck the CI and `contrib` scripts
3636
if: "matrix.platform == 'ubuntu-latest'"
3737
run: |
3838
sudo apt-get -y install shellcheck
39-
shellcheck ci/ci-tests.sh
39+
shellcheck ci/*.sh -aP ci
40+
shellcheck contrib/*.sh -aP contrib
4041
- name: Set RUSTFLAGS to deny warnings
4142
if: "matrix.toolchain == '1.63.0'"
4243
run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV"
44+
- name: Enable caching for bitcoind
45+
id: cache-bitcoind
46+
uses: actions/cache@v4
47+
with:
48+
path: bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
49+
key: bitcoind-${{ runner.os }}-${{ runner.arch }}
50+
- name: Enable caching for electrs
51+
id: cache-electrs
52+
uses: actions/cache@v4
53+
with:
54+
path: bin/electrs-${{ runner.os }}-${{ runner.arch }}
55+
key: electrs-${{ runner.os }}-${{ runner.arch }}
56+
- name: Download bitcoind/electrs
57+
if: "matrix.platform != 'windows-latest' && (steps.cache-bitcoind.outputs.cache-hit != 'true' || steps.cache-electrs.outputs.cache-hit != 'true')"
58+
run: |
59+
source ./contrib/download_bitcoind_electrs.sh
60+
mkdir bin
61+
mv "$BITCOIND_EXE" bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
62+
mv "$ELECTRS_EXE" bin/electrs-${{ runner.os }}-${{ runner.arch }}
63+
- name: Set bitcoind/electrs environment variables
64+
run: |
65+
echo "BITCOIND_EXE=$( pwd )/bin/bitcoind-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV"
66+
echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV"
4367
- name: Run CI script
4468
shell: bash # Default on Winblows is powershell
4569
run: CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh
@@ -50,7 +74,7 @@ jobs:
5074
runs-on: ubuntu-latest
5175
steps:
5276
- name: Checkout source code
53-
uses: actions/checkout@v3
77+
uses: actions/checkout@v4
5478
with:
5579
fetch-depth: 0
5680
- name: Install Rust stable toolchain
@@ -72,14 +96,14 @@ jobs:
7296
TOOLCHAIN: stable
7397
steps:
7498
- name: Checkout source code
75-
uses: actions/checkout@v3
99+
uses: actions/checkout@v4
76100
- name: Install Rust ${{ env.TOOLCHAIN }} toolchain
77101
run: |
78102
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }}
79103
rustup override set ${{ env.TOOLCHAIN }}
80104
- name: Cache routing graph snapshot
81105
id: cache-graph
82-
uses: actions/cache@v3
106+
uses: actions/cache@v4
83107
with:
84108
path: lightning/net_graph-2023-12-10.bin
85109
key: ldk-net_graph-v0.0.118-2023-12-10.bin
@@ -96,7 +120,7 @@ jobs:
96120
EXPECTED_ROUTING_GRAPH_SNAPSHOT_SHASUM: e94b38ef4b3ce683893bf6a3ee28d60cb37c73b059403ff77b7e7458157968c2
97121
- name: Cache scorer snapshot
98122
id: cache-scorer
99-
uses: actions/cache@v3
123+
uses: actions/cache@v4
100124
with:
101125
path: lightning/scorer-2023-12-10.bin
102126
key: ldk-scorer-v0.0.118-2023-12-10.bin
@@ -142,7 +166,7 @@ jobs:
142166
TOOLCHAIN: stable
143167
steps:
144168
- name: Checkout source code
145-
uses: actions/checkout@v3
169+
uses: actions/checkout@v4
146170
with:
147171
fetch-depth: 0
148172
- name: Install Rust ${{ env.TOOLCHAIN }} toolchain
@@ -165,7 +189,7 @@ jobs:
165189
TOOLCHAIN: stable
166190
steps:
167191
- name: Checkout source code
168-
uses: actions/checkout@v3
192+
uses: actions/checkout@v4
169193
with:
170194
fetch-depth: 0
171195
- name: Install Rust ${{ env.TOOLCHAIN }} toolchain
@@ -194,7 +218,7 @@ jobs:
194218
TOOLCHAIN: 1.63
195219
steps:
196220
- name: Checkout source code
197-
uses: actions/checkout@v3
221+
uses: actions/checkout@v4
198222
- name: Install Rust ${{ env.TOOLCHAIN }} toolchain
199223
run: |
200224
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }}
@@ -217,7 +241,7 @@ jobs:
217241
TOOLCHAIN: stable
218242
steps:
219243
- name: Checkout source code
220-
uses: actions/checkout@v3
244+
uses: actions/checkout@v4
221245
- name: Install Rust ${{ env.TOOLCHAIN }} toolchain
222246
run: |
223247
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }}
@@ -235,7 +259,7 @@ jobs:
235259
TOOLCHAIN: 1.63.0
236260
steps:
237261
- name: Checkout source code
238-
uses: actions/checkout@v3
262+
uses: actions/checkout@v4
239263
- name: Install Rust ${{ env.TOOLCHAIN }} toolchain
240264
run: |
241265
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }}
@@ -250,7 +274,7 @@ jobs:
250274
runs-on: ubuntu-latest
251275
if: github.ref_name != 'main' # `main` has no diff with itself
252276
steps:
253-
- uses: actions/checkout@v3
277+
- uses: actions/checkout@v4
254278
with:
255279
fetch-depth: 0
256280
- name: Relative diff

‎ci/check-compiles.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/sh
22
set -e
33
set -x
4-
echo Testing $(git log -1 --oneline)
4+
echo "Testing $(git log -1 --oneline)"
55
cargo check
66
cargo doc
77
cargo doc --document-private-items

‎ci/check-each-commit.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,4 @@ if [ "$(git log --pretty="%H %D" | grep "^[0-9a-f]*.* $1")" = "" ]; then
1212
echo "It seems like the current checked-out commit is not based on $1"
1313
exit 1
1414
fi
15-
git rebase --exec ci/check-compiles.sh $1
15+
git rebase --exec ci/check-compiles.sh "$1"

‎ci/ci-tests.sh‎

Lines changed: 16 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -14,50 +14,6 @@ function PIN_RELEASE_DEPS {
1414
return 0 # Don't fail the script if our rustc is higher than the last check
1515
}
1616

17-
# The tests of `lightning-transaction-sync` require `electrs` and `bitcoind`
18-
# binaries. Here, we download the binaries, validate them, and export their
19-
# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the
20-
# `electrsd`/`bitcoind` crates in our tests.
21-
function DOWNLOAD_ELECTRS_AND_BITCOIND {
22-
ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases"
23-
ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254"
24-
BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/"
25-
BITCOIND_VERSION="25.1"
26-
if [[ "$HOST_PLATFORM" == *linux* ]]; then
27-
ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip
28-
ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1"
29-
BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz
30-
BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b"
31-
elif [[ "$HOST_PLATFORM" == *darwin* ]]; then
32-
ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip
33-
ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd"
34-
BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz
35-
BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1"
36-
else
37-
echo -e "\n\nUnsupported platform. Exiting.."
38-
exit 1
39-
fi
40-
41-
DL_TMP_DIR=$(mktemp -d)
42-
trap 'rm -rf -- "$DL_TMP_DIR"' EXIT
43-
44-
pushd "$DL_TMP_DIR"
45-
ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME"
46-
curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL"
47-
echo "$ELECTRS_DL_HASH $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c
48-
unzip "$ELECTRS_DL_FILE_NAME"
49-
export ELECTRS_EXE="$DL_TMP_DIR"/electrs
50-
chmod +x "$ELECTRS_EXE"
51-
52-
BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME"
53-
curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL"
54-
echo "$BITCOIND_DL_HASH $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c
55-
tar xzf "$BITCOIND_DL_FILE_NAME"
56-
export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind
57-
chmod +x "$BITCOIND_EXE"
58-
popd
59-
}
60-
6117
PIN_RELEASE_DEPS # pin the release dependencies in our main workspace
6218

6319
# Starting with version 1.10.0, the `regex` crate has an MSRV of rustc 1.65.0.
@@ -71,9 +27,6 @@ PIN_RELEASE_DEPS # pin the release dependencies in our main workspace
7127

7228
export RUST_BACKTRACE=1
7329

74-
# Build `lightning-transaction-sync` in no_download mode.
75-
export RUSTFLAGS="$RUSTFLAGS --cfg no_download"
76-
7730
echo -e "\n\nBuilding and testing all workspace crates..."
7831
cargo test --verbose --color always
7932
cargo check --verbose --color always
@@ -91,20 +44,22 @@ cargo check --verbose --color always --features rpc-client,rest-client,tokio
9144
popd
9245

9346
if [[ "$HOST_PLATFORM" != *windows* ]]; then
94-
echo -e "\n\nBuilding and testing Transaction Sync Clients with features"
95-
pushd lightning-transaction-sync
96-
97-
DOWNLOAD_ELECTRS_AND_BITCOIND
98-
99-
cargo test --verbose --color always --features esplora-blocking
100-
cargo check --verbose --color always --features esplora-blocking
101-
cargo test --verbose --color always --features esplora-async
102-
cargo check --verbose --color always --features esplora-async
103-
cargo test --verbose --color always --features esplora-async-https
104-
cargo check --verbose --color always --features esplora-async-https
105-
cargo test --verbose --color always --features electrum
106-
cargo check --verbose --color always --features electrum
107-
popd
47+
if [ -z "$BITCOIND_EXE" ] || [ -z "$ELECTRS_EXE" ]; then
48+
echo -e "\n\nSkipping testing Transaction Sync Clients due to BITCOIND_EXE or ELECTRS_EXE being unset."
49+
else
50+
echo -e "\n\nBuilding and testing Transaction Sync Clients with features"
51+
pushd lightning-transaction-sync
52+
53+
cargo test --verbose --color always --features esplora-blocking
54+
cargo check --verbose --color always --features esplora-blocking
55+
cargo test --verbose --color always --features esplora-async
56+
cargo check --verbose --color always --features esplora-async
57+
cargo test --verbose --color always --features esplora-async-https
58+
cargo check --verbose --color always --features esplora-async-https
59+
cargo test --verbose --color always --features electrum
60+
cargo check --verbose --color always --features electrum
61+
popd
62+
fi
10863
fi
10964

11065
echo -e "\n\nTest futures builds"

‎ci/rustfmt.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ VERS=""
1414

1515
# Run fmt
1616
TMP_FILE=$(mktemp)
17-
find . -name '*.rs' -type f |sort >$TMP_FILE
18-
for file in $(comm -23 $TMP_FILE rustfmt_excluded_files); do
17+
find . -name '*.rs' -type f |sort >"$TMP_FILE"
18+
for file in $(comm -23 "$TMP_FILE" rustfmt_excluded_files); do
1919
echo "Checking formatting of $file"
20-
rustfmt $VERS --edition 2021 --check $file
20+
rustfmt $VERS --edition 2021 --check "$file"
2121
done

‎contrib/download_bitcoind_electrs.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
set -eox pipefail
3+
4+
# The tests of `lightning-transaction-sync` require `electrs` and `bitcoind`
5+
# binaries. Here, we download the binaries, validate them, and export their
6+
# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the
7+
# `electrsd`/`bitcoind` crates in our tests.
8+
9+
HOST_PLATFORM="$(rustc --version --verbose | grep "host:" | awk '{ print $2 }')"
10+
ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases"
11+
ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254"
12+
BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/"
13+
BITCOIND_VERSION="25.1"
14+
if [[ "$HOST_PLATFORM" == *linux* ]]; then
15+
ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip
16+
ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1"
17+
BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz
18+
BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b"
19+
elif [[ "$HOST_PLATFORM" == *darwin* ]]; then
20+
ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip
21+
ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd"
22+
BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz
23+
BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1"
24+
else
25+
printf "\n\n"
26+
echo "Unsupported platform: $HOST_PLATFORM Exiting.."
27+
exit 1
28+
fi
29+
30+
DL_TMP_DIR=$(mktemp -d)
31+
trap 'rm -rf -- "$DL_TMP_DIR"' EXIT
32+
33+
pushd "$DL_TMP_DIR"
34+
ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME"
35+
curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL"
36+
echo "$ELECTRS_DL_HASH $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c
37+
unzip "$ELECTRS_DL_FILE_NAME"
38+
export ELECTRS_EXE="$DL_TMP_DIR"/electrs
39+
chmod +x "$ELECTRS_EXE"
40+
41+
BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME"
42+
curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL"
43+
echo "$BITCOIND_DL_HASH $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c
44+
tar xzf "$BITCOIND_DL_FILE_NAME"
45+
export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind
46+
chmod +x "$BITCOIND_EXE"
47+
popd

‎contrib/run-rustfmt.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ VERS=""
1414

1515
# Run fmt
1616
TMP_FILE=$(mktemp)
17-
find . -name '*.rs' -type f |sort >$TMP_FILE
18-
for file in $(comm -23 $TMP_FILE rustfmt_excluded_files); do
17+
find . -name '*.rs' -type f |sort >"$TMP_FILE"
18+
for file in $(comm -23 "$TMP_FILE" rustfmt_excluded_files); do
1919
echo "Formatting $file..."
20-
rustfmt $VERS --edition 2021 $file
20+
rustfmt $VERS --edition 2021 "$file"
2121
done

‎lightning-transaction-sync/Cargo.toml‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,7 @@ electrum-client = { version = "0.19.0", optional = true }
3434
lightning = { version = "0.0.123-beta", path = "../lightning", default-features = false, features = ["std", "_test_utils"] }
3535
tokio = { version = "1.35.0", features = ["full"] }
3636

37-
[target.'cfg(all(not(target_os = "windows"), not(no_download)))'.dev-dependencies]
38-
electrsd = { version = "0.27.3", default-features = false, features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }
39-
40-
[target.'cfg(all(not(target_os = "windows"), no_download))'.dev-dependencies]
37+
[target.'cfg(not(target_os = "windows"))'.dev-dependencies]
4138
electrsd = { version = "0.27.3", default-features = false, features = ["legacy"] }
4239

4340
[lints]

0 commit comments

Comments
 (0)