fix(ci): speed up snapshot downloading on CI (#6592)

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

Author: hanabi1224

.github/workflows/forest.yml

@@ -36,6 +36,7 @@ env:
   CXX: sccache clang++
   FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters
   SHELL_IMAGE: busybox
+  FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH: /var/tmp/forest_snapshot_calibnet_latest.forest.car.zst
 jobs:
   build-macos:
     name: Build MacOS

docs/docs/users/reference/env_variables.md

@@ -56,6 +56,7 @@ process.
 | `FOREST_ETH_BLOCK_CACHE_SIZE`                             | positive integer                 | 500                                            | 1                                                             | The size of Eth block cache                                                                                           |
 | `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK`            | 1 or true                        | false                                          | 1                                                             | Whether or not to backfill full tipsets from the p2p network                                                          |
 | `FOREST_STRICT_JSON`                                      | 1 or true                        | false                                          | 1                                                             | Enable strict JSON validation to detect duplicate keys in RPC requests                                                |
+| `FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH`                      | URL or file path                 | empty                                          | `/var/tmp/forest_snapshot_calibnet.forest.car.zst`            | Override snapshot path for `--auto-download-snapshot`                                                                 |
 
 ### `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT`
 

scripts/tests/calibnet_kademlia_check.sh

@@ -24,6 +24,7 @@ cat <<- EOF > $CONFIG_PATH
 	kademlia = true
 EOF
 
+handle_auto_download_snapshot_env
 $FOREST_PATH --chain calibnet --encrypt-keystore false --auto-download-snapshot --config "$CONFIG_PATH" --save-token ./admin_token --rpc-address 127.0.0.1:12345 --metrics-address 127.0.0.1:6117 --healthcheck-address 127.0.0.1:2347 &
 FOREST_NODE_PID=$!
 # Verify that more peers are connected via kademlia

scripts/tests/calibnet_no_discovery_check.sh

@@ -11,6 +11,7 @@ function shutdown {
 
 trap shutdown EXIT
 
+handle_auto_download_snapshot_env
 $FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --exit-after-init
 $FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" &
 FOREST_NODE_PID=$!

scripts/tests/calibnet_stateless_mode_check.sh

@@ -24,6 +24,7 @@ cat <<- EOF > $CONFIG_PATH
 	kademlia = false
 EOF
 
+handle_auto_download_snapshot_env
 # Disable discovery to not connect to more nodes
 $FOREST_PATH --chain calibnet --encrypt-keystore false --auto-download-snapshot --config "$CONFIG_PATH" --rpc false --metrics-address 127.0.0.1:6117 --healthcheck-address 127.0.0.1:2347 &
 FOREST_NODE_PID=$!

scripts/tests/harness.sh

@@ -16,13 +16,27 @@ LOG_DIRECTORY=$TMP_DIR/logs
 export TMP_DIR
 export LOG_DIRECTORY
 
+# Use `aria2` to fetch the latest calibnet snapshot 
+# when `FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH` is set
+# This optimization is used on CI to speed up snapshot downloading
+function handle_auto_download_snapshot_env {
+  if [[ -n "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH:-}" ]]; then
+    echo "Downloading calibnet snapshot to ${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}"
+    mkdir -p "$(dirname "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")"
+    aria2c -x5 -c https://forest-archive.chainsafe.dev/latest/calibnet/ \
+      -d "$(dirname "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")" \
+      -o "$(basename "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")"
+  fi
+}
+
 function forest_import_non_calibnet_snapshot {
   echo "Importing a non calibnet snapshot"
   $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --import-snapshot ./test-snapshots/chain4.car
 }
 
 function forest_download_and_import_snapshot {
   echo "Downloading and importing snapshot"
+  handle_auto_download_snapshot_env
   $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --height=-200 --auto-download-snapshot
 }
 

src/daemon/mod.rs

@@ -709,8 +709,19 @@ async fn maybe_set_snapshot_path(
         (false, _, _) => {}   // noop - don't need a snapshot
         (true, true, _) => {} // noop - we need a snapshot, and we have one
         (true, false, true) => {
-            let url = crate::cli_shared::snapshot::stable_url(vendor, chain)?;
-            config.client.snapshot_path = Some(url.to_string().into());
+            const AUTO_SNAPSHOT_PATH_ENV_KEY: &str = "FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH";
+            match std::env::var(AUTO_SNAPSHOT_PATH_ENV_KEY) {
+                Ok(path) if !path.is_empty() => {
+                    tracing::info!(
+                        "importing snapshot from {path} set by `{AUTO_SNAPSHOT_PATH_ENV_KEY}`"
+                    );
+                    config.client.snapshot_path = Some(path.into());
+                }
+                _ => {
+                    let url = crate::cli_shared::snapshot::stable_url(vendor, chain)?;
+                    config.client.snapshot_path = Some(url.to_string().into());
+                }
+            }
         }
         (true, false, false) => {
             // we need a snapshot, don't have one, and don't have permission to download one, so ask the user