fix(ci): prefetch state computation snapshots to make codecov less flaky (#6496)

Author: hanabi1224

.github/workflows/coverage.yml

@@ -51,7 +51,6 @@ jobs:
       - uses: actions/checkout@v6
       - uses: jdx/mise-action@v3
       - uses: taiki-e/install-action@cargo-llvm-cov
-      - uses: taiki-e/install-action@nextest
       - name: Fetch proof params and RPC test snapshots
         run: |
           cargo run --bin forest-dev --no-default-features --profile quick -- fetch-test-snapshots --actor-bundle $FOREST_ACTOR_BUNDLE_PATH

.github/workflows/unit-tests.yml

@@ -37,6 +37,7 @@ env:
   CXX: "sccache clang++"
   FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters
   RUST_LOG: error
+  FOREST_ACTOR_BUNDLE_PATH: /var/tmp/forest_actor_bundle.car.zst
 
 jobs:
   tests-release:
@@ -59,13 +60,6 @@ jobs:
         continue-on-error: true
       - name: Checkout Sources
         uses: actions/checkout@v6
-      - uses: actions/cache/restore@v5
-        with:
-          path: |
-            /home/runner/.cache/forest/test/rpc-snapshots/rpc_test
-          # Broad key + prefix-based restore so the latest rpcsnap-<hash> is picked up.
-          key: rpcsnap-
-          enableCrossOsArchive: true
       - name: Setup sccache
         uses: mozilla-actions/[email protected]
         timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
@@ -77,7 +71,7 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Fetch proof params and RPC test snapshots
         run: |
-          cargo run --bin forest-dev --no-default-features --profile quick -- fetch-test-snapshots
+          cargo run --bin forest-dev --no-default-features --profile quick -- fetch-test-snapshots --actor-bundle $FOREST_ACTOR_BUNDLE_PATH
           ls -ahl $FIL_PROOFS_PARAMETER_CACHE
       - uses: jdx/mise-action@v3
       - run: |
@@ -86,14 +80,3 @@ jobs:
           # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times
           RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld"
           FOREST_TEST_SKIP_PROOF_PARAM_CHECK: 1
-      - id: get-cache-hash
-        run: |
-          ls -lhR ~/.cache/forest/test/rpc-snapshots/rpc_test/*
-          echo "hash=$(openssl md5 ~/.cache/forest/test/rpc-snapshots/rpc_test/* | sort | openssl md5 | cut -d ' ' -f 2)" >> $GITHUB_OUTPUT
-        shell: bash
-      - uses: actions/cache/save@v5
-        with:
-          path: |
-            /home/runner/.cache/forest/test/rpc-snapshots/rpc_test
-          key: rpcsnap-${{ steps.get-cache-hash.outputs.hash }}
-          enableCrossOsArchive: true

Cargo.lock

@@ -115,7 +115,6 @@ is-terminal = "0.4"
 itertools = "0.14"
 jsonrpsee = { version = "0.26", features = ["server", "ws-client", "http-client", "macros"] }
 jsonwebtoken = { version = "10", features = ["aws_lc_rs"] }
-justjson = "0.3"
 keccak-hash = "0.12"
 kubert-prometheus-process = "0.2"
 lazy-regex = "3"
@@ -196,6 +195,7 @@ similar = "2"
 slotmap = "1"
 smallvec = "1"
 smart-default = "0.7"
+sonic-rs = "0.5"
 spire_enum = "1"
 stacker = "0.1"
 static_assertions = "1"

Cargo.toml

@@ -6,6 +6,9 @@ mod state_cmd;
 use crate::cli_shared::cli::HELP_MESSAGE;
 use crate::networks::generate_actor_bundle;
 use crate::rpc::Client;
+use crate::state_manager::utils::state_compute::{
+    get_state_snapshot_file, list_state_snapshot_files,
+};
 use crate::utils::net::{DownloadFileOption, download_file_with_cache};
 use crate::utils::proofs_api::ensure_proof_params_downloaded;
 use crate::utils::version::FOREST_VERSION_STRING;
@@ -63,8 +66,27 @@ async fn fetch_test_snapshots(actor_bundle: Option<PathBuf>) -> anyhow::Result<(
         println!("Wrote the actors bundle to {}", actor_bundle.display());
     }
 
+    // Prepare state computation and validation snapshots
+    fetch_state_tests().await?;
+
     // Prepare RPC test snapshots
-    fetch_rpc_tests().await
+    fetch_rpc_tests().await?;
+
+    Ok(())
+}
+
+pub async fn fetch_state_tests() -> anyhow::Result<()> {
+    let files = list_state_snapshot_files().await?;
+    let mut joinset = JoinSet::new();
+    for file in files {
+        joinset.spawn(async move { get_state_snapshot_file(&file).await });
+    }
+    for result in joinset.join_all().await {
+        if let Err(e) = result {
+            tracing::warn!("{e}");
+        }
+    }
+    Ok(())
 }
 
 async fn fetch_rpc_tests() -> anyhow::Result<()> {

src/dev/subcommands/mod.rs

@@ -7,7 +7,6 @@
 //! This means JSON like `{"/":"cid1", "/":"cid2"}` will keep only the last value, which can lead to unexpected behavior in RPC calls.
 
 use ahash::HashSet;
-use justjson::Value;
 
 pub const STRICT_JSON_ENV: &str = "FOREST_STRICT_JSON";
 
@@ -36,28 +35,22 @@ pub fn validate_json_for_duplicates(json_str: &str) -> Result<(), String> {
         return Ok(());
     }
 
-    fn check_value(value: &Value) -> Result<(), String> {
-        match value {
-            Value::Object(obj) => {
+    fn check_value(value: &sonic_rs::Value) -> Result<(), String> {
+        match value.as_ref() {
+            sonic_rs::ValueRef::Object(obj) => {
                 let mut seen = HashSet::default();
-                for entry in obj.iter() {
-                    let key = entry
-                        .key
-                        .as_str()
-                        .ok_or_else(|| "Invalid JSON key".to_string())?;
-
+                for (key, value) in obj.iter() {
                     if !seen.insert(key) {
                         return Err(format!(
-                            "duplicate key '{}' in JSON object - this likely indicates malformed input. \
-                            Set {}=0 to disable this check",
-                            key, STRICT_JSON_ENV
+                            "duplicate key '{key}' in JSON object - this likely indicates malformed input. \
+                            Set {STRICT_JSON_ENV}=0 to disable this check"
                         ));
                     }
-                    check_value(&entry.value)?;
+                    check_value(value)?;
                 }
                 Ok(())
             }
-            Value::Array(arr) => {
+            sonic_rs::ValueRef::Array(arr) => {
                 for item in arr.iter() {
                     check_value(item)?;
                 }
@@ -67,7 +60,7 @@ pub fn validate_json_for_duplicates(json_str: &str) -> Result<(), String> {
         }
     }
     // defer to serde_json for invalid JSON
-    let value = match Value::from_json(json_str) {
+    let value: sonic_rs::Value = match sonic_rs::from_str(json_str) {
         Ok(v) => v,
         Err(_) => return Ok(()),
     };