feat(download): support zst archives in reth download (paradigmxyz#18237)

Co-authored-by: Matthias Seitz <[email protected]>

Author: 2 people

Cargo.lock

@@ -72,6 +72,7 @@ human_bytes.workspace = true
 eyre.workspace = true
 clap = { workspace = true, features = ["derive", "env"] }
 lz4.workspace = true
+zstd.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 tar.workspace = true

crates/cli/commands/Cargo.toml

@@ -15,10 +15,12 @@ use std::{
 use tar::Archive;
 use tokio::task;
 use tracing::info;
+use zstd::stream::read::Decoder as ZstdDecoder;
 
 const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
 const MERKLE_BASE_URL: &str = "https://downloads.merkle.io";
-const EXTENSION_TAR_FILE: &str = ".tar.lz4";
+const EXTENSION_TAR_LZ4: &str = ".tar.lz4";
+const EXTENSION_TAR_ZSTD: &str = ".tar.zst";
 
 #[derive(Debug, Parser)]
 pub struct DownloadCommand<C: ChainSpecParser> {
@@ -148,7 +150,27 @@ impl<R: Read> Read for ProgressReader<R> {
     }
 }
 
-/// Downloads and extracts a snapshot with blocking approach
+/// Supported compression formats for snapshots
+#[derive(Debug, Clone, Copy)]
+enum CompressionFormat {
+    Lz4,
+    Zstd,
+}
+
+impl CompressionFormat {
+    /// Detect compression format from file extension
+    fn from_url(url: &str) -> Result<Self> {
+        if url.ends_with(EXTENSION_TAR_LZ4) {
+            Ok(Self::Lz4)
+        } else if url.ends_with(EXTENSION_TAR_ZSTD) {
+            Ok(Self::Zstd)
+        } else {
+            Err(eyre::eyre!("Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}", url))
+        }
+    }
+}
+
+/// Downloads and extracts a snapshot, blocking until finished.
 fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
     let client = reqwest::blocking::Client::builder().build()?;
     let response = client.get(url).send()?.error_for_status()?;
@@ -160,11 +182,18 @@ fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
     })?;
 
     let progress_reader = ProgressReader::new(response, total_size);
+    let format = CompressionFormat::from_url(url)?;
 
-    let decoder = Decoder::new(progress_reader)?;
-    let mut archive = Archive::new(decoder);
-
-    archive.unpack(target_dir)?;
+    match format {
+        CompressionFormat::Lz4 => {
+            let decoder = Decoder::new(progress_reader)?;
+            Archive::new(decoder).unpack(target_dir)?;
+        }
+        CompressionFormat::Zstd => {
+            let decoder = ZstdDecoder::new(progress_reader)?;
+            Archive::new(decoder).unpack(target_dir)?;
+        }
+    }
 
     info!(target: "reth::cli", "Extraction complete.");
     Ok(())
@@ -191,9 +220,5 @@ async fn get_latest_snapshot_url() -> Result<String> {
         .trim()
         .to_string();
 
-    if !filename.ends_with(EXTENSION_TAR_FILE) {
-        return Err(eyre::eyre!("Unexpected snapshot filename format: {}", filename));
-    }
-
     Ok(format!("{MERKLE_BASE_URL}/{filename}"))
 }