feat(xtool): add compressed-size

.github/workflows/hf-xet-tests.yml

@@ -45,4 +45,9 @@ jobs:
       - name: Check Cargo.lock has no uncommitted changes
         run: |
           # the Build wheel step would update hf_xet/Cargo.lock if it is out of date
-          test -z "$(git status --porcelain hf_xet/Cargo.lock)" || (echo "hf_xet/Cargo.lock has uncommitted changes!" && exit 1)
+          if [ -n "$(git status --porcelain hf_xet/Cargo.lock)" ]; then
+            echo "::error::hf_xet/Cargo.lock has uncommitted changes!"
+            echo "Diff:"
+            git diff hf_xet/Cargo.lock
+            exit 1
+          fi

Cargo.toml

@@ -57,6 +57,7 @@ getrandom = "0.3"
 git-url-parse = "0.4"
 git2 = "0.20"
 half = "2.4"
+humansize = "2.1"
 heapify = "0.2"
 heed = "0.11"
 http = "1"

data/Cargo.toml

@@ -42,6 +42,7 @@ xet_runtime = { path = "../xet_runtime" }
 anyhow = { workspace = true }
 async-trait = { workspace = true }
 bytes = { workspace = true }
+humansize = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true }
 jsonwebtoken = { workspace = true }

data/src/bin/xtool.rs

@@ -12,6 +12,7 @@ use data::data_client::default_config;
 use data::migration_tool::hub_client_token_refresher::HubClientTokenRefresher;
 use data::migration_tool::migrate::migrate_files_impl;
 use hub_client::{BearerCredentialHelper, HubClient, Operation, RepoInfo};
+use humansize::{BINARY, DECIMAL, format_size};
 use merklehash::MerkleHash;
 use utils::auth::TokenRefresher;
 use walkdir::WalkDir;
@@ -76,6 +77,8 @@ enum Command {
     Dedup(DedupArg),
     /// Queries reconstruction information about a file.
     Query(QueryArg),
+    /// Calculates the compressed size of a xet-file by summing url_range sizes.
+    CompressedSize(CompressedSizeArg),
 }
 
 #[derive(Args)]
@@ -116,6 +119,12 @@ struct QueryArg {
     bytes_range: Option<FileRange>,
 }
 
+#[derive(Args)]
+struct CompressedSizeArg {
+    /// Xet-hash of a file.
+    hash: String,
+}
+
 impl Command {
     async fn run(self, hub_client: HubClient) -> Result<()> {
         match self {
@@ -161,6 +170,44 @@ impl Command {
 
                 Ok(())
             },
+            Command::CompressedSize(arg) => {
+                let file_hash = MerkleHash::from_hex(&arg.hash)?;
+                // Query reconstruction for full file (no Range header)
+                let ret = query_reconstruction(file_hash, None, hub_client).await?;
+
+                match ret {
+                    Some(response) => {
+                        let mut total_compressed_size = 0u64;
+
+                        for fetch_infos in response.fetch_info.values() {
+                            for fetch_info in fetch_infos {
+                                let range_size = fetch_info.url_range.end - fetch_info.url_range.start;
+                                total_compressed_size += range_size;
+                            }
+                        }
+
+                        let total_uncompressed_size: u64 =
+                            response.terms.iter().map(|term| term.unpacked_length as u64).sum();
+
+                        // Count unique XORBs
+                        let unique_xorbs: std::collections::HashSet<_> =
+                            response.terms.iter().map(|term| &term.hash).collect();
+
+                        println!("Compressed Size:   {}", format_bytes_with_units(total_compressed_size));
+                        println!("Uncompressed Size: {}", format_bytes_with_units(total_uncompressed_size));
+                        println!(
+                            "Compression Ratio:  {:.2}%",
+                            (total_compressed_size as f64 / total_uncompressed_size as f64) * 100.0
+                        );
+                        println!("XORBs:             {} unique", unique_xorbs.len());
+                        Ok(())
+                    },
+                    None => {
+                        eprintln!("No reconstruction information found for hash {}", arg.hash);
+                        Ok(())
+                    },
+                }
+            },
         }
     }
 }
@@ -193,6 +240,13 @@ fn is_git_special_files(path: &str) -> bool {
     matches!(path, ".git" | ".gitignore" | ".gitattributes")
 }
 
+/// Format bytes with binary and decimal units on one line
+fn format_bytes_with_units(bytes: u64) -> String {
+    let binary = format_size(bytes, BINARY);
+    let decimal = format_size(bytes, DECIMAL);
+    format!("{} bytes {} {}", bytes, binary, decimal)
+}
+
 async fn query_reconstruction(
     file_hash: MerkleHash,
     bytes_range: Option<FileRange>,