Integrate linecount analysis into tarball processing

This enhances the tarball processing pipeline to include SLOC analysis by adding `crates_io_linecount` dependency to the tarball processing crate and extending the `TarballInfo` struct with a `linecount_stats` field.

The integration occurs seamlessly during tarball file processing, where each qualifying source file is analyzed and its statistics are accumulated. All tarball processing test snapshots are updated to include linecount data, demonstrating the feature works correctly across various crate structures.

The integration preserves existing functionality while adding minimal overhead to the tarball validation and processing pipeline.

Author: Turbo87

crates/crates_io_tarball/Cargo.toml

@@ -13,6 +13,7 @@ builder = ["dep:flate2", "dep:tar"]
 [dependencies]
 astral-tokio-tar = "=0.5.2"
 cargo-manifest = "=0.19.1"
+crates_io_linecount = { path = "../crates_io_linecount" }
 flate2 = { version = "=1.1.2", optional = true }
 serde = { version = "=1.0.219", features = ["derive"] }
 serde_json = "=1.0.140"

crates/crates_io_tarball/src/lib.rs

@@ -30,6 +30,7 @@ const DEFAULT_BUF_SIZE: usize = 128 * 1024;
 pub struct TarballInfo {
     pub manifest: Manifest,
     pub vcs_info: Option<CargoVcsInfo>,
+    pub linecount_stats: crates_io_linecount::LinecountStats,
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -74,6 +75,7 @@ pub async fn process_tarball<R: tokio::io::AsyncRead + Unpin>(
     let mut vcs_info = None;
     let mut paths = Vec::new();
     let mut manifests = BTreeMap::new();
+    let mut linecount_stats = crates_io_linecount::LinecountStats::new();
     let mut entries = archive.entries()?;
 
     while let Some(entry) = entries.next().await {
@@ -103,6 +105,12 @@ pub async fn process_tarball<R: tokio::io::AsyncRead + Unpin>(
 
         paths.push(in_pkg_path.to_path_buf());
 
+        // Check if this file should be counted for line statistics
+        let is_file = entry_type.is_file();
+        let language_type_for_counting = is_file
+            .then(|| crates_io_linecount::should_count_path(in_pkg_path))
+            .flatten();
+
         // Let's go hunting for the VCS info and crate manifest. The only valid place for these is
         // in the package root in the tarball.
         let in_pkg_path_str = in_pkg_path.to_string_lossy();
@@ -121,6 +129,11 @@ pub async fn process_tarball<R: tokio::io::AsyncRead + Unpin>(
             validate_manifest(&manifest)?;
 
             manifests.insert(owned_entry_path, manifest);
+        } else if let Some(language_type) = language_type_for_counting {
+            // If this is a file that we want to count, read it and update the line count stats.
+            let mut contents = Vec::new();
+            entry.read_to_end(&mut contents).await?;
+            linecount_stats.add_file(language_type, &contents);
         }
     }
 
@@ -146,7 +159,11 @@ pub async fn process_tarball<R: tokio::io::AsyncRead + Unpin>(
 
     manifest.complete_from_abstract_filesystem(&PathsFileSystem(paths))?;
 
-    Ok(TarballInfo { manifest, vcs_info })
+    Ok(TarballInfo {
+        manifest,
+        vcs_info,
+        linecount_stats,
+    })
 }
 
 struct PathsFileSystem(Vec<PathBuf>);

crates/crates_io_tarball/src/snapshots/crates_io_tarball__tests__app.snap

@@ -76,4 +76,15 @@ TarballInfo {
         badges: None,
     },
     vcs_info: None,
+    linecount_stats: LinecountStats {
+        languages: {
+            Rust: LanguageStats {
+                code_lines: 1,
+                comment_lines: 0,
+                files: 1,
+            },
+        },
+        total_code_lines: 1,
+        total_comment_lines: 0,
+    },
 }

crates/crates_io_tarball/src/snapshots/crates_io_tarball__tests__lib.snap

@@ -80,4 +80,15 @@ TarballInfo {
         badges: None,
     },
     vcs_info: None,
+    linecount_stats: LinecountStats {
+        languages: {
+            Rust: LanguageStats {
+                code_lines: 1,
+                comment_lines: 0,
+                files: 1,
+            },
+        },
+        total_code_lines: 1,
+        total_comment_lines: 0,
+    },
 }

crates/crates_io_tarball/src/snapshots/crates_io_tarball__tests__lib_with_bins_and_example.snap

@@ -140,4 +140,15 @@ TarballInfo {
         badges: None,
     },
     vcs_info: None,
+    linecount_stats: LinecountStats {
+        languages: {
+            Rust: LanguageStats {
+                code_lines: 3,
+                comment_lines: 0,
+                files: 3,
+            },
+        },
+        total_code_lines: 3,
+        total_comment_lines: 0,
+    },
 }

crates/crates_io_tarball/src/snapshots/crates_io_tarball__tests__process_tarball_test.snap

@@ -57,4 +57,9 @@ TarballInfo {
         badges: None,
     },
     vcs_info: None,
+    linecount_stats: LinecountStats {
+        languages: {},
+        total_code_lines: 0,
+        total_comment_lines: 0,
+    },
 }

crates/crates_io_tarball/src/snapshots/crates_io_tarball__tests__process_tarball_test_incomplete_vcs_info.snap

@@ -61,4 +61,9 @@ TarballInfo {
             path_in_vcs: "",
         },
     ),
+    linecount_stats: LinecountStats {
+        languages: {},
+        total_code_lines: 0,
+        total_comment_lines: 0,
+    },
 }

crates/crates_io_tarball/src/snapshots/crates_io_tarball__tests__process_tarball_test_lowercase_manifest.snap

@@ -61,4 +61,9 @@ TarballInfo {
         badges: None,
     },
     vcs_info: None,
+    linecount_stats: LinecountStats {
+        languages: {},
+        total_code_lines: 0,
+        total_comment_lines: 0,
+    },
 }

crates/crates_io_tarball/src/snapshots/crates_io_tarball__tests__process_tarball_test_manifest.snap

@@ -71,4 +71,9 @@ TarballInfo {
         badges: None,
     },
     vcs_info: None,
+    linecount_stats: LinecountStats {
+        languages: {},
+        total_code_lines: 0,
+        total_comment_lines: 0,
+    },
 }

crates/crates_io_tarball/src/snapshots/crates_io_tarball__tests__process_tarball_test_manifest_with_boolean_readme.snap

@@ -63,4 +63,9 @@ TarballInfo {
         badges: None,
     },
     vcs_info: None,
+    linecount_stats: LinecountStats {
+        languages: {},
+        total_code_lines: 0,
+        total_comment_lines: 0,
+    },
 }