very first experimental support for multi-pack index verification (#279)

Author: Byron

git-odb/src/cache.rs

@@ -1,5 +1,8 @@
-use std::ops::{Deref, DerefMut};
-use std::{cell::RefCell, sync::Arc};
+use std::{
+    cell::RefCell,
+    ops::{Deref, DerefMut},
+    sync::Arc,
+};
 
 use crate::Cache;
 

git-odb/src/store_impls/dynamic/handle.rs

@@ -71,8 +71,7 @@ pub(crate) mod index_lookup {
 
     use git_hash::oid;
 
-    use crate::store::handle::IntraPackLookup;
-    use crate::store::{handle, types};
+    use crate::store::{handle, handle::IntraPackLookup, types};
 
     pub(crate) struct Outcome<'a> {
         pub object_index: handle::IndexForObjectInPack,

git-pack/src/index/access.rs

@@ -2,10 +2,9 @@ use std::mem::size_of;
 
 use byteorder::{BigEndian, ByteOrder};
 
-use crate::index::EntryIndex;
 use crate::{
     data,
-    index::{self, FAN_LEN},
+    index::{self, EntryIndex, FAN_LEN},
 };
 
 const N32_SIZE: usize = size_of::<u32>();

git-pack/src/multi_index/access.rs

@@ -2,10 +2,9 @@ use std::path::{Path, PathBuf};
 
 use byteorder::{BigEndian, ByteOrder};
 
-use crate::multi_index::EntryIndex;
 use crate::{
     data,
-    multi_index::{File, PackIndex, Version},
+    multi_index::{EntryIndex, File, PackIndex, Version},
 };
 
 /// Represents an entry within a multi index file, effectively mapping object [`IDs`][git_hash::ObjectId] to pack data

git-pack/src/multi_index/verify.rs

@@ -1,7 +1,9 @@
-use crate::multi_index::File;
-use git_features::progress::Progress;
 use std::sync::atomic::AtomicBool;
 
+use git_features::progress::Progress;
+
+use crate::multi_index::File;
+
 ///
 pub mod integrity {
     /// Returned by [`multi_index::File::verify_integrity()`][crate::multi_index::File::verify_integrity()].

git-pack/src/verify.rs

@@ -1,6 +1,6 @@
+use std::{path::Path, sync::atomic::AtomicBool};
+
 use git_features::progress::Progress;
-use std::path::Path;
-use std::sync::atomic::AtomicBool;
 
 ///
 pub mod checksum {

git-pack/tests/pack/index.rs

@@ -1,9 +1,10 @@
 mod file {
     const SHA1_SIZE: usize = git_hash::Kind::Sha1.len_in_bytes();
 
+    use std::sync::atomic::AtomicBool;
+
     use git_object::{self as object};
     use git_odb::pack;
-    use std::sync::atomic::AtomicBool;
 
     use crate::{
         fixture_path, hex_to_id,

git-pack/tests/pack/multi_index.rs

@@ -1,6 +1,6 @@
-use git_pack::multi_index::File;
 use std::path::PathBuf;
 
+use git_pack::multi_index::File;
 use git_testtools::hex_to_id;
 
 fn multi_index() -> (File, PathBuf) {
@@ -52,10 +52,12 @@ fn access() {
 }
 
 mod verify {
-    use crate::pack::multi_index::multi_index;
+    use std::sync::atomic::AtomicBool;
+
     use common_macros::b_tree_map;
     use git_features::progress;
-    use std::sync::atomic::AtomicBool;
+
+    use crate::pack::multi_index::multi_index;
 
     #[test]
     fn checksum() {

gitoxide-core/src/pack/verify.rs

@@ -2,10 +2,10 @@ use std::{io, path::Path, str::FromStr, sync::atomic::AtomicBool};
 
 use anyhow::{anyhow, Context as AnyhowContext, Result};
 use bytesize::ByteSize;
+use git_pack_for_configuration_only::index::traverse::Outcome;
 use git_repository as git;
 use git_repository::{
     easy::object,
-    hash::ObjectId,
     odb,
     odb::{pack, pack::index},
     Progress,
@@ -113,19 +113,27 @@ pub fn pack_or_pack_index<W1, W2>(
         algorithm,
         should_interrupt,
     }: Context<'_, W1, W2>,
-) -> Result<(ObjectId, Option<index::traverse::Outcome>)>
+) -> Result<()>
 where
     W1: io::Write,
     W2: io::Write,
 {
     let path = path.as_ref();
-    let ext = path.extension().and_then(|ext| ext.to_str()).ok_or_else(|| {
-        anyhow!(
-            "Cannot determine data type on path without extension '{}', expecting default extensions 'idx' and 'pack'",
-            path.display()
-        )
-    })?;
+    let ext = path.extension().and_then(|ext| ext.to_str()).unwrap_or("");
     let object_hash = git::hash::Kind::Sha1; // TODO: make it configurable via Context/CLI
+    const CACHE_SIZE: usize = 64;
+    let cache = || -> EitherCache<CACHE_SIZE> {
+        if matches!(algorithm, Algorithm::LessMemory) {
+            if output_statistics.is_some() {
+                // turn off acceleration as we need to see entire chains all the time
+                EitherCache::Left(pack::cache::Never)
+            } else {
+                EitherCache::Right(pack::cache::lru::StaticLinkedList::<CACHE_SIZE>::default())
+            }
+        } else {
+            EitherCache::Left(pack::cache::Never)
+        }
+    };
     let res = match ext {
         "pack" => {
             let pack = odb::pack::data::File::at(path, object_hash).with_context(|| "Could not open pack file")?;
@@ -148,19 +156,6 @@ where
                     e
                 })
                 .ok();
-            const CACHE_SIZE: usize = 64;
-            let cache = || -> EitherCache<CACHE_SIZE> {
-                if matches!(algorithm, Algorithm::LessMemory) {
-                    if output_statistics.is_some() {
-                        // turn off acceleration as we need to see entire chains all the time
-                        EitherCache::Left(pack::cache::Never)
-                    } else {
-                        EitherCache::Right(pack::cache::lru::StaticLinkedList::<CACHE_SIZE>::default())
-                    }
-                } else {
-                    EitherCache::Left(pack::cache::Never)
-                }
-            };
 
             idx.verify_integrity(
                 pack.as_ref().map(|p| git::odb::pack::index::verify::PackContext {
@@ -176,18 +171,42 @@ where
             .map(|o| (o.actual_index_checksum, o.pack_traverse_outcome))
             .with_context(|| "Verification failure")?
         }
+        "" => {
+            match path.file_name() {
+                Some(file_name) if file_name == "multi-pack-index" => {
+                    let multi_index = git::odb::pack::multi_index::File::at(path)?;
+                    let res = multi_index.verify_integrity(mode, algorithm.into(), cache, thread_limit, progress, should_interrupt)?;
+                    for stats in res.pack_traverse_outcomes {
+                        output_outcome(&mut out, output_statistics, &stats)?;
+                    }
+                    return Ok(())
+                },
+                _ => return Err(anyhow!(
+                        "Cannot determine data type on path without extension '{}', expecting default extensions 'idx' and 'pack'",
+                        path.display()
+                    ))
+            }
+        }
         ext => return Err(anyhow!("Unknown extension {:?}, expecting 'idx' or 'pack'", ext)),
     };
     if let Some(stats) = res.1.as_ref() {
-        #[cfg_attr(not(feature = "serde1"), allow(clippy::single_match))]
-        match output_statistics {
-            Some(OutputFormat::Human) => drop(print_statistics(&mut out, stats)),
-            #[cfg(feature = "serde1")]
-            Some(OutputFormat::Json) => serde_json::to_writer_pretty(out, stats)?,
-            _ => {}
-        };
+        output_outcome(&mut out, output_statistics, stats)?;
     }
-    Ok(res)
+    Ok(())
+}
+
+fn output_outcome<W1>(mut out: &mut W1, output_statistics: Option<OutputFormat>, stats: &Outcome) -> anyhow::Result<()>
+where
+    W1: io::Write,
+{
+    #[cfg_attr(not(feature = "serde1"), allow(clippy::single_match))]
+    match output_statistics {
+        Some(OutputFormat::Human) => drop(print_statistics(&mut out, stats)),
+        #[cfg(feature = "serde1")]
+        Some(OutputFormat::Json) => serde_json::to_writer_pretty(out, stats)?,
+        _ => {}
+    };
+    Ok(())
 }
 
 fn print_statistics(out: &mut impl io::Write, stats: &index::traverse::Outcome) -> io::Result<()> {