Implementation of shallow libgit2 fetches behind an unstable flag

Author: emmaworley

src/cargo/core/features.rs

@@ -770,6 +770,7 @@ unstable_cli_options!(
     dual_proc_macros: bool = ("Build proc-macros for both the host and the target"),
     features: Option<Vec<String>>,
     gc: bool = ("Track cache usage and \"garbage collect\" unused files"),
+    git: Option<GitFeatures> = ("Enable support for shallow git fetch operations"),
     gitoxide: Option<GitoxideFeatures> = ("Use gitoxide for the given git interactions, or all of them if no argument is given"),
     host_config: bool = ("Enable the `[host]` section in the .cargo/config.toml file"),
     lints: bool = ("Pass `[lints]` to the linting tools"),
@@ -875,13 +876,47 @@ where
 }
 
 #[derive(Debug, Copy, Clone, Default, Deserialize)]
-pub struct GitoxideFeatures {
-    /// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
-    pub fetch: bool,
+pub struct GitFeatures {
     /// When cloning the index, perform a shallow clone. Maintain shallowness upon subsequent fetches.
     pub shallow_index: bool,
     /// When cloning git dependencies, perform a shallow clone and maintain shallowness on subsequent fetches.
     pub shallow_deps: bool,
+}
+
+impl GitFeatures {
+    fn all() -> Self {
+        GitFeatures {
+            shallow_index: true,
+            shallow_deps: true,
+        }
+    }
+}
+
+fn parse_git(it: impl Iterator<Item = impl AsRef<str>>) -> CargoResult<Option<GitFeatures>> {
+    let mut out = GitFeatures::default();
+    let GitFeatures {
+        shallow_index,
+        shallow_deps,
+    } = &mut out;
+
+    for e in it {
+        match e.as_ref() {
+            "shallow-index" => *shallow_index = true,
+            "shallow-deps" => *shallow_deps = true,
+            _ => {
+                bail!(
+                    "unstable 'git' only takes 'shallow-index' and 'shallow-deps' as valid inputs"
+                )
+            }
+        }
+    }
+    Ok(Some(out))
+}
+
+#[derive(Debug, Copy, Clone, Default, Deserialize)]
+pub struct GitoxideFeatures {
+    /// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
+    pub fetch: bool,
     /// Checkout git dependencies using `gitoxide` (submodules are still handled by git2 ATM, and filters
     /// like linefeed conversions are unsupported).
     pub checkout: bool,
@@ -895,9 +930,7 @@ impl GitoxideFeatures {
     fn all() -> Self {
         GitoxideFeatures {
             fetch: true,
-            shallow_index: true,
             checkout: true,
-            shallow_deps: true,
             internal_use_git2: false,
         }
     }
@@ -907,9 +940,7 @@ impl GitoxideFeatures {
     fn safe() -> Self {
         GitoxideFeatures {
             fetch: true,
-            shallow_index: false,
             checkout: true,
-            shallow_deps: false,
             internal_use_git2: false,
         }
     }
@@ -921,21 +952,17 @@ fn parse_gitoxide(
     let mut out = GitoxideFeatures::default();
     let GitoxideFeatures {
         fetch,
-        shallow_index,
         checkout,
-        shallow_deps,
         internal_use_git2,
     } = &mut out;
 
     for e in it {
         match e.as_ref() {
             "fetch" => *fetch = true,
-            "shallow-index" => *shallow_index = true,
-            "shallow-deps" => *shallow_deps = true,
             "checkout" => *checkout = true,
             "internal-use-git2" => *internal_use_git2 = true,
             _ => {
-                bail!("unstable 'gitoxide' only takes `fetch`, 'shallow-index', 'shallow-deps' and 'checkout' as valid inputs")
+                bail!("unstable 'gitoxide' only takes `fetch` and 'checkout' as valid inputs")
             }
         }
     }
@@ -1109,6 +1136,12 @@ impl CliUnstable {
             "doctest-xcompile" => self.doctest_xcompile = parse_empty(k, v)?,
             "dual-proc-macros" => self.dual_proc_macros = parse_empty(k, v)?,
             "gc" => self.gc = parse_empty(k, v)?,
+            "git" => {
+                self.git = v.map_or_else(
+                    || Ok(Some(GitFeatures::all())),
+                    |v| parse_git(v.split(',')),
+                )?
+            }
             "gitoxide" => {
                 self.gitoxide = v.map_or_else(
                     || Ok(Some(GitoxideFeatures::all())),

src/cargo/sources/git/mod.rs

@@ -16,7 +16,7 @@ mod utils;
 
 /// For `-Zgitoxide` integration.
 pub mod fetch {
-    use crate::core::features::GitoxideFeatures;
+    use crate::core::features::GitFeatures;
     use crate::Config;
 
     /// The kind remote repository to fetch.
@@ -37,18 +37,19 @@ pub mod fetch {
             repo_is_shallow: bool,
             config: &Config,
         ) -> gix::remote::fetch::Shallow {
-            let has_feature = |cb: &dyn Fn(GitoxideFeatures) -> bool| {
+            let has_feature = |cb: &dyn Fn(GitFeatures) -> bool| {
                 config
                     .cli_unstable()
-                    .gitoxide
+                    .git
                     .map_or(false, |features| cb(features))
             };
 
             // maintain shallow-ness and keep downloading single commits, or see if we can do shallow clones
             if !repo_is_shallow {
                 match self {
-                    RemoteKind::GitDependency if has_feature(&|git| git.shallow_deps) => {}
-                    RemoteKind::Registry if has_feature(&|git| git.shallow_index) => {}
+                    RemoteKind::GitDependency if has_feature(&|features| features.shallow_deps) => {
+                    }
+                    RemoteKind::Registry if has_feature(&|features| features.shallow_index) => {}
                     _ => return gix::remote::fetch::Shallow::NoChange,
                 }
             };

src/cargo/sources/git/oxide.rs

@@ -29,10 +29,9 @@ pub fn with_retry_and_progress(
 ) -> CargoResult<()> {
     std::thread::scope(|s| {
         let mut progress_bar = Progress::new("Fetch", config);
-        let is_shallow = config
-            .cli_unstable()
-            .gitoxide
-            .map_or(false, |gix| gix.shallow_deps || gix.shallow_index);
+        let is_shallow = config.cli_unstable().git.map_or(false, |features| {
+            features.shallow_deps || features.shallow_index
+        });
         network::retry::with_retry(config, || {
             let progress_root: Arc<gix::progress::tree::Root> =
                 gix::progress::tree::root::Options {

src/cargo/sources/git/source.rs

@@ -108,8 +108,8 @@ impl<'cfg> GitSource<'cfg> {
             &source_id,
             config
                 .cli_unstable()
-                .gitoxide
-                .map_or(false, |gix| gix.fetch && gix.shallow_deps),
+                .git
+                .map_or(false, |features| features.shallow_deps),
         );
 
         let source = GitSource {

src/cargo/sources/git/utils.rs

@@ -1117,6 +1117,9 @@ pub fn fetch(
             if tags {
                 opts.download_tags(git2::AutotagOption::All);
             }
+            if let gix::remote::fetch::Shallow::DepthAtRemote(depth) = shallow {
+                opts.depth(0i32.saturating_add_unsigned(depth.get()));
+            }
             // The `fetch` operation here may fail spuriously due to a corrupt
             // repository. It could also fail, however, for a whole slew of other
             // reasons (aka network related reasons). We want Cargo to automatically

src/cargo/sources/registry/mod.rs

@@ -476,8 +476,8 @@ impl<'cfg> RegistrySource<'cfg> {
             source_id,
             config
                 .cli_unstable()
-                .gitoxide
-                .map_or(false, |gix| gix.fetch && gix.shallow_index)
+                .git
+                .map_or(false, |features| features.shallow_index)
                 && !source_id.is_sparse(),
         );
         let ops = if source_id.is_sparse() {

src/doc/src/reference/unstable.md

@@ -1160,19 +1160,32 @@ to run with `gitoxide` with the `-Zgitoxide=operation[,operationN]` syntax.
 Valid operations are the following:
 
 * `fetch` - All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
+* `checkout` *(planned)* - checkout the worktree, with support for filters and submodules.
+
+## git
+
+* Tracking Issue: [TODO]()
+
+With the 'git' unstable feature, both `gitoxide` and `git2` will perform shallow fetches of the crate
+index and git dependencies.
+
+While `-Zgit` enables all currently implemented features, one can individually select when to perform
+shallow fetches with the `-Zgit=operation[,operationN]` syntax.
+
+Valid operations are the following:
+
 * `shallow-index` - perform a shallow clone of the index.
 * `shallow-deps` - perform a shallow clone of git dependencies.
-* `checkout` *(planned)* - checkout the worktree, with support for filters and submodules.
 
 **Details on shallow clones**
 
-* To enable shallow clones, add `-Zgitoxide=fetch,shallow_deps` for fetching git dependencies or `-Zgitoxide=fetch,shallow_index` for fetching registry index.
+* To enable shallow clones, add `-Zgit=shallow-deps` for fetching git dependencies or `-Zgit=shallow-index` for fetching registry index.
 * Shallow-cloned and shallow-checked-out git repositories reside at their own `-shallow` suffixed directories, i.e,
   - `~/.cargo/registry/index/*-shallow`
   - `~/.cargo/git/db/*-shallow`
   - `~/.cargo/git/checkouts/*-shallow`
 * When the unstable feature is on, fetching/cloning a git repository is always a shallow fetch. This roughly equals to `git fetch --depth 1` everywhere.
-* Even with the presence of `Cargo.lock` or specifying a commit `{ rev = "…" }`, gitoxide is still smart enough to shallow fetch without unshallowing the existing repository.
+* Even with the presence of `Cargo.lock` or specifying a commit `{ rev = "…" }`, gitoxide is still smart enough to shallow fetch without unshallowing the existing repository. TODO: does this hold for libgit2?
 
 ## script