Refactor: drop gitoxide from josh-core (#1337)

While gitoxide provides a well designed set of APIs, it's more oriented
towards implementing a git client than a server, and it lacks some
high-level APIs like treebuilder etc. Having it alongside the regular
git2 implementation works but creates additional I/O pressure which has
performance implications in scenarios when a lot of transactions are
opened (graphql). It would still make sense to integrate gitoxide in the
future as APIs improve, but only as a complete replacement of git2 and
not side by side

* Remove gix from josh-core
* I kept gix in proxy since it still provides nice Rust APIs for repo
init and config manipulation, however...
* ...I updated the version of gix to latest and disabled all extra
features, which reduced dependencies a lot

Author: vlad-ivanov-name

Cargo.lock

@@ -19,7 +19,7 @@ codegen-units = 1
 defer = "0.1.0"
 env_logger = "0.10.0"
 futures = "0.3.28"
-gix = "0.54.1"
+gix = { version = "0.63.0", default-features = false }
 hyper-reverse-proxy = "0.5.1"
 lazy_static = "1.4.0"
 libc = "0.2.148"

Cargo.toml

@@ -34,7 +34,6 @@ sled = "0.34.7"
 strfmt = "0.2.4"
 toml = { workspace = true }
 tracing = { workspace = true }
-gix = { workspace = true }
 juniper = { workspace = true }
 form_urlencoded = "1.2.1"
 

josh-core/Cargo.toml

@@ -68,7 +68,6 @@ struct Transaction2 {
 pub struct Transaction {
     t2: std::cell::RefCell<Transaction2>,
     repo: git2::Repository,
-    oxide_repo: gix::Repository,
     ref_prefix: String,
 }
 
@@ -83,7 +82,6 @@ impl Transaction {
                 git2::RepositoryOpenFlags::NO_SEARCH,
                 &[] as &[&std::ffi::OsStr],
             )?,
-            gix::ThreadSafeRepository::open(path)?.to_thread_local(),
             ref_prefix,
         ))
     }
@@ -95,11 +93,7 @@ impl Transaction {
             load(&path)?
         };
 
-        Ok(Transaction::new(
-            repo,
-            gix::ThreadSafeRepository::open(path)?.to_thread_local(),
-            None,
-        ))
+        Ok(Transaction::new(repo, None))
     }
 
     pub fn status(&self, _msg: &str) {
@@ -108,11 +102,7 @@ impl Transaction {
         /* t2.out.flush().ok(); */
     }
 
-    fn new(
-        repo: git2::Repository,
-        oxide_repo: gix::Repository,
-        ref_prefix: Option<&str>,
-    ) -> Transaction {
+    fn new(repo: git2::Repository, ref_prefix: Option<&str>) -> Transaction {
         log::debug!("new transaction");
         let path_tree = DB
             .lock()
@@ -151,7 +141,6 @@ impl Transaction {
                 walks: 0,
             }),
             repo,
-            oxide_repo,
             ref_prefix: ref_prefix.unwrap_or("").to_string(),
         }
     }
@@ -164,10 +153,6 @@ impl Transaction {
         &self.repo
     }
 
-    pub fn oxide_repo(&self) -> &gix::Repository {
-        &self.oxide_repo
-    }
-
     pub fn refname(&self, r: &str) -> String {
         format!("{}{}", self.ref_prefix, r)
     }

josh-core/src/cache.rs

@@ -1,4 +1,5 @@
 use super::*;
+use std::collections::HashMap;
 
 pub fn walk2(
     filter: filter::Filter,

josh-core/src/compat.rs

@@ -1,8 +1,7 @@
-use git2;
-
-use super::*;
+use crate::*;
+use itertools::Itertools;
 use std::collections::{BTreeSet, HashMap};
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use tracing::{info, span, Level};
 
 pub type KnownViews = HashMap<String, (git2::Oid, BTreeSet<String>)>;
@@ -13,28 +12,33 @@ lazy_static! {
 }
 
 pub fn list_refs(
-    repo: &gix::Repository,
+    repo: &git2::Repository,
     upstream_repo: &str,
-) -> JoshResult<Vec<(PathBuf, gix::ObjectId)>> {
+) -> JoshResult<Vec<(String, git2::Oid)>> {
     let mut refs = vec![];
 
     let prefix = ["refs", "josh", "upstream", &to_ns(upstream_repo)]
         .iter()
-        .collect::<PathBuf>();
+        .join("/");
 
-    for group in [
-        prefix.join("refs").join("heads"),
-        prefix.join("refs").join("tags"),
+    for glob in [
+        format!("{}/refs/heads/*", &prefix),
+        format!("{}/refs/tags/*", &prefix),
     ]
     .iter()
     {
-        for reference in repo.references()?.prefixed(group)? {
+        for reference in repo.references_glob(&glob)? {
             let reference =
                 reference.map_err(|e| josh_error(&format!("unable to obtain reference: {}", e)))?;
 
-            let name = reference.name().to_path().strip_prefix(&prefix)?;
-            let oid = gix::ObjectId::from(reference.target().id());
-            refs.push((name.to_owned(), oid))
+            if let (Some(name), Some(target)) = (reference.name(), reference.target()) {
+                let name = name
+                    .strip_prefix(&prefix)
+                    .and_then(|name| name.strip_prefix('/'))
+                    .ok_or_else(|| josh_error("bug: unexpected result of a glob"))?;
+
+                refs.push((name.to_owned(), target))
+            }
         }
     }
 

josh-core/src/history.rs

@@ -34,10 +34,7 @@ extern crate pest_derive;
 #[macro_use]
 extern crate serde_json;
 
-use std::collections::HashMap;
-
 pub mod cache;
-pub mod compat;
 pub mod filter;
 pub mod graphql;
 pub mod history;
@@ -367,14 +364,14 @@ pub fn normalize_path(path: &std::path::Path) -> std::path::PathBuf {
     ret
 }
 
-type Users = HashMap<String, User>;
+type Users = std::collections::HashMap<String, User>;
 
 #[derive(Debug, serde::Deserialize)]
 struct User {
     pub groups: toml::value::Array,
 }
 
-type Groups = HashMap<String, HashMap<String, Group>>;
+type Groups = std::collections::HashMap<String, std::collections::HashMap<String, Group>>;
 #[derive(Debug, serde::Deserialize)]
 struct Group {
     pub whitelist: String,

josh-core/src/housekeeping.rs

@@ -16,7 +16,6 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
 defer = { workspace = true }
-gix = { workspace = true }
 clap = { workspace = true }
 rs_tracing = { workspace = true }
 juniper = { workspace = true }

josh-core/src/lib.rs

@@ -17,7 +17,6 @@ use hyper::service::{make_service_fn, service_fn};
 use hyper::{Request, Response, Server, StatusCode};
 
 use indoc::formatdoc;
-use josh::compat::GitOxideCompatExt;
 use josh::{josh_error, JoshError, JoshResult};
 use josh_rpc::calls::RequestedCommand;
 use serde::Serialize;
@@ -433,7 +432,7 @@ async fn do_filter(
             )),
         )?;
 
-        let resolve_ref = |ref_value: &str| -> JoshResult<gix::ObjectId> {
+        let resolve_ref = |ref_value: &str| -> JoshResult<git2::Oid> {
             let josh_name = [
                 "refs",
                 "josh",
@@ -445,42 +444,39 @@ async fn do_filter(
             .collect::<PathBuf>();
 
             Ok(transaction
-                .oxide_repo()
+                .repo()
                 .find_reference(josh_name.to_str().unwrap())
                 .map_err(|e| josh_error(&format!("Could not find ref: {}", e)))?
-                .id()
-                .into())
+                .target()
+                .ok_or(josh_error("Could not resolve ref"))?)
         };
 
         let (refs_list, head_ref) = match &head_ref {
             HeadRef::Explicit(ref_value)
                 if ref_value.starts_with("refs/") || ref_value == "HEAD" =>
             {
                 let object = resolve_ref(&ref_value)?;
-                let list = vec![(PathBuf::from(ref_value), object)];
+                let list = vec![(ref_value.clone(), object)];
 
                 (list, ref_value.clone())
             }
             HeadRef::Explicit(ref_value) => {
                 // When it's not something starting with refs/ or HEAD, it's
                 // probably sha1
-                let list = vec![(
-                    PathBuf::from(ref_value),
-                    gix::ObjectId::from_str(&ref_value)?,
-                )];
-                let synthetic_ref = format!("refs/heads/_{}", ref_value);
+                let list = vec![(ref_value.to_string(), git2::Oid::from_str(&ref_value)?)];
 
+                let synthetic_ref = format!("refs/heads/_{}", ref_value);
                 (list, synthetic_ref)
             }
             HeadRef::Implicit => {
                 // When user did not explicitly request a ref to filter,
                 // start with a list of all existing refs
                 let mut list =
-                    josh::housekeeping::list_refs(transaction.oxide_repo(), &meta.config.repo)?;
+                    josh::housekeeping::list_refs(transaction.repo(), &meta.config.repo)?;
 
                 let head_ref = head_ref.get().to_string();
                 if let Ok(object) = resolve_ref(&head_ref) {
-                    list.push((PathBuf::from(&head_ref), object));
+                    list.push((head_ref.clone(), object));
                 }
 
                 (list, head_ref)
@@ -497,11 +493,6 @@ async fn do_filter(
             head_ref
         };
 
-        let refs_list = refs_list
-            .iter()
-            .map(|(reference, oid)| (reference.to_str().unwrap().to_string(), oid.to_git2()))
-            .collect::<Vec<_>>();
-
         let t2 = josh::cache::Transaction::open(&repo_path.join("overlay"), None)?;
         t2.repo()
             .odb()?