feat: provide facilities to perform rewrite-tracking for tree-diffs in plumbing.

This effectively pulls down a higher-level implementation in `gix` to the plumbing
level, to allow it to be used there as well.

Author: Byron

Cargo.lock

@@ -15,7 +15,7 @@ autotests = false
 [features]
 default = ["blob"]
 ## Enable diffing of blobs using imara-diff, which also allows for a generic rewrite tracking implementation.
-blob = ["dep:imara-diff", "dep:gix-filter", "dep:gix-worktree", "dep:gix-path", "dep:gix-fs", "dep:gix-command", "dep:gix-tempfile", "dep:gix-trace"]
+blob = ["dep:imara-diff", "dep:gix-filter", "dep:gix-worktree", "dep:gix-path", "dep:gix-fs", "dep:gix-command", "dep:gix-tempfile", "dep:gix-trace", "dep:gix-traverse"]
 ## Data structures implement `serde::Serialize` and `serde::Deserialize`.
 serde = ["dep:serde", "gix-hash/serde", "gix-object/serde"]
 ## Make it possible to compile to the `wasm32-unknown-unknown` target.
@@ -34,6 +34,7 @@ gix-path = { version = "^0.10.11", path = "../gix-path", optional = true }
 gix-fs = { version = "^0.11.3", path = "../gix-fs", optional = true }
 gix-tempfile = { version = "^14.0.0", path = "../gix-tempfile", optional = true }
 gix-trace = { version = "^0.1.10", path = "../gix-trace", optional = true }
+gix-traverse = { version = "^0.41.0", path = "../gix-traverse", optional = true }
 
 thiserror = "1.0.32"
 imara-diff = { version = "0.1.7", optional = true }

gix-diff/Cargo.toml

@@ -47,6 +47,12 @@ pub mod rewrites;
 pub mod tree;
 pub use tree::function::diff as tree;
 
+///
+#[cfg(feature = "blob")]
+pub mod tree_with_rewrites;
+#[cfg(feature = "blob")]
+pub use tree_with_rewrites::function::diff as tree_with_rewrites;
+
 ///
 #[cfg(feature = "blob")]
 pub mod blob;

gix-diff/src/lib.rs

@@ -0,0 +1,277 @@
+use bstr::BStr;
+use gix_object::TreeRefIter;
+
+use super::{Action, ChangeRef, Error, Options};
+use crate::rewrites;
+use crate::rewrites::tracker;
+
+/// Call `for_each` repeatedly with all changes that are needed to convert `lhs` to `rhs`.
+/// Provide a `resource_cache` to speed up obtaining blobs for similarity checks.
+/// `tree_diff_state` can be used to re-use tree-diff memory between calls.
+/// `objects` are used to lookup trees while performing the diff.
+/// Use `options` to further configure how the rename tracking is performed.
+///
+/// Reusing `resource_cache` between multiple invocations saves a lot of IOps as it avoids the creation
+/// of a temporary `resource_cache` that triggers reading or checking for multiple gitattribute files.
+/// Note that it's recommended to call [`clear_resource_cache()`](`crate::blob::Platform::clear_resource_cache()`)
+/// between the calls to avoid runaway memory usage, as the cache isn't limited.
+///
+/// Note that to do rename tracking like `git` does, one has to configure the `resource_cache` with
+/// a conversion pipeline that uses [`crate::blob::pipeline::Mode::ToGit`].
+///
+/// `rhs` or `lhs` can be empty to indicate deletion or addition of an entire tree.
+///
+/// Note that the rewrite outcome is only available if [rewrite-tracking was enabled](Options::rewrites).
+pub fn diff<E>(
+    lhs: TreeRefIter<'_>,
+    rhs: TreeRefIter<'_>,
+    resource_cache: &mut crate::blob::Platform,
+    tree_diff_state: &mut crate::tree::State,
+    objects: &impl gix_object::FindObjectOrHeader,
+    for_each: impl FnMut(ChangeRef<'_>) -> Result<Action, E>,
+    options: Options,
+) -> Result<Option<rewrites::Outcome>, Error>
+where
+    E: Into<Box<dyn std::error::Error + Sync + Send + 'static>>,
+{
+    let mut delegate = Delegate {
+        src_tree: lhs,
+        recorder: crate::tree::Recorder::default().track_location(options.location),
+        visit: for_each,
+        location: options.location,
+        objects,
+        tracked: options.rewrites.map(rewrites::Tracker::new),
+        err: None,
+    };
+    match crate::tree(lhs, rhs, tree_diff_state, objects, &mut delegate) {
+        Ok(()) => {
+            let outcome = delegate.process_tracked_changes(resource_cache)?;
+            match delegate.err {
+                Some(err) => Err(Error::ForEach(err.into())),
+                None => Ok(outcome),
+            }
+        }
+        Err(crate::tree::Error::Cancelled) => delegate
+            .err
+            .map_or(Err(Error::Diff(crate::tree::Error::Cancelled)), |err| {
+                Err(Error::ForEach(err.into()))
+            }),
+        Err(err) => Err(err.into()),
+    }
+}
+
+struct Delegate<'a, 'old, VisitFn, E, Objects> {
+    src_tree: TreeRefIter<'old>,
+    recorder: crate::tree::Recorder,
+    objects: &'a Objects,
+    visit: VisitFn,
+    tracked: Option<rewrites::Tracker<crate::tree::visit::Change>>,
+    location: Option<crate::tree::recorder::Location>,
+    err: Option<E>,
+}
+
+impl<VisitFn, E, Objects> Delegate<'_, '_, VisitFn, E, Objects>
+where
+    Objects: gix_object::FindObjectOrHeader,
+    VisitFn: for<'delegate> FnMut(ChangeRef<'_>) -> Result<Action, E>,
+    E: Into<Box<dyn std::error::Error + Sync + Send + 'static>>,
+{
+    /// Call `visit` on an attached version of `change`.
+    fn emit_change(
+        change: crate::tree::visit::Change,
+        location: &BStr,
+        visit: &mut VisitFn,
+        stored_err: &mut Option<E>,
+    ) -> crate::tree::visit::Action {
+        use crate::tree::visit::Change::*;
+        let change = match change {
+            Addition {
+                entry_mode,
+                oid,
+                relation,
+            } => ChangeRef::Addition {
+                location,
+                relation,
+                entry_mode,
+                id: oid,
+            },
+            Deletion {
+                entry_mode,
+                oid,
+                relation,
+            } => ChangeRef::Deletion {
+                entry_mode,
+                location,
+                relation,
+                id: oid,
+            },
+            Modification {
+                previous_entry_mode,
+                previous_oid,
+                entry_mode,
+                oid,
+            } => ChangeRef::Modification {
+                location,
+                previous_entry_mode,
+                entry_mode,
+                previous_id: previous_oid,
+                id: oid,
+            },
+        };
+        match visit(change) {
+            Ok(Action::Cancel) => crate::tree::visit::Action::Cancel,
+            Ok(Action::Continue) => crate::tree::visit::Action::Continue,
+            Err(err) => {
+                *stored_err = Some(err);
+                crate::tree::visit::Action::Cancel
+            }
+        }
+    }
+
+    fn process_tracked_changes(
+        &mut self,
+        diff_cache: &mut crate::blob::Platform,
+    ) -> Result<Option<rewrites::Outcome>, Error> {
+        use crate::rewrites::tracker::Change as _;
+        let tracked = match self.tracked.as_mut() {
+            Some(t) => t,
+            None => return Ok(None),
+        };
+
+        let outcome = tracked.emit(
+            |dest, source| match source {
+                Some(source) => {
+                    let (oid, mode) = dest.change.oid_and_entry_mode();
+                    let change = ChangeRef::Rewrite {
+                        source_location: source.location,
+                        source_entry_mode: source.entry_mode,
+                        source_id: source.id,
+                        source_relation: source.change.relation(),
+                        entry_mode: mode,
+                        id: oid.to_owned(),
+                        relation: dest.change.relation(),
+                        diff: source.diff,
+                        location: dest.location,
+                        copy: match source.kind {
+                            tracker::visit::SourceKind::Rename => false,
+                            tracker::visit::SourceKind::Copy => true,
+                        },
+                    };
+                    match (self.visit)(change) {
+                        Ok(Action::Cancel) => crate::tree::visit::Action::Cancel,
+                        Ok(Action::Continue) => crate::tree::visit::Action::Continue,
+                        Err(err) => {
+                            self.err = Some(err);
+                            crate::tree::visit::Action::Cancel
+                        }
+                    }
+                }
+                None => Self::emit_change(dest.change, dest.location, &mut self.visit, &mut self.err),
+            },
+            diff_cache,
+            self.objects,
+            |push| {
+                let mut delegate = tree_to_changes::Delegate::new(push, self.location);
+                let state = gix_traverse::tree::breadthfirst::State::default();
+                gix_traverse::tree::breadthfirst(self.src_tree, state, self.objects, &mut delegate)
+            },
+        )?;
+        Ok(Some(outcome))
+    }
+}
+
+impl<VisitFn, E, Objects> crate::tree::Visit for Delegate<'_, '_, VisitFn, E, Objects>
+where
+    Objects: gix_object::FindObjectOrHeader,
+    VisitFn: for<'delegate> FnMut(ChangeRef<'_>) -> Result<Action, E>,
+    E: Into<Box<dyn std::error::Error + Sync + Send + 'static>>,
+{
+    fn pop_front_tracked_path_and_set_current(&mut self) {
+        self.recorder.pop_front_tracked_path_and_set_current();
+    }
+
+    fn push_back_tracked_path_component(&mut self, component: &BStr) {
+        self.recorder.push_back_tracked_path_component(component);
+    }
+
+    fn push_path_component(&mut self, component: &BStr) {
+        self.recorder.push_path_component(component);
+    }
+
+    fn pop_path_component(&mut self) {
+        self.recorder.pop_path_component();
+    }
+
+    fn visit(&mut self, change: crate::tree::visit::Change) -> crate::tree::visit::Action {
+        match self.tracked.as_mut() {
+            Some(tracked) => tracked
+                .try_push_change(change, self.recorder.path())
+                .map_or(crate::tree::visit::Action::Continue, |change| {
+                    Self::emit_change(change, self.recorder.path(), &mut self.visit, &mut self.err)
+                }),
+            None => Self::emit_change(change, self.recorder.path(), &mut self.visit, &mut self.err),
+        }
+    }
+}
+
+mod tree_to_changes {
+    use crate::tree::visit::Change;
+    use gix_object::tree::EntryRef;
+
+    use bstr::BStr;
+
+    pub struct Delegate<'a> {
+        push: &'a mut dyn FnMut(Change, &BStr),
+        recorder: gix_traverse::tree::Recorder,
+    }
+
+    impl<'a> Delegate<'a> {
+        pub fn new(push: &'a mut dyn FnMut(Change, &BStr), location: Option<crate::tree::recorder::Location>) -> Self {
+            let location = location.map(|t| match t {
+                crate::tree::recorder::Location::FileName => gix_traverse::tree::recorder::Location::FileName,
+                crate::tree::recorder::Location::Path => gix_traverse::tree::recorder::Location::Path,
+            });
+            Self {
+                push,
+                recorder: gix_traverse::tree::Recorder::default().track_location(location),
+            }
+        }
+    }
+
+    impl gix_traverse::tree::Visit for Delegate<'_> {
+        fn pop_front_tracked_path_and_set_current(&mut self) {
+            self.recorder.pop_front_tracked_path_and_set_current();
+        }
+
+        fn push_back_tracked_path_component(&mut self, component: &BStr) {
+            self.recorder.push_back_tracked_path_component(component);
+        }
+
+        fn push_path_component(&mut self, component: &BStr) {
+            self.recorder.push_path_component(component);
+        }
+
+        fn pop_path_component(&mut self) {
+            self.recorder.pop_path_component();
+        }
+
+        fn visit_tree(&mut self, _entry: &EntryRef<'_>) -> gix_traverse::tree::visit::Action {
+            gix_traverse::tree::visit::Action::Continue
+        }
+
+        fn visit_nontree(&mut self, entry: &EntryRef<'_>) -> gix_traverse::tree::visit::Action {
+            if entry.mode.is_blob() {
+                (self.push)(
+                    Change::Modification {
+                        previous_entry_mode: entry.mode,
+                        previous_oid: gix_hash::ObjectId::null(entry.oid.kind()),
+                        entry_mode: entry.mode,
+                        oid: entry.oid.to_owned(),
+                    },
+                    self.recorder.path(),
+                );
+            }
+            gix_traverse::tree::visit::Action::Continue
+        }
+    }
+}

gix-diff/src/tree_with_rewrites/change.rs

@@ -0,0 +1,39 @@
+use crate::tree::recorder::Location;
+use crate::Rewrites;
+
+mod change;
+pub use change::{Change, ChangeRef};
+
+/// The error returned by [`tree_with_rewrites()`](super::tree_with_rewrites()).
+#[derive(Debug, thiserror::Error)]
+#[allow(missing_docs)]
+pub enum Error {
+    #[error(transparent)]
+    Diff(#[from] crate::tree::Error),
+    #[error("The user-provided callback failed")]
+    ForEach(#[source] Box<dyn std::error::Error + Send + Sync + 'static>),
+    #[error("Failure during rename tracking")]
+    RenameTracking(#[from] crate::rewrites::tracker::emit::Error),
+}
+
+/// Returned by the [`tree_with_rewrites()`](super::tree_with_rewrites()) function to control flow.
+#[derive(Default, Clone, Copy, PartialOrd, PartialEq, Ord, Eq, Hash)]
+pub enum Action {
+    /// Continue the traversal of changes.
+    #[default]
+    Continue,
+    /// Stop the traversal of changes and stop calling the function that returned it.
+    Cancel,
+}
+
+/// Options for use in [`tree_with_rewrites()`](super::tree_with_rewrites()).
+#[derive(Default, Clone, Debug)]
+pub struct Options {
+    /// Determine how locations of changes, i.e. their repository-relative path, should be tracked.
+    /// If `None`, locations will always be empty.
+    pub location: Option<Location>,
+    /// If not `None`, rename tracking will be performed accordingly.
+    pub rewrites: Option<Rewrites>,
+}
+
+pub(super) mod function;

gix-diff/src/tree_with_rewrites/function.rs

@@ -7,6 +7,7 @@ fn hex_to_id(hex: &str) -> gix_hash::ObjectId {
 mod blob;
 mod rewrites;
 mod tree;
+mod tree_with_rewrites;
 
 mod util {
     use std::collections::HashMap;