Skip big files in create_wd_tree()

Author: Byron

crates/gitbutler-branch-actions/src/branch_manager/branch_creation.rs

@@ -13,6 +13,7 @@ use gitbutler_error::error::Marker;
 use gitbutler_oplog::SnapshotExt;
 use gitbutler_oxidize::GixRepositoryExt;
 use gitbutler_project::access::WorktreeWritePermission;
+use gitbutler_project::AUTO_TRACK_LIMIT_BYTES;
 use gitbutler_reference::{Refname, RemoteRefname};
 use gitbutler_repo::logging::{LogUntil, RepositoryExt as _};
 use gitbutler_repo::{
@@ -305,7 +306,7 @@ impl BranchManager<'_> {
 
         // We don't support having two branches applied that conflict with each other
         {
-            let uncommited_changes_tree_id = repo.create_wd_tree()?.id();
+            let uncommited_changes_tree_id = repo.create_wd_tree(AUTO_TRACK_LIMIT_BYTES)?.id();
             let gix_repo = self.ctx.gix_repository_for_merging_non_persisting()?;
             let merges_cleanly = gix_repo
                 .merges_cleanly_compat(

crates/gitbutler-branch-actions/src/virtual.rs

@@ -26,6 +26,7 @@ use gitbutler_oxidize::{
     git2_signature_to_gix_signature, git2_to_gix_object_id, gix_to_git2_oid, GixRepositoryExt,
 };
 use gitbutler_project::access::WorktreeWritePermission;
+use gitbutler_project::AUTO_TRACK_LIMIT_BYTES;
 use gitbutler_reference::{normalize_branch_name, Refname, RemoteRefname};
 use gitbutler_repo::{
     logging::{LogUntil, RepositoryExt as _},
@@ -1089,7 +1090,7 @@ pub fn is_remote_branch_mergeable(
 
     let base_tree = find_base_tree(ctx.repo(), &branch_commit, &target_commit)?;
 
-    let wd_tree = ctx.repo().create_wd_tree()?;
+    let wd_tree = ctx.repo().create_wd_tree(AUTO_TRACK_LIMIT_BYTES)?;
 
     let branch_tree = branch_commit.tree().context("failed to find branch tree")?;
     let gix_repo_in_memory = ctx.gix_repository_for_merging()?.with_object_memory();

crates/gitbutler-edit-mode/src/lib.rs

@@ -20,6 +20,7 @@ use gitbutler_operating_modes::{
 };
 use gitbutler_oxidize::{git2_to_gix_object_id, gix_to_git2_index, GixRepositoryExt};
 use gitbutler_project::access::{WorktreeReadPermission, WorktreeWritePermission};
+use gitbutler_project::AUTO_TRACK_LIMIT_BYTES;
 use gitbutler_reference::{ReferenceName, Refname};
 use gitbutler_repo::{rebase::cherry_rebase, RepositoryExt};
 use gitbutler_repo::{signature, SignaturePurpose};
@@ -234,7 +235,7 @@ pub(crate) fn save_and_return_to_workspace(
     let parents = commit.parents().collect::<Vec<_>>();
 
     // Recommit commit
-    let tree = repository.create_wd_tree()?;
+    let tree = repository.create_wd_tree(AUTO_TRACK_LIMIT_BYTES)?;
 
     let (_, committer) = repository.signatures()?;
     let commit_headers = commit

crates/gitbutler-oplog/src/oplog.rs

@@ -20,7 +20,7 @@ use gitbutler_oxidize::{
 };
 use gitbutler_project::{
     access::{WorktreeReadPermission, WorktreeWritePermission},
-    Project,
+    Project, AUTO_TRACK_LIMIT_BYTES,
 };
 use gitbutler_repo::RepositoryExt;
 use gitbutler_repo::SignaturePurpose;
@@ -30,8 +30,6 @@ use gix::object::tree::diff::Change;
 use gix::prelude::ObjectIdExt;
 use tracing::instrument;
 
-const SNAPSHOT_FILE_LIMIT_BYTES: u64 = 32 * 1024 * 1024;
-
 /// The Oplog allows for crating snapshots of the current state of the project as well as restoring to a previous snapshot.
 /// Snapshots include the state of the working directory as well as all additional GitButler state (e.g. virtual branches, conflict state).
 /// The data is stored as git trees in the following shape:
@@ -312,7 +310,7 @@ impl OplogExt for Project {
         let old_wd_tree_id = tree_from_applied_vbranches(&gix_repo, commit.parent(0)?.id())?;
         let old_wd_tree = repo.find_tree(old_wd_tree_id)?;
 
-        repo.ignore_large_files_in_diffs(SNAPSHOT_FILE_LIMIT_BYTES)?;
+        repo.ignore_large_files_in_diffs(AUTO_TRACK_LIMIT_BYTES)?;
 
         let mut diff_opts = git2::DiffOptions::new();
         diff_opts
@@ -602,7 +600,7 @@ fn restore_snapshot(
     let workdir_tree_id = tree_from_applied_vbranches(&gix_repo, snapshot_commit_id)?;
     let workdir_tree = repo.find_tree(workdir_tree_id)?;
 
-    repo.ignore_large_files_in_diffs(SNAPSHOT_FILE_LIMIT_BYTES)?;
+    repo.ignore_large_files_in_diffs(AUTO_TRACK_LIMIT_BYTES)?;
 
     // Define the checkout builder
     let mut checkout_builder = git2::build::CheckoutBuilder::new();
@@ -739,7 +737,7 @@ fn lines_since_snapshot(project: &Project, repo: &git2::Repository) -> Result<us
     // This looks at the diff between the tree of the currently selected as 'default' branch (where new changes go)
     // and that same tree in the last snapshot. For some reason, comparing workdir to the workdir subree from
     // the snapshot simply does not give us what we need here, so instead using tree to tree comparison.
-    repo.ignore_large_files_in_diffs(SNAPSHOT_FILE_LIMIT_BYTES)?;
+    repo.ignore_large_files_in_diffs(AUTO_TRACK_LIMIT_BYTES)?;
 
     let oplog_state = OplogHandle::new(&project.gb_dir());
     let Some(oplog_commit_id) = oplog_state.oplog_head()? else {

crates/gitbutler-project/src/lib.rs

@@ -18,3 +18,6 @@ pub fn configure_git2() {
     // These settings are only changed from `main` of applications.
     git2::opts::strict_object_creation(false);
 }
+
+/// The maximum size of files to automatically start tracking, i.e. untracked files we pick up for tree-creation.
+pub const AUTO_TRACK_LIMIT_BYTES: u64 = 32 * 1024 * 1024;

crates/gitbutler-repo/src/repository_ext.rs

@@ -42,8 +42,15 @@ pub trait RepositoryExt {
     fn sign_buffer(&self, buffer: &[u8]) -> Result<BString>;
     fn checkout_tree_builder<'a>(&'a self, tree: &'a git2::Tree<'a>) -> CheckoutTreeBuidler<'a>;
     fn maybe_find_branch_by_refname(&self, name: &Refname) -> Result<Option<git2::Branch>>;
-    /// Based on the index, add all data similar to `git add .` and create a tree from it, which is returned.
-    fn create_wd_tree(&self) -> Result<Tree>;
+    /// Add all untracked and modified files in the worktree to
+    /// the object database, and create a tree from it.
+    ///
+    /// Use `untracked_limit_in_bytes` to control the maximum file size for untracked files
+    /// before we stop tracking them automatically. Set it to 0 to disable the limit.
+    ///
+    /// It should also be noted that this will fail if run on an empty branch
+    /// or if the HEAD branch has no commits.
+    fn create_wd_tree(&self, untracked_limit_in_bytes: u64) -> Result<Tree>;
 
     /// Returns the `gitbutler/workspace` branch if the head currently points to it, or fail otherwise.
     /// Use it before any modification to the repository, or extra defensively each time the
@@ -105,15 +112,8 @@ impl RepositoryExt for git2::Repository {
         Ok(branch)
     }
 
-    /// Add all untracked and modified files in the worktree to
-    /// the object database, and create a tree from it.
-    ///
-    /// Note that right now, it doesn't skip big files.
-    ///
-    /// It should also be noted that this will fail if run on an empty branch
-    /// or if the HEAD branch has no commits.
     #[instrument(level = tracing::Level::DEBUG, skip(self), err(Debug))]
-    fn create_wd_tree(&self) -> Result<Tree> {
+    fn create_wd_tree(&self, untracked_limit_in_bytes: u64) -> Result<Tree> {
         use bstr::ByteSlice;
         use gix::dir::walk::EmissionMode;
         use gix::status;