Merge pull request #284 from joehasson/feat/include-directory-inodes-in-size-calculations

Fix: Directories themselves do not contribute to size like they do in du

Author: Byron

src/aggregate.rs

@@ -52,8 +52,7 @@ pub fn aggregate(
                 Ok(entry) => {
                     let file_size = match entry.client_state {
                         Some(Ok(ref m))
-                            if !m.is_dir()
-                                && (walk_options.count_hard_links || inodes.add(m))
+                            if (walk_options.count_hard_links || inodes.add(m))
                                 && (walk_options.cross_filesystems
                                     || crossdev::is_same_device(device_id, m)) =>
                         {

src/interactive/app/tests/utils.rs

@@ -234,12 +234,12 @@ pub fn sample_01_tree() -> Tree {
     {
         let mut add_node = make_add_node(&mut tree);
         #[cfg(not(windows))]
-        let root_size = 1259070;
+        let root_size = 1275454;
         #[cfg(windows)]
         let root_size = 1259069;
-        let rn = add_node("", root_size, 10, None);
+        let rn = add_node("", root_size, 14, None);
         {
-            let sn = add_node(&fixture_str("sample-01"), root_size, 10, Some(rn));
+            let sn = add_node(&fixture_str("sample-01"), root_size, 14, Some(rn));
             {
                 add_node(".hidden.666", 666, 0, Some(sn));
                 add_node("a", 256, 0, Some(sn));
@@ -248,16 +248,26 @@ pub fn sample_01_tree() -> Tree {
                 add_node("c.lnk", 1, 0, Some(sn));
                 #[cfg(windows)]
                 add_node("c.lnk", 0, 0, Some(sn));
-                let dn = add_node("dir", 1258024, 5, Some(sn));
+
+                #[cfg(not(windows))]
+                let dn = add_node("dir", 1270312, 8, Some(sn));
+                #[cfg(windows)]
+                let dn = add_node("dir", 1258024, 8, Some(sn));
                 {
                     add_node("1000bytes", 1000, 0, Some(dn));
                     add_node("dir-a.1mb", 1_000_000, 0, Some(dn));
                     add_node("dir-a.kb", 1024, 0, Some(dn));
-                    let en = add_node("empty-dir", 0, 1, Some(dn));
+                    #[cfg(not(windows))]
+                    let en = add_node("empty-dir", 4096, 2, Some(dn));
+                    #[cfg(windows)]
+                    let en = add_node("empty-dir", 0, 2, Some(dn));
                     {
                         add_node(".gitkeep", 0, 0, Some(en));
                     }
-                    let sub = add_node("sub", 256_000, 1, Some(dn));
+                    #[cfg(not(windows))]
+                    let sub = add_node("sub", 260_096, 2, Some(dn));
+                    #[cfg(windows)]
+                    let sub = add_node("sub", 256_000, 2, Some(dn));
                     {
                         add_node("dir-sub-a.256kb", 256_000, 0, Some(sub));
                     }
@@ -274,8 +284,11 @@ pub fn sample_02_tree(use_native_separator: bool) -> (Tree, TreeIndex) {
     let root_index: TreeIndex;
     {
         let mut add_node = make_add_node(&mut tree);
+        #[cfg(not(windows))]
+        let root_size = 17924;
+        #[cfg(windows)]
         let root_size = 1540;
-        root_index = add_node("", root_size, 6, None);
+        root_index = add_node("", root_size, 10, None);
         {
             let sn = add_node(
                 format!(
@@ -288,21 +301,30 @@ pub fn sample_02_tree(use_native_separator: bool) -> (Tree, TreeIndex) {
                 )
                 .as_str(),
                 root_size,
-                6,
+                10,
                 Some(root_index),
             );
             {
                 add_node("a", 256, 0, Some(sn));
                 add_node("b", 1, 0, Some(sn));
-                let dn = add_node("dir", 1283, 4, Some(sn));
+                #[cfg(not(windows))]
+                let dn = add_node("dir", 13571, 7, Some(sn));
+                #[cfg(windows)]
+                let dn = add_node("dir", 1283, 7, Some(sn));
                 {
                     add_node("c", 257, 0, Some(dn));
                     add_node("d", 2, 0, Some(dn));
-                    let en = add_node("empty-dir", 0, 1, Some(dn));
+                    #[cfg(not(windows))]
+                    let en = add_node("empty-dir", 4096, 2, Some(dn));
+                    #[cfg(windows)]
+                    let en = add_node("empty-dir", 0, 2, Some(dn));
                     {
                         add_node(".gitkeep", 0, 0, Some(en));
                     }
-                    let sub = add_node("sub", 1024, 1, Some(dn));
+                    #[cfg(not(windows))]
+                    let sub = add_node("sub", 5120, 2, Some(dn));
+                    #[cfg(windows)]
+                    let sub = add_node("sub", 1024, 2, Some(dn));
                     {
                         add_node("e", 1024, 0, Some(sub));
                     }
@@ -331,5 +353,5 @@ pub fn make_add_node(
 }
 
 pub fn debug(item: impl fmt::Debug) -> String {
-    format!("{item:?}")
+    format!("{item:#?}")
 }

src/traverse.rs

@@ -19,6 +19,7 @@ pub type Tree = StableGraph<EntryData, (), Directed>;
 pub struct EntryData {
     pub name: PathBuf,
     /// The entry's size in bytes. If it's a directory, the size is the aggregated file size of all children
+    /// plus the  size of the directory entry itself
     pub size: u128,
     pub mtime: SystemTime,
     pub entry_count: Option<u64>,
@@ -128,6 +129,7 @@ impl EntryInfo {
 pub fn set_entry_info_or_panic(
     tree: &mut Tree,
     node_idx: TreeIndex,
+    node_own_size: u128,
     EntryInfo {
         size,
         entries_count,
@@ -136,8 +138,8 @@ pub fn set_entry_info_or_panic(
     let node = tree
         .node_weight_mut(node_idx)
         .expect("node for parent index we just retrieved");
-    node.size = size;
-    node.entry_count = entries_count;
+    node.size = size + node_own_size;
+    node.entry_count = entries_count.map(|n| n + 1);
 }
 
 pub fn parent_or_panic(tree: &mut Tree, parent_node_idx: TreeIndex) -> TreeIndex {
@@ -146,7 +148,7 @@ pub fn parent_or_panic(tree: &mut Tree, parent_node_idx: TreeIndex) -> TreeIndex
         .expect("every node in the iteration has a parent")
 }
 
-pub fn pop_or_panic(v: &mut Vec<EntryInfo>) -> EntryInfo {
+pub fn pop_or_panic<T>(v: &mut Vec<T>) -> T {
     v.pop().expect("sizes per level to be in sync with graph")
 }
 
@@ -168,6 +170,9 @@ pub struct BackgroundTraversal {
     parent_node_idx: TreeIndex,
     directory_info_per_depth_level: Vec<EntryInfo>,
     current_directory_at_depth: EntryInfo,
+    parent_node_size: u128,
+    parent_node_size_per_depth_level: Vec<u128>,
+    previous_node_size: u128,
     previous_depth: usize,
     inodes: InodeFilter,
     throttle: Option<Throttle>,
@@ -234,6 +239,9 @@ impl BackgroundTraversal {
             stats: TraversalStats::default(),
             previous_node_idx: root_idx,
             parent_node_idx: root_idx,
+            previous_node_size: 0,
+            parent_node_size: 0,
+            parent_node_size_per_depth_level: Vec::new(),
             directory_info_per_depth_level: Vec::new(),
             current_directory_at_depth: EntryInfo::default(),
             previous_depth: 0,
@@ -279,10 +287,10 @@ impl BackgroundTraversal {
                         let mut file_count = 0u64;
                         match &entry.client_state {
                             Some(Ok(ref m)) => {
-                                if !m.is_dir()
-                                    && (self.walk_options.count_hard_links || self.inodes.add(m))
-                                    && (self.walk_options.cross_filesystems
-                                        || crossdev::is_same_device(device_id, m))
+                                if self.walk_options.count_hard_links
+                                    || self.inodes.add(m)
+                                        && (self.walk_options.cross_filesystems
+                                            || crossdev::is_same_device(device_id, m))
                                 {
                                     file_count = 1;
                                     if self.walk_options.apparent_size {
@@ -326,13 +334,19 @@ impl BackgroundTraversal {
                                     size: file_size,
                                     entries_count: Some(file_count),
                                 };
+
+                                self.parent_node_size_per_depth_level
+                                    .push(self.previous_node_size);
+
                                 self.parent_node_idx = self.previous_node_idx;
+                                self.parent_node_size = self.previous_node_size;
                             }
                             (n, p) if n < p => {
                                 for _ in n..p {
                                     set_entry_info_or_panic(
                                         &mut traversal.tree,
                                         self.parent_node_idx,
+                                        self.parent_node_size,
                                         self.current_directory_at_depth,
                                     );
                                     let dir_info =
@@ -343,6 +357,8 @@ impl BackgroundTraversal {
 
                                     self.parent_node_idx =
                                         parent_or_panic(&mut traversal.tree, self.parent_node_idx);
+                                    self.parent_node_size =
+                                        pop_or_panic(&mut self.parent_node_size_per_depth_level);
                                 }
                                 self.current_directory_at_depth.size += file_size;
                                 *self
@@ -352,6 +368,7 @@ impl BackgroundTraversal {
                                 set_entry_info_or_panic(
                                     &mut traversal.tree,
                                     self.parent_node_idx,
+                                    self.parent_node_size,
                                     self.current_directory_at_depth,
                                 );
                             }
@@ -372,6 +389,7 @@ impl BackgroundTraversal {
                             .tree
                             .add_edge(self.parent_node_idx, entry_index, ());
                         self.previous_node_idx = entry_index;
+                        self.previous_node_size = file_size;
                         self.previous_depth = entry.depth;
                     }
                     Err(_) => {
@@ -406,6 +424,7 @@ impl BackgroundTraversal {
                     set_entry_info_or_panic(
                         &mut traversal.tree,
                         self.parent_node_idx,
+                        self.parent_node_size,
                         self.current_directory_at_depth,
                     );
                     self.parent_node_idx =
@@ -415,6 +434,7 @@ impl BackgroundTraversal {
                 set_entry_info_or_panic(
                     &mut traversal.tree,
                     self.root_idx,
+                    root_size,
                     EntryInfo {
                         size: root_size,
                         entries_count: (self.stats.entries_traversed > 0)