OPT 9 - Calculate exact total nodes needed

Author: radevgit

Cargo.toml

@@ -22,7 +22,7 @@ codegen-units = 1
 panic = "abort"
 opt-level = 3
 # for samply
-debug = 1
+#debug = 1
 
 [dev-dependencies]
 rand = "0.9"

examples/perf_build.rs

@@ -115,5 +115,11 @@ OPT 8 - removed unecessary vector initializations (2.3%)
   Average Build:    75.44ms
   Overall time:     8124.56ms
 ___________________________________________________
+OPT 9 - Calculate exact total nodes needed
+  Total Add time:   397.41ms
+  Average Add:      3.97ms
+  Total Build time: 7442.48ms
+  Average Build:    74.42ms
+  Overall time:     7965.43ms
 
  */

examples/perf_build_i32.rs

@@ -97,6 +97,14 @@ OPT 8 - removed unecessary vector initializations (2.3%)
   Total Build time: 6417.60ms
   Average Build:    64.18ms
   Overall time:     6895.65ms
+________________________________________________
+OPT 9 - Calculate exact total nodes needed
+Build Summary (100 runs):
+  Total Add time:   271.15ms
+  Average Add:      2.71ms
+  Total Build time: 6414.16ms
+  Average Build:    64.14ms
+  Overall time:     6755.35ms
 
 
 */

src/hilbert_rtree.rs

@@ -68,6 +68,24 @@ fn estimate_total_nodes(num_items: usize, node_size: usize) -> usize {
     (num_items * node_size) / (node_size - 1) + 1
 }
 
+/// Helper: Calculate EXACT total nodes by simulating tree construction (O(log n) - tree depth)
+#[inline]
+fn calculate_exact_total_nodes(num_items: usize, node_size: usize) -> usize {
+    if num_items == 0 {
+        return 0;
+    }
+    let mut total_nodes = num_items;
+    let mut count = num_items;
+    loop {
+        count = count.div_ceil(node_size);
+        total_nodes += count;
+        if count <= 1 {
+            break;
+        }
+    }
+    total_nodes
+}
+
 /// Helper: Calculate required buffer size for estimated nodes
 #[inline]
 fn estimate_buffer_size(num_items: usize, node_size: usize) -> usize {
@@ -147,36 +165,40 @@ impl HilbertRTree {
         let num_items = self.num_items;
         let node_size = self.node_size;
 
-        // Calculate total nodes and level bounds
+        // Calculate exact total nodes needed (O(log n) - only tree depth iterations)
+        let total_nodes = calculate_exact_total_nodes(num_items, node_size);
+        let data_size = HEADER_SIZE + total_nodes * (size_of::<Box>() + size_of::<u32>());
+        
+        // Reserve all needed space at once (avoids reallocation during build)
+        if data_size > self.data.capacity() {
+            self.data.reserve(data_size - self.data.capacity());
+            self.allocated_capacity = self.data.capacity();
+        }
+        
+        // CRITICAL: Must zero-fill parent node memory. The build process writes parent nodes
+        // incrementally in the loop below, and during tree traversal we may read indices/boxes
+        // from parent positions before they're written. Zero values act as sentinels.
+        // This is unavoidable without additional bookkeeping to track which positions are initialized.
+        if self.data.len() < data_size {
+            self.data.resize(data_size, 0);
+        }
+
+        // Calculate level bounds
         let mut level_bounds = Vec::with_capacity(16); // Max tree depth ~16 for 1M items
         let mut count = num_items;
-        let mut total_nodes = num_items;
-        level_bounds.push(total_nodes);
+        let mut level_total_nodes = num_items;
+        level_bounds.push(level_total_nodes);
 
         // Create parent levels until we have a single root
         loop {
             count = count.div_ceil(node_size);
-            total_nodes += count;
-            level_bounds.push(total_nodes);
+            level_total_nodes += count;
+            level_bounds.push(level_total_nodes);
             if count <= 1 {
                 break;
             }
         }
 
-        // Ensure data buffer has sufficient capacity and length for all writes
-        let data_size = HEADER_SIZE + total_nodes * (size_of::<Box>() + size_of::<u32>());
-        if data_size > self.data.capacity() {
-            self.data.reserve(data_size - self.data.capacity());
-            self.allocated_capacity = self.data.capacity();
-        }
-        // Set length to required size (uninitialized, but we'll overwrite everything)
-        // This avoids the zero-fill overhead of resize()
-        if self.data.len() < data_size {
-            unsafe {
-                self.data.set_len(data_size);
-            }
-        }
-
         // Write header
         self.data[0] = 0xfb; // magic
         self.data[1] = 0x01; // version 1 + double type (8)

src/hilbert_rtree_i32.rs

@@ -78,6 +78,24 @@ fn estimate_total_nodes(num_items: usize, node_size: usize) -> usize {
     (num_items * node_size) / (node_size - 1) + 1
 }
 
+/// Helper: Calculate EXACT total nodes by simulating tree construction (O(log n) - tree depth)
+#[inline]
+fn calculate_exact_total_nodes(num_items: usize, node_size: usize) -> usize {
+    if num_items == 0 {
+        return 0;
+    }
+    let mut total_nodes = num_items;
+    let mut count = num_items;
+    loop {
+        count = count.div_ceil(node_size);
+        total_nodes += count;
+        if count <= 1 {
+            break;
+        }
+    }
+    total_nodes
+}
+
 /// Helper: Calculate required buffer size for estimated nodes
 #[inline]
 fn estimate_buffer_size(num_items: usize, node_size: usize) -> usize {
@@ -216,17 +234,32 @@ impl HilbertRTreeI32 {
         let num_items = self.num_items;
         let node_size = self.node_size;
 
-        // Calculate total nodes and level bounds
+        // Calculate exact total nodes needed (O(log n) - only tree depth iterations)
+        let total_nodes = calculate_exact_total_nodes(num_items, node_size);
+        let data_size = HEADER_SIZE + total_nodes * (size_of::<BoxI32>() + size_of::<u32>());
+        
+        // Reserve all needed space at once (avoids reallocation during build)
+        if data_size > self.data.capacity() {
+            self.data.reserve(data_size - self.data.capacity());
+            self.allocated_capacity = self.data.capacity();
+        }
+        
+        // Resize to final size (zero-fill is necessary - build reads uninitialized parent positions)
+        if self.data.len() < data_size {
+            self.data.resize(data_size, 0);
+        }
+
+        // Calculate level bounds
         let mut level_bounds = Vec::with_capacity(16); // Max tree depth ~16 for 1M items
         let mut count = num_items;
-        let mut total_nodes = num_items;
-        level_bounds.push(total_nodes);
+        let mut level_total_nodes = num_items;
+        level_bounds.push(level_total_nodes);
 
         // Create parent levels until we have a single root
         loop {
             count = count.div_ceil(node_size);
-            total_nodes += count;
-            level_bounds.push(total_nodes);
+            level_total_nodes += count;
+            level_bounds.push(level_total_nodes);
             if count <= 1 {
                 break;
             }