reserve_exact clearly not working

Signed-off-by: Connor Tsui <[email protected]>

Author: connortsui20

vortex-array/src/builders/listview.rs

@@ -255,10 +255,16 @@ impl<O: IntegerPType, S: IntegerPType> ArrayBuilder for ListViewBuilder<O, S> {
         // Otherwise, after removing any leading and trailing elements, we can simply bulk append
         // the entire array.
         let listview = listview.rebuild(ListViewRebuildMode::TrimElements);
+        let extend_length = listview.len();
 
         self.nulls.append_validity_mask(array.validity_mask());
-        self.elements_builder.extend_from_array(listview.elements());
 
+        self.elements_builder
+            .reserve_exact(listview.elements().len());
+        self.offsets_builder.reserve_exact(extend_length);
+        self.sizes_builder.reserve_exact(extend_length);
+
+        self.elements_builder.extend_from_array(listview.elements());
         self.sizes_builder.extend_from_array(
             compute::cast(listview.sizes(), self.sizes_builder.dtype())
                 .vortex_expect(
@@ -269,12 +275,10 @@ impl<O: IntegerPType, S: IntegerPType> ArrayBuilder for ListViewBuilder<O, S> {
 
         // Adjust all of the offsets.
         let new_offsets = listview.offsets().to_primitive(); // This should be cheap.
-        let num_new_offsets = new_offsets.len();
 
         let curr_builder_len = self.len();
-        self.offsets_builder.reserve_exact(num_new_offsets);
 
-        let uninit_range = self.offsets_builder.uninit_range(num_new_offsets);
+        let uninit_range = self.offsets_builder.uninit_range(extend_length);
 
         fn adjust_offsets<'a, O: IntegerPType, A: IntegerPType>(
             mut uninit_range: UninitRange<'a, O>,

vortex-buffer/src/buffer_mut.rs

@@ -193,6 +193,10 @@ impl<T> BufferMut<T> {
         // Make sure we at least double in size each time we re-allocate to amortize the cost
         let new_capacity = new_capacity.max(self.bytes.capacity() * 2);
 
+        if new_capacity > 15000 {
+            println!("{new_capacity}");
+        }
+
         let mut bytes = BytesMut::with_capacity(new_capacity);
         bytes.align_empty(self.alignment);
         bytes.extend_from_slice(&self.bytes);

vortex/examples/potentially_not_slow_list.rs

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use std::path::PathBuf;
+
+use vortex::file::{VortexOpenOptions, VortexWriteOptions};
+
+#[tokio::main]
+pub async fn main() -> anyhow::Result<()> {
+    let args = std::env::args().collect::<Vec<_>>();
+    let file_name = args[1].clone();
+    let path = PathBuf::from(file_name);
+
+    let array_stream = VortexOpenOptions::new()
+        .open(path)
+        .await?
+        .scan()?
+        .into_array_stream()?;
+
+    // Pre-allocate a 1GB buffer for the output file.
+    let memory_writer = Vec::with_capacity(1024 * 1024 * 1024);
+    let writer = VortexWriteOptions::default()
+        .write(memory_writer, array_stream)
+        .await?;
+
+    println!("Wrote {} bytes to new file", writer.size());
+
+    Ok(())
+}