Docs oriented cleanup and some minor refactoring (#2432)

The "major" pieces are are:
1. Pull the README as the main docs for the `vortex` crate.
2. Remove some things from the top level namespace of `vortex-array`,
which is still too busy IMO, made a few others private.
3. Remove some unused code, and consolidating module with a bunch of
small files and nicely sized files.

You can see the current state/changes by running:
```bash
cargo doc --no-deps --open -p vortex
```

Author: AdamGS

README.md

@@ -5,8 +5,8 @@
 [![Documentation](https://docs.rs/vortex-array/badge.svg)](https://docs.rs/vortex-array)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/vortex-array)](https://pypi.org/project/vortex-array/)
 
-> [!TIP]
-> Check out the [Docs](https://docs.vortex.dev/)
+> \[!TIP\]
+> Check out our [Docs](https://docs.vortex.dev/)
 
 Vortex is an extensible, state-of-the-art columnar file format, with associated tools for working with compressed Apache
 Arrow arrays
@@ -21,7 +21,7 @@ decompression on GPUs.
 Vortex is intended to be to columnar file formats what Apache DataFusion is to query engines: highly extensible,
 extremely fast, & batteries-included.
 
-> [!CAUTION]
+> \[!CAUTION\]
 > This library is still under rapid development and is a work in progress!
 >
 > Some key features are not yet implemented, both the API and the serialized format are likely to change in breaking

vortex-array/src/aliases/mod.rs

@@ -1,4 +1,14 @@
+//! Re-exports of third-party crates we use in the API.
+//!
+//! The HashMap/Set should be preferred over the standard library variants or other alternatives.
+//! Currently defers to the excellent [hashbrown](https://docs.rs/hashbrown/latest/hashbrown/) crate.
+
 pub mod hash_map;
 pub mod hash_set;
 
 pub use hashbrown::DefaultHashBuilder;
+
+pub mod paste {
+    //! Re-export of [`paste`](https://docs.rs/paste/latest/paste/).
+    pub use paste::paste;
+}

vortex-array/src/arrays/chunked/compute/min_max.rs

@@ -3,7 +3,7 @@ use vortex_scalar::Scalar;
 
 use crate::arrays::{ChunkedArray, ChunkedEncoding};
 use crate::compute::{min_max, MinMaxFn, MinMaxResult};
-use crate::{partial_max, partial_min};
+use crate::partial_ord::{partial_max, partial_min};
 
 impl MinMaxFn<ChunkedArray> for ChunkedEncoding {
     fn min_max(&self, array: &ChunkedArray) -> VortexResult<Option<MinMaxResult>> {

vortex-array/src/arrays/constant/variants.rs

@@ -1,10 +1,8 @@
 use vortex_dtype::FieldName;
-use vortex_error::{VortexError, VortexExpect as _, VortexResult};
-use vortex_scalar::Scalar;
+use vortex_error::VortexResult;
 
 use crate::arrays::constant::ConstantArray;
 use crate::arrays::ConstantEncoding;
-use crate::iter::Accessor;
 use crate::variants::{
     BinaryArrayTrait, BoolArrayTrait, ExtensionArrayTrait, ListArrayTrait, NullArrayTrait,
     PrimitiveArrayTrait, StructArrayTrait, Utf8ArrayTrait,
@@ -57,16 +55,6 @@ impl NullArrayTrait for ConstantArray {}
 
 impl BoolArrayTrait for ConstantArray {}
 
-impl<T> Accessor<T> for ConstantArray
-where
-    T: Clone,
-    T: TryFrom<Scalar, Error = VortexError>,
-{
-    fn value_unchecked(&self, _index: usize) -> T {
-        T::try_from(self.scalar()).vortex_expect("Failed to convert scalar to value")
-    }
-}
-
 impl PrimitiveArrayTrait for ConstantArray {}
 
 impl Utf8ArrayTrait for ConstantArray {}

vortex-array/src/arrays/primitive/mod.rs

@@ -1,5 +1,5 @@
 use std::fmt::{Debug, Display};
-use std::{iter, ptr};
+use std::iter;
 mod accessor;
 
 use arrow_buffer::BooleanBufferBuilder;
@@ -11,7 +11,6 @@ use vortex_mask::Mask;
 
 use crate::builders::ArrayBuilder;
 use crate::encoding::encoding_ids;
-use crate::iter::Accessor;
 use crate::stats::StatsSet;
 use crate::validity::{Validity, ValidityMetadata};
 use crate::variants::PrimitiveArrayTrait;
@@ -302,31 +301,6 @@ impl VariantsVTable<PrimitiveArray> for PrimitiveEncoding {
     }
 }
 
-impl<T: NativePType> Accessor<T> for PrimitiveArray {
-    #[inline]
-    fn value_unchecked(&self, index: usize) -> T {
-        self.as_slice::<T>()[index]
-    }
-
-    #[inline]
-    fn decode_batch(&self, start_idx: usize) -> Vec<T> {
-        let batch_size = <Self as Accessor<T>>::batch_size(self, start_idx);
-        let mut v = Vec::<T>::with_capacity(batch_size);
-        let null_slice = self.as_slice::<T>();
-
-        unsafe {
-            v.set_len(batch_size);
-            ptr::copy_nonoverlapping(
-                null_slice.as_ptr().add(start_idx),
-                v.as_mut_ptr(),
-                batch_size,
-            );
-        }
-
-        v
-    }
-}
-
 impl PrimitiveArrayTrait for PrimitiveArray {}
 
 impl<T: NativePType> FromIterator<T> for PrimitiveArray {

vortex-array/src/children.rs

@@ -5,13 +5,16 @@ use crate::aliases::hash_set::HashSet;
 use crate::stats::PRUNING_STATS;
 use crate::{Array, EncodingId};
 
+/// Extendable compression interface, allowing implementations to explore different choices.
 pub trait CompressionStrategy {
+    /// Compress input array.
     fn compress(&self, array: &Array) -> VortexResult<Array>;
 
+    /// A set of the IDs of the encodings the compressor can choose from.
     fn used_encodings(&self) -> HashSet<EncodingId>;
 }
 
-/// Check that compression did not alter the length of the validity array.
+/// Verify that compression did not alter the length of the validity array.
 pub fn check_validity_unchanged(arr: &Array, compressed: &Array) {
     let _ = arr;
     let _ = compressed;
@@ -37,7 +40,7 @@ pub fn check_validity_unchanged(arr: &Array, compressed: &Array) {
     }
 }
 
-/// Check that compression did not alter the dtype
+/// Verify that compression did not alter the dtype.
 pub fn check_dtype_unchanged(arr: &Array, compressed: &Array) {
     let _ = arr;
     let _ = compressed;
@@ -54,7 +57,7 @@ pub fn check_dtype_unchanged(arr: &Array, compressed: &Array) {
     }
 }
 
-// Check that compression preserved the statistics.
+/// Verify that compression preserved the statistics.
 pub fn check_statistics_unchanged(arr: &Array, compressed: &Array) {
     let _ = arr;
     let _ = compressed;

vortex-array/src/compress.rs

@@ -18,8 +18,9 @@ use crate::encoding::{Encoding, EncodingId};
 use crate::iter::{ArrayIterator, ArrayIteratorAdapter};
 use crate::stats::{Precision, Stat, StatsSet};
 use crate::stream::{ArrayStream, ArrayStreamAdapter};
+use crate::visitor::{ChildrenVisitor, NamedChildrenVisitor};
 use crate::vtable::{EncodingVTable, VTableRef};
-use crate::{ArrayChildrenIterator, ChildrenCollector, ContextRef, NamedChildrenCollector};
+use crate::ContextRef;
 
 mod owned;
 mod statistics;
@@ -274,22 +275,22 @@ impl Array {
         match &self.0 {
             InnerArray::Owned(d) => d.children.to_vec(),
             InnerArray::Viewed(_) => {
-                let mut collector = ChildrenCollector::default();
+                let mut visitor = ChildrenVisitor::default();
                 self.vtable()
-                    .accept(self, &mut collector)
+                    .accept(self, &mut visitor)
                     .vortex_expect("Failed to get children");
-                collector.children()
+                visitor.children
             }
         }
     }
 
     /// Returns a Vec of Arrays with all the array's child arrays.
     pub fn named_children(&self) -> Vec<(String, Array)> {
-        let mut collector = NamedChildrenCollector::default();
+        let mut visitor = NamedChildrenVisitor::default();
         self.vtable()
-            .accept(&self.clone(), &mut collector)
+            .accept(&self.clone(), &mut visitor)
             .vortex_expect("Failed to get children");
-        collector.children()
+        visitor.children
     }
 
     /// Returns the number of child arrays
@@ -300,7 +301,7 @@ impl Array {
         }
     }
 
-    pub fn depth_first_traversal(&self) -> ArrayChildrenIterator {
+    pub fn depth_first_traversal(&self) -> impl Iterator<Item = Array> {
         ArrayChildrenIterator::new(self.clone())
     }
 
@@ -461,3 +462,26 @@ impl Iterator for ArrayChunkIterator {
         }
     }
 }
+
+/// A depth-first pre-order iterator over a Array.
+struct ArrayChildrenIterator {
+    stack: Vec<Array>,
+}
+
+impl ArrayChildrenIterator {
+    pub fn new(array: Array) -> Self {
+        Self { stack: vec![array] }
+    }
+}
+
+impl Iterator for ArrayChildrenIterator {
+    type Item = Array;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let next = self.stack.pop()?;
+        for child in next.children().into_iter().rev() {
+            self.stack.push(child);
+        }
+        Some(next)
+    }
+}

vortex-array/src/data/mod.rs

@@ -1,3 +1,9 @@
+//! An encoding of an array that we cannot interpret.
+//!
+//! Vortex allows for pluggable encodings. This can lead to issues when one process produces a file
+//! using a custom encoding, and then another process without knowledge of the encoding attempts
+//! to read it.
+
 use std::any::Any;
 use std::fmt::{Debug, Formatter};
 
@@ -12,12 +18,6 @@ use crate::vtable::{
 };
 use crate::{Array, Canonical};
 
-/// An encoding of an array that we cannot interpret.
-///
-/// Vortex allows for pluggable encodings. This can lead to issues when one process produces a file
-/// using a custom encoding, and then another process without knowledge of the encoding attempts
-/// to read it.
-///
 /// `OpaqueEncoding` allows deserializing these arrays. Many common operations will fail, but it
 /// allows deserialization and introspection in a type-erased manner on the children and metadata.
 ///

vortex-array/src/encoding/opaque.rs

@@ -1,11 +1,51 @@
+//! Iterator over slices of an array, and related utilities.
+
 use itertools::Itertools;
+use vortex_dtype::DType;
 use vortex_error::VortexResult;
 
 use crate::arrays::ChunkedArray;
-use crate::iter::ArrayIterator;
 use crate::stream::{ArrayStream, ArrayStreamAdapter};
 use crate::{Array, IntoArray};
 
+/// Iterator of array with a known [`DType`].
+///
+/// Its up to implementations to guarantee all arrays have the same [`DType`].
+pub trait ArrayIterator: Iterator<Item = VortexResult<Array>> {
+    fn dtype(&self) -> &DType;
+}
+
+pub struct ArrayIteratorAdapter<I> {
+    dtype: DType,
+    inner: I,
+}
+
+impl<I> ArrayIteratorAdapter<I> {
+    pub fn new(dtype: DType, inner: I) -> Self {
+        Self { dtype, inner }
+    }
+}
+
+impl<I> Iterator for ArrayIteratorAdapter<I>
+where
+    I: Iterator<Item = VortexResult<Array>>,
+{
+    type Item = VortexResult<Array>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.inner.next()
+    }
+}
+
+impl<I> ArrayIterator for ArrayIteratorAdapter<I>
+where
+    I: Iterator<Item = VortexResult<Array>>,
+{
+    fn dtype(&self) -> &DType {
+        &self.dtype
+    }
+}
+
 pub trait ArrayIteratorExt: ArrayIterator {
     fn into_stream(self) -> impl ArrayStream
     where