add more exclusion rules (beyond the old compressor)

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>

Author: connortsui20

vortex-btrblocks/public-api.lock

@@ -284,6 +284,8 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::F
 
 impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::FoRScheme
 
+pub fn vortex_btrblocks::schemes::integer::FoRScheme::ancestor_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::AncestorExclusion>
+
 pub fn vortex_btrblocks::schemes::integer::FoRScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
 
 pub fn vortex_btrblocks::schemes::integer::FoRScheme::expected_compression_ratio(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult<f64>
@@ -468,6 +470,8 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::Z
 
 impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::ZigZagScheme
 
+pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::ancestor_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::AncestorExclusion>
+
 pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
 
 pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::descendant_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::DescendantExclusion>

vortex-btrblocks/src/schemes/decimal.rs

@@ -36,6 +36,7 @@ impl Scheme for DecimalScheme {
         matches!(canonical, Canonical::Decimal(_))
     }
 
+    /// Children: primitive=0.
     fn num_children(&self) -> usize {
         1
     }

vortex-btrblocks/src/schemes/float.rs

@@ -74,14 +74,6 @@ impl rle::RLEConfig for FloatRLEConfig {
     fn generate_stats(array: &ArrayRef) -> FloatStats {
         FloatStats::generate(&array.to_primitive())
     }
-
-    fn compress_values(
-        compressor: &CascadingCompressor,
-        values: &PrimitiveArray,
-        ctx: CompressorContext,
-    ) -> VortexResult<ArrayRef> {
-        compressor.compress_canonical(Canonical::Primitive(values.clone()), ctx)
-    }
 }
 
 impl RLEStats for FloatStats {
@@ -110,6 +102,7 @@ impl Scheme for ALPScheme {
         is_float_primitive(canonical)
     }
 
+    /// Children: encoded_ints=0.
     fn num_children(&self) -> usize {
         1
     }
@@ -220,11 +213,11 @@ impl Scheme for NullDominatedSparseScheme {
         is_float_primitive(canonical)
     }
 
+    /// Children: indices=0.
     fn num_children(&self) -> usize {
         1
     }
 
-    // TODO(connor): There seems to be stuff missing here...
     /// The indices of a null-dominated sparse array should not be sparse-encoded again.
     fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
         vec![DescendantExclusion {

vortex-btrblocks/src/schemes/integer.rs

@@ -98,14 +98,6 @@ impl rle::RLEConfig for IntRLEConfig {
     fn generate_stats(array: &ArrayRef) -> IntegerStats {
         IntegerStats::generate(&array.to_primitive())
     }
-
-    fn compress_values(
-        compressor: &CascadingCompressor,
-        values: &PrimitiveArray,
-        ctx: CompressorContext,
-    ) -> VortexResult<ArrayRef> {
-        compressor.compress_canonical(Canonical::Primitive(values.clone()), ctx)
-    }
 }
 
 impl RLEStats for IntegerStats {
@@ -134,6 +126,24 @@ impl Scheme for FoRScheme {
         is_integer_primitive(canonical)
     }
 
+    /// Dict codes always start at 0, so FoR (which subtracts the min) is a no-op.
+    fn ancestor_exclusions(&self) -> Vec<AncestorExclusion> {
+        vec![
+            AncestorExclusion {
+                ancestor: IntDictScheme.id(),
+                children: ChildSelection::One(1),
+            },
+            AncestorExclusion {
+                ancestor: FloatDictScheme.id(),
+                children: ChildSelection::One(1),
+            },
+            AncestorExclusion {
+                ancestor: StringDictScheme.id(),
+                children: ChildSelection::One(1),
+            },
+        ]
+    }
+
     fn expected_compression_ratio(
         &self,
         _compressor: &CascadingCompressor,
@@ -225,6 +235,7 @@ impl Scheme for ZigZagScheme {
         is_integer_primitive(canonical)
     }
 
+    /// Children: encoded=0.
     fn num_children(&self) -> usize {
         1
     }
@@ -249,6 +260,24 @@ impl Scheme for ZigZagScheme {
         ]
     }
 
+    /// Dict codes are unsigned integers (0..cardinality). ZigZag only helps negatives.
+    fn ancestor_exclusions(&self) -> Vec<AncestorExclusion> {
+        vec![
+            AncestorExclusion {
+                ancestor: IntDictScheme.id(),
+                children: ChildSelection::One(1),
+            },
+            AncestorExclusion {
+                ancestor: FloatDictScheme.id(),
+                children: ChildSelection::One(1),
+            },
+            AncestorExclusion {
+                ancestor: StringDictScheme.id(),
+                children: ChildSelection::One(1),
+            },
+        ]
+    }
+
     fn expected_compression_ratio(
         &self,
         compressor: &CascadingCompressor,
@@ -365,6 +394,7 @@ impl Scheme for SparseScheme {
         }
     }
 
+    /// Children: values=0, indices=1.
     fn num_children(&self) -> usize {
         2
     }
@@ -494,6 +524,7 @@ impl Scheme for RunEndScheme {
         is_integer_primitive(canonical)
     }
 
+    /// Children: values=0, ends=1.
     fn num_children(&self) -> usize {
         2
     }
@@ -507,12 +538,23 @@ impl Scheme for RunEndScheme {
         }]
     }
 
-    // TODO(connor): There seems to be stuff missing here...
+    /// Dict values (child 0) are all unique by definition, so run-end encoding them is
+    /// pointless. Codes (child 1) can have runs and may benefit from RunEnd.
     fn ancestor_exclusions(&self) -> Vec<AncestorExclusion> {
-        vec![AncestorExclusion {
-            ancestor: FloatDictScheme.id(),
-            children: ChildSelection::One(0),
-        }]
+        vec![
+            AncestorExclusion {
+                ancestor: IntDictScheme.id(),
+                children: ChildSelection::One(0),
+            },
+            AncestorExclusion {
+                ancestor: FloatDictScheme.id(),
+                children: ChildSelection::One(0),
+            },
+            AncestorExclusion {
+                ancestor: StringDictScheme.id(),
+                children: ChildSelection::One(0),
+            },
+        ]
     }
 
     fn expected_compression_ratio(

vortex-btrblocks/src/schemes/rle.rs

@@ -9,6 +9,9 @@ use vortex_array::Canonical;
 use vortex_array::IntoArray;
 use vortex_array::ToCanonical;
 use vortex_array::arrays::PrimitiveArray;
+use vortex_compressor::builtins::FloatDictScheme;
+use vortex_compressor::builtins::StringDictScheme;
+use vortex_compressor::scheme::AncestorExclusion;
 use vortex_compressor::scheme::ChildSelection;
 use vortex_compressor::scheme::DescendantExclusion;
 #[cfg(feature = "unstable_encodings")]
@@ -37,8 +40,6 @@ pub trait RLEStats {
     fn source(&self) -> &PrimitiveArray;
 }
 
-// TODO(connor): This trait is super confusing, we should probably just remove it and hardcode the
-// only 2 implementations (integer and float).
 /// Configuration trait for RLE schemes.
 ///
 /// Implement this trait to define the behavior of an RLE scheme for a specific
@@ -55,13 +56,6 @@ pub trait RLEConfig: Debug + Send + Sync + 'static {
 
     /// Generates statistics for the given array.
     fn generate_stats(array: &ArrayRef) -> Self::Stats;
-
-    /// Compress the values array after RLE encoding.
-    fn compress_values(
-        compressor: &CascadingCompressor,
-        values: &PrimitiveArray,
-        ctx: CompressorContext,
-    ) -> VortexResult<ArrayRef>;
 }
 
 /// RLE scheme that is generic over a configuration type.
@@ -92,6 +86,7 @@ impl<C: RLEConfig> Scheme for RLEScheme<C> {
         C::matches(canonical)
     }
 
+    /// Children: values=0, indices=1, offsets=2.
     fn num_children(&self) -> usize {
         3
     }
@@ -106,6 +101,24 @@ impl<C: RLEConfig> Scheme for RLEScheme<C> {
         }]
     }
 
+    /// Dict values (child 0) are all unique by definition, so RLE is pointless on them.
+    fn ancestor_exclusions(&self) -> Vec<AncestorExclusion> {
+        vec![
+            AncestorExclusion {
+                ancestor: IntDictScheme.id(),
+                children: ChildSelection::One(0),
+            },
+            AncestorExclusion {
+                ancestor: FloatDictScheme.id(),
+                children: ChildSelection::One(0),
+            },
+            AncestorExclusion {
+                ancestor: StringDictScheme.id(),
+                children: ChildSelection::One(0),
+            },
+        ]
+    }
+
     fn expected_compression_ratio(
         &self,
         compressor: &CascadingCompressor,

vortex-btrblocks/src/schemes/string.rs

@@ -62,6 +62,7 @@ impl Scheme for FSSTScheme {
         is_utf8_string(canonical)
     }
 
+    /// Children: lengths=0, code_offsets=1.
     fn num_children(&self) -> usize {
         2
     }
@@ -124,11 +125,11 @@ impl Scheme for NullDominatedSparseScheme {
         is_utf8_string(canonical)
     }
 
+    /// Children: indices=0.
     fn num_children(&self) -> usize {
         1
     }
 
-    // TODO(connor): There seems to be stuff missing here...
     /// The indices of a null-dominated sparse array should not be sparse-encoded again.
     fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
         vec![

vortex-btrblocks/src/schemes/temporal.rs

@@ -53,6 +53,7 @@ impl Scheme for TemporalScheme {
         true
     }
 
+    /// Children: days=0, seconds=1, subseconds=2.
     fn num_children(&self) -> usize {
         3
     }

vortex-compressor/public-api.lock

@@ -402,14 +402,6 @@ pub fn vortex_compressor::ctx::CompressorContext::fmt(&self, f: &mut core::fmt::
 
 pub const vortex_compressor::ctx::MAX_CASCADE: usize
 
-pub mod vortex_compressor::root_list_children
-
-pub const vortex_compressor::root_list_children::ELEMENTS: usize
-
-pub const vortex_compressor::root_list_children::OFFSETS: usize
-
-pub const vortex_compressor::root_list_children::SIZES: usize
-
 pub mod vortex_compressor::scheme
 
 pub enum vortex_compressor::scheme::ChildSelection
@@ -1040,8 +1032,6 @@ impl vortex_compressor::CascadingCompressor
 
 pub fn vortex_compressor::CascadingCompressor::compress(&self, array: &vortex_array::array::ArrayRef) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
 
-pub fn vortex_compressor::CascadingCompressor::compress_canonical(&self, array: vortex_array::canonical::Canonical, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
-
 pub fn vortex_compressor::CascadingCompressor::compress_child(&self, child: &vortex_array::array::ArrayRef, parent_ctx: &vortex_compressor::ctx::CompressorContext, parent_id: vortex_compressor::scheme::SchemeId, child_index: usize) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
 
 pub fn vortex_compressor::CascadingCompressor::execution_ctx(&self) -> parking_lot::mutex::MutexGuard<'_, vortex_array::executor::ExecutionCtx>

vortex-compressor/src/builtins/dict/mod.rs

@@ -47,8 +47,9 @@ impl Scheme for IntDictScheme {
         }
     }
 
+    /// Children: values=0, codes=1.
     fn num_children(&self) -> usize {
-        1
+        2
     }
 
     fn expected_compression_ratio(
@@ -142,11 +143,19 @@ impl Scheme for FloatDictScheme {
         }
     }
 
+    /// Children: values=0, codes=1.
     fn num_children(&self) -> usize {
         2
     }
 
-    // TODO(connor): There seems to be stuff missing here...
+    /// Float dict codes (child 1) are compact unsigned integers that should not be
+    /// dict-encoded again. Float dict values (child 0) flow through ALP into integer-land,
+    /// where integer dict encoding is redundant since the values are already deduplicated at
+    /// the float level.
+    ///
+    /// Additional exclusions for codes (IntSequenceScheme, IntRunEndScheme, FoRScheme,
+    /// ZigZagScheme, SparseScheme, RLE) are expressed as pull rules on those schemes in
+    /// vortex-btrblocks.
     fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
         vec![
             DescendantExclusion {
@@ -235,11 +244,16 @@ impl Scheme for StringDictScheme {
         }
     }
 
+    /// Children: values=0, codes=1.
     fn num_children(&self) -> usize {
         2
     }
 
-    // TODO(connor): There seems to be stuff missing here...
+    /// String dict codes (child 1) are compact unsigned integers that should not be dict-encoded
+    /// again.
+    ///
+    /// Additional exclusions for codes (IntSequenceScheme, FoRScheme, ZigZagScheme, SparseScheme,
+    /// RunEndScheme, RLE, etc.) are expressed as pull rules on those schemes in `vortex-btrblocks`.
     fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
         vec![DescendantExclusion {
             excluded: IntDictScheme.id(),