add more rules and clean up

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>

Author: connortsui20

vortex-btrblocks/src/builder.rs

@@ -13,6 +13,7 @@ use crate::SchemeId;
 use crate::schemes::decimal;
 use crate::schemes::float;
 use crate::schemes::integer;
+use crate::schemes::rle;
 use crate::schemes::string;
 use crate::schemes::temporal;
 
@@ -26,13 +27,14 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[
     &integer::IntConstantScheme,
     // NOTE: FoR must precede BitPacking to avoid unnecessary patches.
     &integer::FoRScheme,
-    &integer::BitPackingScheme,
+    // NOTE: ZigZag should precede BitPacking because we don't want negative numbers.
     &integer::ZigZagScheme,
+    &integer::BitPackingScheme,
     &integer::SparseScheme,
     &integer::IntDictScheme,
     &integer::RunEndScheme,
     &integer::SequenceScheme,
-    &integer::RLE_INTEGER_SCHEME,
+    &rle::RLE_INTEGER_SCHEME,
     #[cfg(feature = "pco")]
     &integer::PcoScheme,
     // Float schemes.
@@ -42,7 +44,7 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[
     &float::ALPRDScheme,
     &float::FloatDictScheme,
     &float::NullDominatedSparseScheme,
-    &float::RLE_FLOAT_SCHEME,
+    &rle::RLE_FLOAT_SCHEME,
     #[cfg(feature = "pco")]
     &float::PcoScheme,
     // Decimal schemes.

vortex-btrblocks/src/schemes/float.rs

@@ -3,15 +3,13 @@
 
 //! Float compression schemes.
 
-use vortex_alp::ALP;
 use vortex_alp::ALPArray;
 use vortex_alp::RDEncoder;
 use vortex_alp::alp_encode;
 use vortex_array::ArrayRef;
 use vortex_array::Canonical;
 use vortex_array::IntoArray;
 use vortex_array::ToCanonical;
-use vortex_array::arrays::PrimitiveArray;
 use vortex_array::dtype::PType;
 use vortex_compressor::scheme::ChildSelection;
 use vortex_compressor::scheme::DescendantExclusion;
@@ -28,15 +26,12 @@ use crate::Scheme;
 use crate::SchemeExt;
 use crate::compress_patches;
 use crate::estimate_compression_ratio_with_sampling;
-use crate::schemes::rle;
-use crate::schemes::rle::RLEScheme;
-use crate::schemes::rle::RLEStats;
 
 /// ALP (Adaptive Lossless floating-Point) encoding.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub struct ALPScheme;
 
-/// ALPRD (ALP with Right Division) encoding variant.
+/// ALPRD (ALP with Real Double) encoding variant.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub struct ALPRDScheme;
 
@@ -51,47 +46,14 @@ pub struct NullDominatedSparseScheme;
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub struct PcoScheme;
 
-/// Configuration for float RLE compression.
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub struct FloatRLEConfig;
-
 // Re-export builtin schemes from vortex-compressor.
 pub use vortex_compressor::builtins::FloatConstantScheme;
 pub use vortex_compressor::builtins::FloatDictScheme;
 pub use vortex_compressor::builtins::FloatUncompressedScheme;
 pub use vortex_compressor::builtins::is_float_primitive;
 pub use vortex_compressor::stats::FloatStats;
 
-impl rle::RLEConfig for FloatRLEConfig {
-    type Stats = FloatStats;
-
-    const SCHEME_NAME: &'static str = "vortex.float.rle";
-
-    fn matches(canonical: &Canonical) -> bool {
-        is_float_primitive(canonical)
-    }
-
-    fn generate_stats(array: &ArrayRef) -> FloatStats {
-        FloatStats::generate(&array.to_primitive())
-    }
-}
-
-impl RLEStats for FloatStats {
-    fn value_count(&self) -> u32 {
-        FloatStats::value_count(self)
-    }
-
-    fn average_run_length(&self) -> u32 {
-        FloatStats::average_run_length(self)
-    }
-
-    fn source(&self) -> &PrimitiveArray {
-        FloatStats::source(self)
-    }
-}
-
-/// RLE scheme for float compression.
-pub const RLE_FLOAT_SCHEME: RLEScheme<FloatRLEConfig> = RLEScheme::new();
+pub use crate::schemes::rle::RLE_FLOAT_SCHEME;
 
 impl Scheme for ALPScheme {
     fn scheme_name(&self) -> &'static str {
@@ -113,16 +75,14 @@ impl Scheme for ALPScheme {
         data: &mut ArrayAndStats,
         ctx: CompressorContext,
     ) -> VortexResult<f64> {
-        // ALP encodes floats as integers. Without integer compression afterward, the
-        // encoded ints are the same size.
+        // ALP encodes floats as integers. Without integer compression afterward, the encoded ints
+        // are the same size.
         if ctx.finished_cascading() {
             return Ok(0.0);
         }
 
-        let stats = data.float_stats();
-
         // We don't support ALP for f16.
-        if stats.source().ptype() == PType::F16 {
+        if data.float_stats().source().ptype() == PType::F16 {
             return Ok(0.0);
         }
 
@@ -138,19 +98,16 @@ impl Scheme for ALPScheme {
         let stats = data.float_stats();
 
         let alp_encoded = alp_encode(&stats.source().to_primitive(), None)?;
-        let alp = alp_encoded.as_::<ALP>();
-        let alp_ints = alp.encoded().to_primitive();
 
         // Compress the ALP ints.
-
         let compressed_alp_ints =
-            compressor.compress_child(&alp_ints.into_array(), &ctx, self.id(), 0)?;
+            compressor.compress_child(alp_encoded.encoded(), &ctx, self.id(), 0)?;
 
         // Patches are not compressed. They should be infrequent, and if they are not then we want
         // to keep them linear for easy indexing.
-        let patches = alp.patches().map(compress_patches).transpose()?;
+        let patches = alp_encoded.patches().map(compress_patches).transpose()?;
 
-        Ok(ALPArray::new(compressed_alp_ints, alp.exponents(), patches).into_array())
+        Ok(ALPArray::new(compressed_alp_ints, alp_encoded.exponents(), patches).into_array())
     }
 }
 
@@ -169,9 +126,7 @@ impl Scheme for ALPRDScheme {
         data: &mut ArrayAndStats,
         ctx: CompressorContext,
     ) -> VortexResult<f64> {
-        let stats = data.float_stats();
-
-        if stats.source().ptype() == PType::F16 {
+        if data.float_stats().source().ptype() == PType::F16 {
             return Ok(0.0);
         }
 
@@ -235,11 +190,11 @@ impl Scheme for NullDominatedSparseScheme {
         let stats = data.float_stats();
 
         if stats.value_count() == 0 {
-            // All nulls should use ConstantScheme.
+            // All nulls should use ConstantScheme instead of this.
             return Ok(0.0);
         }
 
-        // If the majority is null, will compress well.
+        // If the majority (90%) of values is null, this will compress well.
         if stats.null_count() as f64 / stats.source().len() as f64 > 0.9 {
             return Ok(stats.source().len() as f64 / stats.value_count() as f64);
         }
@@ -261,12 +216,8 @@ impl Scheme for NullDominatedSparseScheme {
 
         if let Some(sparse) = sparse_encoded.as_opt::<Sparse>() {
             let indices = sparse.patches().indices().to_primitive().narrow()?;
-            let compressed_indices = compressor.compress_child(
-                &indices.to_primitive().into_array(),
-                &ctx,
-                self.id(),
-                0,
-            )?;
+            let compressed_indices =
+                compressor.compress_child(&indices.into_array(), &ctx, self.id(), 0)?;
 
             SparseArray::try_new(
                 compressed_indices,
@@ -326,8 +277,8 @@ mod tests {
     use vortex_error::VortexResult;
     use vortex_fastlanes::RLE;
 
-    use super::RLE_FLOAT_SCHEME;
     use crate::BtrBlocksCompressor;
+    use crate::schemes::rle::RLE_FLOAT_SCHEME;
 
     #[test]
     fn test_empty() -> VortexResult<()> {