Merge remote-tracking branch 'origin/develop' into ji/assert-array-eq-more-places

# Conflicts:
#	encodings/fastlanes/src/bitpacking/compress.rs
#	encodings/fastlanes/src/bitpacking/compute/cast.rs
#	encodings/fastlanes/src/delta/compute/cast.rs
#	encodings/fastlanes/src/delta/ops.rs
#	encodings/fastlanes/src/for/compress.rs
#	encodings/fastlanes/src/for/compute/cast.rs
#	encodings/fastlanes/src/rle/compress.rs
#	encodings/fastlanes/src/rle/ops.rs

Author: joseph-isaacs

.github/workflows/bench-pr.yml

@@ -13,6 +13,7 @@ permissions:
   contents: read
   pull-requests: write  # for commenting on PRs
   id-token: write  # enables AWS-GitHub OIDC
+  deployments: write  # for Polar Signals profiling
 
 jobs:
   label_trigger:

.github/workflows/sql-pr.yml

@@ -13,6 +13,7 @@ permissions:
   contents: read
   pull-requests: write  # for commenting on PRs
   id-token: write  # enables AWS-GitHub OIDC
+  deployments: write  # for Polar Signals profiling
 
 jobs:
   label_trigger:

Cargo.lock

@@ -32,7 +32,7 @@ impl VTable for ALPVTable {
     type ComputeVTable = NotSupported;
     type EncodeVTable = Self;
     type SerdeVTable = Self;
-    type PipelineVTable = NotSupported;
+    type OperatorVTable = NotSupported;
 
     fn id(_encoding: &Self::Encoding) -> EncodingId {
         EncodingId::new_ref("vortex.alp")

encodings/alp/src/alp/array.rs

@@ -63,7 +63,7 @@ where
 {
     let values_slice = values.as_slice::<T>();
 
-    let (exponents, encoded, exceptional_positions, exceptional_values, chunk_offsets) =
+    let (exponents, encoded, exceptional_positions, exceptional_values, mut chunk_offsets) =
         T::encode(values_slice, exponents);
 
     let encoded_array = PrimitiveArray::new(encoded, values.validity().clone()).into_array();
@@ -82,7 +82,16 @@ where
                 let (pos, vals): (BufferMut<u64>, BufferMut<T>) = exceptional_positions
                     .into_iter()
                     .zip_eq(exceptional_values)
-                    .filter(|(index, _)| is_valid.value(*index as usize))
+                    .filter(|(index, _)| {
+                        let is_valid = is_valid.value(*index as usize);
+                        if !is_valid {
+                            let patch_chunk = *index as usize / 1024;
+                            for chunk_idx in (patch_chunk + 1)..chunk_offsets.len() {
+                                chunk_offsets[chunk_idx] -= 1;
+                            }
+                        }
+                        is_valid
+                    })
                     .unzip();
                 (pos.freeze(), vals.freeze())
             }
@@ -145,30 +154,26 @@ mod tests {
         let array = PrimitiveArray::new(buffer![1.234f32; 1025], Validity::NonNullable);
         let encoded = alp_encode(&array, None).unwrap();
         assert!(encoded.patches().is_none());
-        assert_eq!(
-            encoded.encoded().to_primitive().as_slice::<i32>(),
-            vec![1234; 1025]
-        );
+        let expected_encoded = PrimitiveArray::from_iter(vec![1234i32; 1025]);
+        assert_arrays_eq!(encoded.encoded(), expected_encoded);
         assert_eq!(encoded.exponents(), Exponents { e: 9, f: 6 });
 
         let decoded = decompress(&encoded);
-        assert_eq!(array.as_slice::<f32>(), decoded.as_slice::<f32>());
+        assert_arrays_eq!(decoded, array);
     }
 
     #[test]
     fn test_nullable_compress() {
         let array = PrimitiveArray::from_option_iter([None, Some(1.234f32), None]);
         let encoded = alp_encode(&array, None).unwrap();
         assert!(encoded.patches().is_none());
-        assert_eq!(
-            encoded.encoded().to_primitive().as_slice::<i32>(),
-            vec![0, 1234, 0]
-        );
+        let expected_encoded = PrimitiveArray::from_option_iter([None, Some(1234i32), None]);
+        assert_arrays_eq!(encoded.encoded(), expected_encoded);
         assert_eq!(encoded.exponents(), Exponents { e: 9, f: 6 });
 
         let decoded = decompress(&encoded);
-        let expected = vec![0f32, 1.234f32, 0f32];
-        assert_eq!(decoded.as_slice::<f32>(), expected.as_slice());
+        let expected = PrimitiveArray::from_option_iter(vec![None, Some(1.234f32), None]);
+        assert_arrays_eq!(decoded, expected);
     }
 
     #[test]
@@ -178,14 +183,13 @@ mod tests {
         let array = PrimitiveArray::new(values.clone(), Validity::NonNullable);
         let encoded = alp_encode(&array, None).unwrap();
         assert!(encoded.patches().is_some());
-        assert_eq!(
-            encoded.encoded().to_primitive().as_slice::<i64>(),
-            vec![1234i64, 2718, 1234, 4000]
-        );
+        let expected_encoded = PrimitiveArray::from_iter(vec![1234i64, 2718, 1234, 4000]);
+        assert_arrays_eq!(encoded.encoded(), expected_encoded);
         assert_eq!(encoded.exponents(), Exponents { e: 16, f: 13 });
 
         let decoded = decompress(&encoded);
-        assert_eq!(values.as_slice(), decoded.as_slice::<f64>());
+        let expected_decoded = PrimitiveArray::new(values, Validity::NonNullable);
+        assert_arrays_eq!(decoded, expected_decoded);
     }
 
     #[test]
@@ -195,10 +199,9 @@ mod tests {
         let array = PrimitiveArray::new(values, Validity::from_iter([true, true, false, true]));
         let encoded = alp_encode(&array, None).unwrap();
         assert!(encoded.patches().is_none());
-        assert_eq!(
-            encoded.encoded().to_primitive().as_slice::<i64>(),
-            vec![1234i64, 2718, 1234, 4000]
-        );
+        let expected_encoded =
+            PrimitiveArray::from_option_iter(buffer![Some(1234i64), Some(2718), None, Some(4000)]);
+        assert_arrays_eq!(encoded.encoded(), expected_encoded);
         assert_eq!(encoded.exponents(), Exponents { e: 16, f: 13 });
 
         let decoded = decompress(&encoded);
@@ -230,7 +233,7 @@ mod tests {
         let original = PrimitiveArray::from_iter([195.26274f32, 195.27837, -48.815685]);
         let alp_arr = alp_encode(&original, None).unwrap();
         let decompressed = alp_arr.to_primitive();
-        assert_eq!(original.as_slice::<f32>(), decompressed.as_slice::<f32>());
+        assert_arrays_eq!(decompressed, original);
     }
 
     #[test]
@@ -280,13 +283,16 @@ mod tests {
         let patches = encoded.patches().unwrap();
 
         let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive();
-        assert_eq!(chunk_offsets.as_slice::<u64>(), &[0, 1, 3]);
+        let expected_offsets = PrimitiveArray::from_iter(vec![0u64, 1, 3]);
+        assert_arrays_eq!(chunk_offsets, expected_offsets);
 
         let patch_indices = patches.indices().to_primitive();
-        assert_eq!(patch_indices.as_slice::<u64>(), &[1023, 1024, 1025]);
+        let expected_indices = PrimitiveArray::from_iter(vec![1023u64, 1024, 1025]);
+        assert_arrays_eq!(patch_indices, expected_indices);
 
         let patch_values = patches.values().to_primitive();
-        assert_eq!(patch_values.as_slice::<f64>(), &[PI, E, PI]);
+        let expected_values = PrimitiveArray::from_iter(vec![PI, E, PI]);
+        assert_arrays_eq!(patch_values, expected_values);
     }
 
     #[test]
@@ -300,13 +306,16 @@ mod tests {
         let patches = encoded.patches().unwrap();
 
         let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive();
-        assert_eq!(chunk_offsets.as_slice::<u64>(), &[0, 1, 1]);
+        let expected_offsets = PrimitiveArray::from_iter(vec![0u64, 1, 1]);
+        assert_arrays_eq!(chunk_offsets, expected_offsets);
 
         let patch_indices = patches.indices().to_primitive();
-        assert_eq!(patch_indices.as_slice::<u64>(), &[0, 2048]);
+        let expected_indices = PrimitiveArray::from_iter(vec![0u64, 2048]);
+        assert_arrays_eq!(patch_indices, expected_indices);
 
         let patch_values = patches.values().to_primitive();
-        assert_eq!(patch_values.as_slice::<f64>(), &[PI, E]);
+        let expected_values = PrimitiveArray::from_iter(vec![PI, E]);
+        assert_arrays_eq!(patch_values, expected_values);
     }
 
     #[test]
@@ -319,13 +328,16 @@ mod tests {
         let patches = encoded.patches().unwrap();
 
         let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive();
-        assert_eq!(chunk_offsets.as_slice::<u64>(), &[0, 1, 1]);
+        let expected_offsets = PrimitiveArray::from_iter(vec![0u64, 1, 1]);
+        assert_arrays_eq!(chunk_offsets, expected_offsets);
 
         let patch_indices = patches.indices().to_primitive();
-        assert_eq!(patch_indices.as_slice::<u64>(), &[0]);
+        let expected_indices = PrimitiveArray::from_iter(vec![0u64]);
+        assert_arrays_eq!(patch_indices, expected_indices);
 
         let patch_values = patches.values().to_primitive();
-        assert_eq!(patch_values.as_slice::<f64>(), &[PI]);
+        let expected_values = PrimitiveArray::from_iter(vec![PI]);
+        assert_arrays_eq!(patch_values, expected_values);
     }
 
     #[test]
@@ -339,12 +351,15 @@ mod tests {
         let patches = encoded.patches().unwrap();
 
         let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive();
-        assert_eq!(chunk_offsets.as_slice::<u64>(), &[0]);
+        let expected_offsets = PrimitiveArray::from_iter(vec![0u64]);
+        assert_arrays_eq!(chunk_offsets, expected_offsets);
 
         let patch_indices = patches.indices().to_primitive();
-        assert_eq!(patch_indices.as_slice::<u64>(), &[0, 100]);
+        let expected_indices = PrimitiveArray::from_iter(vec![0u64, 100]);
+        assert_arrays_eq!(patch_indices, expected_indices);
 
         let patch_values = patches.values().to_primitive();
-        assert_eq!(patch_values.as_slice::<f64>(), &[PI, E]);
+        let expected_values = PrimitiveArray::from_iter(vec![PI, E]);
+        assert_arrays_eq!(patch_values, expected_values);
     }
 }

encodings/alp/src/alp/compress.rs

@@ -45,10 +45,10 @@ register_kernel!(CastKernelAdapter(ALPVTable).lift());
 #[cfg(test)]
 mod tests {
     use rstest::rstest;
-    use vortex_array::IntoArray;
     use vortex_array::arrays::PrimitiveArray;
     use vortex_array::compute::cast;
     use vortex_array::compute::conformance::cast::test_cast_conformance;
+    use vortex_array::{IntoArray, assert_arrays_eq};
     use vortex_buffer::buffer;
     use vortex_dtype::{DType, Nullability, PType};
 
@@ -98,7 +98,7 @@ mod tests {
         );
 
         let decoded = casted.to_canonical().into_primitive();
-        assert_eq!(decoded.as_slice::<i32>(), &[1i32, 2, 3, 4]);
+        assert_arrays_eq!(decoded, PrimitiveArray::from_iter([1i32, 2, 3, 4]));
     }
 
     #[rstest]

encodings/alp/src/alp/compute/cast.rs

@@ -127,7 +127,7 @@ pub trait ALPFloat: private::Sealed + Float + Display + NativePType {
         Buffer<Self::ALPInt>,
         Buffer<u64>,
         Buffer<Self>,
-        Buffer<u64>,
+        BufferMut<u64>,
     ) {
         let exp = exponents.unwrap_or_else(|| Self::find_best_exponents(values));
 
@@ -158,7 +158,7 @@ pub trait ALPFloat: private::Sealed + Float + Display + NativePType {
             encoded_output.freeze(),
             patch_indices.freeze(),
             patch_values.freeze(),
-            chunk_offsets.freeze(),
+            chunk_offsets,
         )
     }
 

encodings/alp/src/alp/mod.rs

@@ -35,7 +35,7 @@ impl VTable for ALPRDVTable {
     type ComputeVTable = NotSupported;
     type EncodeVTable = Self;
     type SerdeVTable = Self;
-    type PipelineVTable = NotSupported;
+    type OperatorVTable = NotSupported;
 
     fn id(_encoding: &Self::Encoding) -> EncodingId {
         EncodingId::new_ref("vortex.alprd")
@@ -265,8 +265,8 @@ impl CanonicalVTable<ALPRDVTable> for ALPRDVTable {
 #[cfg(test)]
 mod test {
     use rstest::rstest;
-    use vortex_array::ToCanonical;
     use vortex_array::arrays::PrimitiveArray;
+    use vortex_array::{ToCanonical, assert_arrays_eq};
 
     use crate::{ALPRDFloat, alp_rd};
 
@@ -294,7 +294,6 @@ mod test {
 
         let decoded = rd_array.to_primitive();
 
-        let maybe_null_reals: Vec<T> = reals.into_iter().map(|v| v.unwrap_or_default()).collect();
-        assert_eq!(decoded.as_slice::<T>(), &maybe_null_reals);
+        assert_arrays_eq!(decoded, PrimitiveArray::from_option_iter(reals));
     }
 }