draft for reaching consensus

Author: klion26

parquet-variant-compute/src/cast_to_variant.rs

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use crate::cast_conversion;
 use crate::{VariantArray, VariantArrayBuilder};
 use arrow::array::{Array, AsArray};
 use arrow::datatypes::{
@@ -27,6 +28,7 @@ use parquet_variant::Variant;
 
 /// Convert the input array of a specific primitive type to a `VariantArray`
 /// row by row
+#[macro_export]
 macro_rules! primitive_conversion {
     ($t:ty, $input:expr, $builder:expr) => {{
         let array = $input.as_primitive::<$t>();
@@ -39,23 +41,28 @@ macro_rules! primitive_conversion {
         }
     }};
 }
-
-/// Convert the input array to a `VariantArray` row by row, using `method`
-/// to downcast the generic array to a specific array type and `cast_fn`
-/// to transform each element to a type compatible with Variant
-macro_rules! cast_conversion {
-    ($t:ty, $method:ident, $cast_fn:expr, $input:expr, $builder:expr) => {{
-        let array = $input.$method::<$t>();
-        for i in 0..array.len() {
-            if array.is_null(i) {
-                $builder.append_null();
-                continue;
-            }
-            let cast_value = $cast_fn(array.value(i));
-            $builder.append_variant(Variant::from(cast_value));
-        }
-    }};
-}
+//
+// /// Convert the input array to a `VariantArray` row by row, using `method`
+// /// to downcast the generic array to a specific array type and `cast_fn`
+// /// to transform each element to a type compatible with Variant
+// macro_rules! cast_conversion {
+//     ($t:ty, $method:ident, $cast_fn:expr, $input:expr, builder => $builder:expr) => {{
+//         let array = $input.$method::<$t>();
+//         for i in 0..array.len() {
+//             if array.is_null(i) {
+//                 $builder.append_null();
+//                 continue;
+//             }
+//             let cast_value = $cast_fn(array.value(i));
+//             $builder.append_variant(Variant::from(cast_value));
+//         }
+//     }};
+//     ($t:ty, $method:ident, $cast_fn:expr, $input:expr, index => $index:expr) => {{
+//         let array = $input.$method::<$t>();
+//         let cast_value = $cast_fn(array.value($index));
+//         Variant::from(cast_value)
+//     }};
+// }
 
 /// Casts a typed arrow [`Array`] to a [`VariantArray`]. This is useful when you
 /// need to convert a specific data type
@@ -87,13 +94,13 @@ pub fn cast_to_variant(input: &dyn Array) -> Result<VariantArray, ArrowError> {
     // todo: handle other types like Boolean, Strings, Date, Timestamp, etc.
     match input_type {
         DataType::Binary => {
-            cast_conversion!(BinaryType, as_bytes, |v| v, input, builder);
+            cast_conversion!(BinaryType, as_bytes, |v| v, input, builder => builder);
         }
         DataType::LargeBinary => {
-            cast_conversion!(LargeBinaryType, as_bytes, |v| v, input, builder);
+            cast_conversion!(LargeBinaryType, as_bytes, |v| v, input, builder => builder);
         }
         DataType::BinaryView => {
-            cast_conversion!(BinaryViewType, as_byte_view, |v| v, input, builder);
+            cast_conversion!(BinaryViewType, as_byte_view, |v| v, input, builder => builder);
         }
         DataType::Int8 => {
             primitive_conversion!(Int8Type, input, builder);
@@ -125,7 +132,7 @@ pub fn cast_to_variant(input: &dyn Array) -> Result<VariantArray, ArrowError> {
                 as_primitive,
                 |v: f16| -> f32 { v.into() },
                 input,
-                builder
+                builder => builder
             );
         }
         DataType::Float32 => {

parquet-variant-compute/src/lib.rs

@@ -47,3 +47,26 @@ pub use variant_array_builder::{VariantArrayBuilder, VariantArrayVariantBuilder}
 
 pub use from_json::batch_json_string_to_variant;
 pub use to_json::batch_variant_to_json_string;
+
+/// Convert the input array to a `VariantArray` row by row, using `method`
+/// to downcast the generic array to a specific array type and `cast_fn`
+/// to transform each element to a type compatible with Variant
+#[macro_export]
+macro_rules! cast_conversion {
+    ($t:ty, $method:ident, $cast_fn:expr, $input:expr, builder => $builder:expr) => {{
+        let array = $input.$method::<$t>();
+        for i in 0..array.len() {
+            if array.is_null(i) {
+                $builder.append_null();
+                continue;
+            }
+            let cast_value = $cast_fn(array.value(i));
+            $builder.append_variant(Variant::from(cast_value));
+        }
+    }};
+    ($t:ty, $method:ident, $cast_fn:expr, $input:expr, index => $index:expr) => {{
+        let array = $input.$method::<$t>();
+        let cast_value = $cast_fn(array.value($index));
+        Variant::from(cast_value)
+    }};
+}

parquet-variant-compute/src/variant_array.rs

@@ -19,12 +19,14 @@
 
 use arrow::array::{Array, ArrayData, ArrayRef, AsArray, BinaryViewArray, StructArray};
 use arrow::buffer::NullBuffer;
-use arrow::datatypes::Int32Type;
+use arrow::datatypes::{Int16Type, Int32Type};
 use arrow_schema::{ArrowError, DataType};
 use parquet_variant::Variant;
 use std::any::Any;
 use std::sync::Arc;
 
+use crate::cast_conversion;
+
 /// An array of Parquet [`Variant`] values
 ///
 /// A [`VariantArray`] wraps an Arrow [`StructArray`] that stores the underlying
@@ -337,6 +339,9 @@ fn typed_value_to_variant(typed_value: &ArrayRef, index: usize) -> Variant<'_, '
             let typed_value = typed_value.as_primitive::<Int32Type>();
             Variant::from(typed_value.value(index))
         }
+        DataType::Int16 => {
+            cast_conversion!(Int16Type, as_primitive, |v| v, typed_value,  index => index)
+        }
         // todo other types here (note this is very similar to cast_to_variant.rs)
         // so it would be great to figure out how to share this code
         _ => {