fix: cast list of structs and other cast fixes

Author: comphead

native/core/src/execution/planner.rs

@@ -328,7 +328,6 @@ impl PhysicalPlanner {
                     )));
                 }
                 let field = input_schema.field(idx);
-                //dbg!(field);
                 Ok(Arc::new(Column::new(field.name().as_str(), idx)))
             }
             ExprStruct::Unbound(unbound) => {
@@ -1092,8 +1091,6 @@ impl PhysicalPlanner {
                 let data_schema = convert_spark_types_to_arrow_schema(scan.data_schema.as_slice());
                 let required_schema: SchemaRef =
                     convert_spark_types_to_arrow_schema(scan.required_schema.as_slice());
-                // dbg!(&data_schema);
-                // dbg!(&required_schema);
                 let partition_schema: SchemaRef =
                     convert_spark_types_to_arrow_schema(scan.partition_schema.as_slice());
                 let projection_vector: Vec<usize> = scan

native/core/src/parquet/parquet_support.rs

@@ -158,11 +158,9 @@ fn cast_array(
     };
     let from_type = array.data_type();
 
+    // Try Comet specific handlers first, then arrow-rs cast if supported,
+    // return uncasted data otherwise
     match (from_type, to_type) {
-        // If Arrow cast supports the cast, delegate the cast to Arrow
-        _ if can_cast_types(from_type, to_type) => {
-            Ok(cast_with_options(&array, to_type, &PARQUET_OPTIONS)?)
-        }
         (Struct(_), Struct(_)) => Ok(cast_struct_to_struct(
             array.as_struct(),
             from_type,
@@ -180,6 +178,10 @@ fn cast_array(
                     .with_timezone(Arc::clone(tz)),
             ))
         }
+        // If Arrow cast supports the cast, delegate the cast to Arrow
+        _ if can_cast_types(from_type, to_type) => {
+            Ok(cast_with_options(&array, to_type, &PARQUET_OPTIONS)?)
+        }
         _ => Ok(array),
     }
 }
@@ -192,11 +194,6 @@ fn cast_list_to_list(
 ) -> DataFusionResult<ArrayRef> {
     match (from_type, to_type) {
         (DataType::List(_), DataType::List(to_inner_type)) => {
-            //dbg!(from_type);
-            //dbg!(to_type);
-            //dbg!(from_inner_type);
-            //dbg!(to_inner_type);
-
             let cast_field = cast_array(
                 Arc::clone(array.values()),
                 to_inner_type.data_type(),
@@ -316,6 +313,8 @@ pub(crate) fn prepare_object_store(
 #[cfg(test)]
 mod tests {
     use crate::parquet::parquet_support::prepare_object_store;
+    use arrow::compute::can_cast_types;
+    use arrow::datatypes::DataType;
     use datafusion::execution::object_store::ObjectStoreUrl;
     use datafusion::execution::runtime_env::RuntimeEnv;
     use object_store::path::Path;

native/spark-expr/src/struct_funcs/get_struct_field.rs

@@ -81,8 +81,6 @@ impl PhysicalExpr for GetStructField {
 
     fn evaluate(&self, batch: &RecordBatch) -> DataFusionResult<ColumnarValue> {
         let child_value = self.child.evaluate(batch)?;
-        //dbg!(&batch);
-        //dbg!(&child_value);
 
         match child_value {
             ColumnarValue::Array(array) => {

spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala

@@ -1962,11 +1962,6 @@ object QueryPlanSerde extends Logging with CometExprShim {
         val childExpr = exprToProtoInternal(child, inputs, binding)
         val ordinalExpr = exprToProtoInternal(ordinal, inputs, binding)
 
-//        val childExpr = exprToProtoInternal(child, inputs, binding)
-//        val ordinalExpr =
-//          exprToProtoInternal(Add(Cast(ordinal, LongType), Literal(1L)), inputs, binding)
-//        scalarFunctionExprToProto("array_element", childExpr, ordinalExpr)
-
         if (childExpr.isDefined && ordinalExpr.isDefined) {
           val listExtractBuilder = ExprOuterClass.ListExtract
             .newBuilder()