refactor: move schema code to kernel module

Signed-off-by: Robert Pack <[email protected]>

Author: roeap

crates/core/src/delta_datafusion/schema_adapter.rs

@@ -1,7 +1,7 @@
 use std::fmt::Debug;
 use std::sync::Arc;
 
-use crate::operations::cast::cast_record_batch;
+use crate::kernel::schema::cast_record_batch;
 use arrow_array::RecordBatch;
 use arrow_schema::{Schema, SchemaRef};
 use datafusion::common::{not_impl_err, ColumnStatistics};

crates/core/src/kernel/mod.rs

@@ -3,28 +3,20 @@
 //! The Kernel module contains all the logic for reading and processing the Delta Lake transaction log.
 
 use delta_kernel::engine::arrow_expression::ArrowEvaluationHandler;
-use std::{any::Any, sync::LazyLock};
+use std::sync::LazyLock;
 
 pub mod arrow;
 pub mod error;
 pub mod models;
 pub mod scalars;
+pub mod schema;
 mod snapshot;
 pub mod transaction;
 
 pub use error::*;
 pub use models::*;
+pub use schema::*;
 pub use snapshot::*;
 
-/// A trait for all kernel types that are used as part of data checking
-pub trait DataCheck {
-    /// The name of the specific check
-    fn get_name(&self) -> &str;
-    /// The SQL expression to use for the check
-    fn get_expression(&self) -> &str;
-
-    fn as_any(&self) -> &dyn Any;
-}
-
 static ARROW_HANDLER: LazyLock<ArrowEvaluationHandler> =
     LazyLock::new(|| ArrowEvaluationHandler {});

crates/core/src/kernel/models/actions.rs

@@ -8,9 +8,8 @@ use maplit::hashset;
 use serde::{Deserialize, Serialize};
 use tracing::warn;
 
-use super::schema::StructType;
-use super::StructTypeExt;
 use crate::kernel::{error::Error, DeltaResult};
+use crate::kernel::{StructType, StructTypeExt};
 use crate::TableProperty;
 
 /// Defines a file format used in table

crates/core/src/kernel/models/mod.rs

@@ -9,10 +9,8 @@ use serde::{Deserialize, Serialize};
 
 pub(crate) mod actions;
 pub(crate) mod fields;
-mod schema;
 
 pub use actions::*;
-pub use schema::*;
 
 #[derive(Debug, Hash, PartialEq, Eq, Clone, Serialize, Deserialize)]
 /// The type of action that was performed on the table

crates/core/src/operations/cast/merge_schema.rs renamed to crates/core/src/kernel/schema/cast/merge_schema.rs

@@ -9,7 +9,8 @@ use arrow_cast::{cast_with_options, CastOptions};
 use arrow_schema::{ArrowError, DataType, FieldRef, Fields, SchemaRef as ArrowSchemaRef};
 use std::sync::Arc;
 
-pub(crate) mod merge_schema;
+mod merge_schema;
+pub(crate) use merge_schema::*;
 
 use crate::DeltaResult;
 
@@ -203,7 +204,6 @@ mod tests {
     use std::ops::Deref;
     use std::sync::Arc;
 
-    use super::merge_schema::{merge_arrow_schema, merge_delta_struct};
     use arrow::array::types::Int32Type;
     use arrow::array::{
         new_empty_array, new_null_array, Array, ArrayData, ArrayRef, AsArray, Int32Array,
@@ -215,11 +215,12 @@ mod tests {
     use delta_kernel::schema::MetadataValue;
     use itertools::Itertools;
 
+    use super::merge_schema::{merge_arrow_schema, merge_delta_struct};
+    use super::{cast_record_batch, is_cast_required};
     use crate::kernel::{
         ArrayType as DeltaArrayType, DataType as DeltaDataType, StructField as DeltaStructField,
         StructType as DeltaStructType,
     };
-    use crate::operations::cast::{cast_record_batch, is_cast_required};
 
     #[test]
     fn test_merge_arrow_schema_with_dict() {

crates/core/src/operations/cast/mod.rs renamed to crates/core/src/kernel/schema/cast/mod.rs

@@ -0,0 +1,18 @@
+use std::any::Any;
+
+pub mod cast;
+pub mod partitions;
+mod schema;
+
+pub use cast::*;
+pub use schema::*;
+
+/// A trait for all kernel types that are used as part of data checking
+pub trait DataCheck {
+    /// The name of the specific check
+    fn get_name(&self) -> &str;
+    /// The SQL expression to use for the check
+    fn get_expression(&self) -> &str;
+
+    fn as_any(&self) -> &dyn Any;
+}

crates/core/src/kernel/schema/mod.rs

@@ -6,8 +6,9 @@ use std::convert::TryFrom;
 use delta_kernel::expressions::Scalar;
 use serde::{Serialize, Serializer};
 
+use super::{DataType, PrimitiveType};
 use crate::errors::DeltaTableError;
-use crate::kernel::{scalars::ScalarExt, DataType, PrimitiveType};
+use crate::kernel::scalars::ScalarExt;
 
 /// A special value used in Hive to represent the null partition in partitioned tables
 pub const NULL_PARTITION_VALUE_DATA_PATH: &str = "__HIVE_DEFAULT_PARTITION__";

crates/core/src/schema/partitions.rs renamed to crates/core/src/kernel/schema/partitions.rs

@@ -10,7 +10,7 @@ pub use delta_kernel::schema::{
 use serde_json::Value;
 
 use crate::kernel::error::Error;
-use crate::kernel::DataCheck;
+use crate::schema::DataCheck;
 use crate::table::GeneratedColumn;
 
 /// Type alias for a top level schema

crates/core/src/kernel/models/schema.rs renamed to crates/core/src/kernel/schema/schema.rs

@@ -75,7 +75,7 @@ pub mod kernel;
 pub mod logstore;
 pub mod operations;
 pub mod protocol;
-pub mod schema;
+pub use kernel::schema;
 pub mod table;
 
 #[cfg(any(test, feature = "integration_test"))]