Move RowFilter, PruningPredicate, and expr_project to vortex-expr (#1820)

Author: gatesn

pyvortex/src/dataset.rs

@@ -10,9 +10,10 @@ use vortex::arrow::infer_schema;
 use vortex::dtype::field::Field;
 use vortex::dtype::DType;
 use vortex::error::VortexResult;
+use vortex::expr::RowFilter;
 use vortex::file::{
-    read_initial_bytes, LayoutContext, LayoutDeserializer, Projection, RowFilter,
-    VortexReadArrayStream, VortexReadBuilder, VortexRecordBatchReader,
+    read_initial_bytes, LayoutContext, LayoutDeserializer, Projection, VortexReadArrayStream,
+    VortexReadBuilder, VortexRecordBatchReader,
 };
 use vortex::io::{ObjectStoreReadAt, TokioFile, VortexReadAt};
 use vortex::sampling_compressor::ALL_ENCODINGS_CONTEXT;

vortex-datafusion/src/persistent/opener.rs

@@ -9,7 +9,8 @@ use futures::{FutureExt as _, StreamExt, TryStreamExt};
 use object_store::ObjectStore;
 use vortex_array::ContextRef;
 use vortex_expr::datafusion::convert_expr_to_vortex;
-use vortex_file::{LayoutContext, LayoutDeserializer, Projection, RowFilter, VortexReadBuilder};
+use vortex_expr::RowFilter;
+use vortex_file::{LayoutContext, LayoutDeserializer, Projection, VortexReadBuilder};
 use vortex_io::{IoDispatcher, ObjectStoreReadAt};
 
 use super::cache::InitialReadCache;

vortex-expr/src/lib.rs

@@ -12,6 +12,9 @@ mod like;
 mod literal;
 mod not;
 mod operators;
+mod project;
+pub mod pruning;
+mod row_filter;
 mod select;
 
 pub use binary::*;
@@ -21,6 +24,8 @@ pub use like::*;
 pub use literal::*;
 pub use not::*;
 pub use operators::*;
+pub use project::*;
+pub use row_filter::*;
 pub use select::*;
 use vortex_array::ArrayData;
 use vortex_dtype::field::Field;

vortex-file/src/read/expr_project.rs vortex-expr/src/project.rsvortex-file/src/read/expr_project.rs renamed to vortex-expr/src/project.rs

@@ -1,11 +1,11 @@
 use std::sync::Arc;
 
 use vortex_dtype::field::Field;
-use vortex_expr::{
-    BinaryExpr, Column, ExprRef, Identity, Like, Literal, Not, Operator, Select, VortexExpr,
-};
 
-use crate::RowFilter;
+use crate::{
+    BinaryExpr, Column, ExprRef, Identity, Like, Literal, Not, Operator, RowFilter, Select,
+    VortexExpr,
+};
 
 /// Restrict expression to only the fields that appear in projection
 ///
@@ -100,9 +100,9 @@ mod tests {
     use std::sync::Arc;
 
     use vortex_dtype::field::Field;
-    use vortex_expr::{BinaryExpr, Column, Identity, Literal, Not, Operator, Select, VortexExpr};
 
-    use crate::read::expr_project::expr_project;
+    use super::*;
+    use crate::{BinaryExpr, Column, Identity, Literal, Not, Operator, Select, VortexExpr};
 
     #[test]
     fn project_and() {

vortex-file/src/pruning.rs vortex-expr/src/pruning.rsvortex-file/src/pruning.rs renamed to vortex-expr/src/pruning.rs

@@ -12,10 +12,9 @@ use vortex_array::ArrayData;
 use vortex_dtype::field::Field;
 use vortex_dtype::Nullability;
 use vortex_error::{VortexExpect as _, VortexResult};
-use vortex_expr::{BinaryExpr, Column, ExprRef, Identity, Literal, Not, Operator};
 use vortex_scalar::Scalar;
 
-use crate::RowFilter;
+use crate::{BinaryExpr, Column, ExprRef, Identity, Literal, Not, Operator, RowFilter};
 
 #[derive(Debug, Clone)]
 pub struct Relation<K, V> {
@@ -34,6 +33,12 @@ impl<K: Display, V: Display> Display for Relation<K, V> {
     }
 }
 
+impl<K: Hash + Eq, V: Hash + Eq> Default for Relation<K, V> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl<K: Hash + Eq, V: Hash + Eq> Relation<K, V> {
     pub fn new() -> Self {
         Relation {
@@ -441,11 +446,11 @@ mod tests {
     use vortex_array::aliases::hash_set::HashSet;
     use vortex_array::stats::Stat;
     use vortex_dtype::field::Field;
-    use vortex_expr::{BinaryExpr, Column, Identity, Literal, Not, Operator};
 
     use crate::pruning::{
         convert_to_pruning_expression, stat_column_field, FieldOrIdentity, PruningPredicate,
     };
+    use crate::{BinaryExpr, Column, Identity, Literal, Not, Operator};
 
     #[test]
     pub fn pruning_equals() {

vortex-file/src/read/filtering.rs vortex-expr/src/row_filter.rsvortex-file/src/read/filtering.rs renamed to vortex-expr/src/row_filter.rs

@@ -10,9 +10,8 @@ use vortex_array::stats::ArrayStatistics;
 use vortex_array::{ArrayData, IntoArrayData};
 use vortex_dtype::field::Field;
 use vortex_error::{VortexExpect, VortexResult};
-use vortex_expr::{split_conjunction, unbox_any, ExprRef, VortexExpr};
 
-use crate::read::expr_project::expr_project;
+use crate::{expr_project, split_conjunction, unbox_any, ExprRef, VortexExpr};
 
 #[derive(Debug, Clone)]
 pub struct RowFilter {

vortex-file/src/lib.rs

@@ -70,7 +70,6 @@ mod read;
 mod write;
 
 mod byte_range;
-mod pruning;
 #[cfg(test)]
 mod tests;
 

vortex-file/src/read/builder/mod.rs

@@ -3,14 +3,13 @@ use std::sync::Arc;
 use initial_read::read_initial_bytes;
 use vortex_array::{ArrayDType, ArrayData};
 use vortex_error::VortexResult;
-use vortex_expr::Select;
+use vortex_expr::{RowFilter, Select};
 use vortex_io::{IoDispatcher, VortexReadAt};
 
 use super::handle::VortexReadHandle;
 use super::InitialRead;
 use crate::read::cache::LayoutMessageCache;
 use crate::read::context::LayoutDeserializer;
-use crate::read::filtering::RowFilter;
 use crate::read::projection::Projection;
 use crate::read::{RowMask, Scan};
 use crate::LayoutPath;

vortex-file/src/read/layouts/chunked.rs

@@ -11,11 +11,11 @@ use vortex_buffer::Buffer;
 use vortex_dtype::field::Field;
 use vortex_dtype::{DType, Nullability, StructDType};
 use vortex_error::{vortex_bail, vortex_err, vortex_panic, VortexExpect as _, VortexResult};
+use vortex_expr::pruning::PruningPredicate;
 use vortex_expr::Select;
 use vortex_flatbuffers::footer as fb;
 
 use crate::layouts::RangedLayoutReader;
-use crate::pruning::PruningPredicate;
 use crate::read::mask::RowMask;
 use crate::{
     Layout, LayoutDeserializer, LayoutId, LayoutPartId, LayoutPath, LayoutReader, LazyDType,
@@ -429,7 +429,7 @@ mod tests {
     use vortex_array::{ArrayDType, ArrayLen, IntoArrayData, IntoArrayVariant};
     use vortex_buffer::Buffer;
     use vortex_dtype::PType;
-    use vortex_expr::{BinaryExpr, Identity, Literal, Operator};
+    use vortex_expr::{BinaryExpr, Identity, Literal, Operator, RowFilter};
     use vortex_flatbuffers::{footer, WriteFlatBuffer};
     use vortex_ipc::messages::{AsyncMessageWriter, EncoderMessage};
 
@@ -438,7 +438,7 @@ mod tests {
     use crate::read::cache::LazyDType;
     use crate::read::layouts::test_read::{filter_read_layout, read_layout, read_layout_data};
     use crate::read::mask::RowMask;
-    use crate::{write, LayoutDeserializer, LayoutMessageCache, LayoutPath, RowFilter, Scan};
+    use crate::{write, LayoutDeserializer, LayoutMessageCache, LayoutPath, Scan};
 
     async fn layout_and_bytes(
         scan: Scan,

vortex-file/src/read/layouts/columnar.rs

@@ -10,15 +10,14 @@ use vortex_array::{ArrayData, IntoArrayData};
 use vortex_dtype::field::Field;
 use vortex_dtype::{FieldName, FieldNames};
 use vortex_error::{vortex_bail, vortex_err, vortex_panic, VortexExpect, VortexResult};
-use vortex_expr::{Column, Select, VortexExpr};
+use vortex_expr::{expr_project, Column, RowFilter, Select, VortexExpr};
 use vortex_flatbuffers::footer;
 
 use crate::read::cache::LazyDType;
-use crate::read::expr_project::expr_project;
 use crate::read::mask::RowMask;
 use crate::{
     Layout, LayoutDeserializer, LayoutId, LayoutPath, LayoutReader, MessageCache, PollRead, Prune,
-    RowFilter, Scan, COLUMNAR_LAYOUT_ID,
+    Scan, COLUMNAR_LAYOUT_ID,
 };
 
 #[derive(Debug)]
@@ -426,13 +425,12 @@ mod tests {
     use vortex_buffer::Buffer;
     use vortex_dtype::field::Field;
     use vortex_dtype::{DType, Nullability};
-    use vortex_expr::{BinaryExpr, Column, Literal, Operator};
+    use vortex_expr::{BinaryExpr, Column, Literal, Operator, RowFilter};
 
     use crate::read::builder::initial_read::read_initial_bytes;
     use crate::read::layouts::test_read::{filter_read_layout, read_layout};
     use crate::{
-        LayoutDeserializer, LayoutMessageCache, LayoutPath, LayoutReader, RowFilter, Scan,
-        VortexFileWriter,
+        LayoutDeserializer, LayoutMessageCache, LayoutPath, LayoutReader, Scan, VortexFileWriter,
     };
 
     async fn layout_and_bytes(