Merge branch 'develop' into ngates/alp-pipeline

Author: gatesn

Cargo.lock

@@ -14,7 +14,6 @@ members = [
     "vortex-dtype",
     "vortex-duckdb",
     "vortex-error",
-    "vortex-expr",
     "vortex-ffi",
     "vortex-file",
     "vortex-flatbuffers",
@@ -230,7 +229,6 @@ vortex-decimal-byte-parts = { version = "0.1.0", path = "encodings/decimal-byte-
 vortex-dict = { version = "0.1.0", path = "./encodings/dict", default-features = false }
 vortex-dtype = { version = "0.1.0", path = "./vortex-dtype", default-features = false }
 vortex-error = { version = "0.1.0", path = "./vortex-error", default-features = false }
-vortex-expr = { version = "0.1.0", path = "./vortex-expr", default-features = false }
 vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false }
 vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false }
 vortex-flatbuffers = { version = "0.1.0", path = "./vortex-flatbuffers", default-features = false }

Cargo.toml

@@ -44,7 +44,6 @@ rand = { workspace = true }
 rstest = { workspace = true }
 vortex-alp = { path = "../alp" }
 vortex-array = { workspace = true, features = ["test-harness"] }
-vortex-expr = { workspace = true }
 vortex-fastlanes = { path = ".", features = ["test-harness"] }
 
 [features]

encodings/fastlanes/Cargo.toml

@@ -9,11 +9,11 @@ use mimalloc::MiMalloc;
 use rand::prelude::StdRng;
 use rand::{Rng, SeedableRng};
 use vortex_array::compute::{filter, warm_up_vtables};
+use vortex_array::expr::{lit, lt, root};
 use vortex_array::{Array, ArrayRef, IntoArray, ToCanonical};
 use vortex_buffer::{BitBuffer, BufferMut};
 use vortex_dtype::NativePType;
 use vortex_error::VortexResult;
-use vortex_expr::{lit, lt, root};
 use vortex_fastlanes::FoRArray;
 use vortex_fastlanes::bitpack_compress::bitpack_to_best_bit_width;
 use vortex_mask::Mask;

encodings/fastlanes/benches/pipeline_bitpacking_compare_scalar.rs

@@ -31,7 +31,6 @@ itertools = { workspace = true }
 rstest = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 vortex-array = { path = "../../vortex-array", features = ["test-harness"] }
-vortex-expr = { path = "../../vortex-expr" }
 vortex-file = { path = "../../vortex-file", features = ["tokio"] }
 vortex-layout = { path = "../../vortex-layout" }
 

encodings/sequence/Cargo.toml

@@ -27,7 +27,6 @@ vortex-btrblocks = { workspace = true }
 vortex-buffer = { workspace = true }
 vortex-dtype = { workspace = true, features = ["arbitrary"] }
 vortex-error = { workspace = true }
-vortex-expr = { workspace = true, features = ["arbitrary"] }
 vortex-file = { workspace = true, features = ["tokio", "zstd"] }
 vortex-io = { workspace = true }
 vortex-layout = { workspace = true, features = ["zstd"] }

fuzz/Cargo.toml

@@ -8,11 +8,11 @@ use itertools::Itertools;
 use libfuzzer_sys::{Corpus, fuzz_target};
 use vortex_array::arrays::ChunkedArray;
 use vortex_array::compute::{Operator, compare, filter};
+use vortex_array::expr::{lit, root};
 use vortex_array::{Array, Canonical, IntoArray, ToCanonical};
 use vortex_buffer::ByteBufferMut;
 use vortex_dtype::{DType, StructFields};
 use vortex_error::{VortexExpect, VortexUnwrap, vortex_panic};
-use vortex_expr::{lit, root};
 use vortex_file::{OpenOptionsSessionExt, WriteOptionsSessionExt, WriteStrategyBuilder};
 use vortex_fuzz::{CompressorStrategy, FuzzFileAction, RUNTIME, SESSION};
 use vortex_layout::layouts::compact::CompactCompressor;

fuzz/fuzz_targets/file_io.rs

@@ -4,8 +4,8 @@
 use libfuzzer_sys::arbitrary::{Arbitrary, Unstructured};
 use vortex_array::ArrayRef;
 use vortex_array::arrays::arbitrary::ArbitraryArray;
-use vortex_expr::Expression;
-use vortex_expr::arbitrary::{filter_expr, projection_expr};
+use vortex_array::expr::Expression;
+use vortex_array::expr::arbitrary::{filter_expr, projection_expr};
 
 use crate::array::CompressorStrategy;
 

fuzz/src/file/mod.rs

@@ -8,7 +8,7 @@ homepage = { workspace = true }
 include = { workspace = true }
 keywords = { workspace = true }
 license = { workspace = true }
-readme = { workspace = true }
+readme = "README.md"
 repository = { workspace = true }
 rust-version = { workspace = true }
 version = { workspace = true }
@@ -54,6 +54,7 @@ rand = { workspace = true }
 rstest = { workspace = true, optional = true }
 rstest_reuse = { workspace = true, optional = true }
 rustc-hash = { workspace = true }
+serde = { workspace = true, optional = true, features = ["derive"] }
 simdutf8 = { workspace = true }
 static_assertions = { workspace = true }
 tabled = { workspace = true, optional = true, default-features = false, features = [
@@ -68,6 +69,7 @@ vortex-flatbuffers = { workspace = true, features = ["array"] }
 vortex-io = { workspace = true }
 vortex-mask = { workspace = true }
 vortex-metrics = { workspace = true }
+vortex-proto = { workspace = true, features = ["expr"] }
 vortex-scalar = { workspace = true }
 vortex-session = { workspace = true }
 vortex-utils = { workspace = true }
@@ -82,6 +84,13 @@ arbitrary = [
 canonical_counter = []
 table-display = ["dep:tabled"]
 test-harness = ["dep:goldenfile", "dep:rstest", "dep:rstest_reuse"]
+serde = [
+    "dep:serde",
+    "vortex-buffer/serde",
+    "vortex-dtype/serde",
+    "vortex-error/serde",
+    "vortex-mask/serde",
+]
 
 [dev-dependencies]
 arrow-cast = { workspace = true }
@@ -126,3 +135,8 @@ harness = false
 [[bench]]
 name = "varbinview_compact"
 harness = false
+
+[[bench]]
+name = "expr_large_struct_pack"
+path = "benches/expr/large_struct_pack.rs"
+harness = false

vortex-array/Cargo.toml

@@ -1,4 +1,10 @@
-# Vortex Expressions
+# Vortex array
+
+TODO
+
+Also contains
+
+## Vortex Expressions
 
 A crate defining serializable predicate expressions. Used predominantly for filter push-down.