Merge vaccum_fab into vacuum

Author: ljeub-pometry

Cargo.lock

@@ -159,6 +159,7 @@ parquet = { version = "56.2.0" }
 arrow-json = { version = "56.2.0" }
 arrow-buffer = { version = "56.2.0" }
 arrow-schema = { version = "56.2.0" }
+serde_arrow = {version = "0.13.6", features = ["arrow-56"]}
 arrow-array = { version = "56.2.0", features = ["chrono-tz"] }
 arrow-ipc = { version = "56.2.0" }
 arrow-csv = { version = "56.2.0" }

Cargo.toml

@@ -28,6 +28,7 @@ arrow.workspace = true
 arrow-array.workspace = true
 arrow-csv.workspace = true
 arrow-schema.workspace = true
+serde_arrow.workspace = true
 parquet.workspace = true
 bytemuck.workspace = true
 rayon.workspace = true

db4-storage/Cargo.toml

@@ -297,7 +297,7 @@ pub fn check_graph_with_nodes_support<
                 let actual_prop = actual_props
                     .unwrap_or_else(|| panic!("Failed to get prop {name} for {node:?}"));
                 assert!(
-                    const_props.contains(&actual_prop),
+                    const_props.iter().any(|c_prop| c_prop == &actual_prop),
                     "failed to get const prop {name} for {node:?}, expected {const_props:?}, got {actual_prop:?}"
                 );
             }

db4-storage/src/pages/test_utils/checkers.rs

@@ -99,7 +99,7 @@ pub fn edges_strat(size: usize) -> impl Strategy<Value = Vec<(VID, VID)>> {
         let srcs = (0usize..num_nodes).prop_map(VID);
         let dsts = (0usize..num_nodes).prop_map(VID);
         num_edges.prop_flat_map(move |num_edges| {
-            collection::vec((srcs.clone(), dsts.clone()), num_edges as usize)
+            collection::vec((srcs.clone(), dsts.clone()), num_edges)
         })
     })
 }
@@ -121,19 +121,16 @@ pub fn edges_strat_with_layers(
     })
 }
 
-pub fn build_raw_edges(
-    len: usize,
-    num_nodes: usize,
-) -> impl Strategy<
-    Value = Vec<(
-        VID,
-        VID,
-        i64,
-        Vec<(String, Prop)>,
-        Vec<(String, Prop)>,
-        Option<&'static str>,
-    )>,
-> {
+pub type EdgeValues = (
+    VID,
+    VID,
+    i64,
+    Vec<(String, Prop)>,
+    Vec<(String, Prop)>,
+    Option<&'static str>,
+);
+
+pub fn build_raw_edges(len: usize, num_nodes: usize) -> impl Strategy<Value = Vec<EdgeValues>> {
     proptest::collection::hash_map((0i32..1000).prop_map(|i| i.to_string()), prop_type(), 0..20)
         .prop_flat_map(move |schema| {
             let (t_props, c_props) = make_props(&schema);

db4-storage/src/pages/test_utils/fixtures.rs

@@ -19,15 +19,15 @@ pub fn prop_type() -> impl Strategy<Value = PropType> {
         PropType::Decimal { scale: 7 }, // decimal breaks the tests because of polars-parquet
     ]);
 
-    // leaf.prop_recursive(3, 10, 10, |inner| {
-    //     let dict = proptest::collection::hash_map(r"\w{1,10}", inner.clone(), 1..10)
-    //         .prop_map(|map| PropType::map(map));
-    //     let list = inner
-    //         .clone()
-    //         .prop_map(|p_type| PropType::List(Box::new(p_type)));
-    //     prop_oneof![inner, list, dict]
-    // })
-    leaf
+    leaf.prop_recursive(3, 10, 10, |inner| {
+        let dict = proptest::collection::hash_map(r"\w{1,10}", inner.clone(), 1..10)
+            .prop_map(PropType::map);
+        // let list = inner
+        //     .clone()
+        //     .prop_map(|p_type| PropType::List(Box::new(p_type)));
+        // prop_oneof![inner, list, dict]
+        prop_oneof![inner, dict]
+    })
 }
 
 pub fn make_props(

db4-storage/src/pages/test_utils/props.rs

@@ -7,7 +7,7 @@ use arrow_schema::DECIMAL128_MAX_PRECISION;
 use bigdecimal::ToPrimitive;
 use raphtory_api::core::entities::properties::{
     meta::PropMapper,
-    prop::{Prop, PropType},
+    prop::{Prop, PropType, SerdeMap, arrow_dtype_from_prop_type, struct_array_from_props},
 };
 use raphtory_core::{
     entities::{
@@ -197,7 +197,20 @@ impl Properties {
             }
             // PropColumn::Array(lazy_vec) => todo!(),
             // PropColumn::List(lazy_vec) => todo!(),
-            // PropColumn::Map(lazy_vec) => todo!(),
+            PropColumn::Map(lazy_vec) => {
+                let dt = meta
+                    .get_dtype(col_id)
+                    .as_ref()
+                    .map(arrow_dtype_from_prop_type)
+                    .unwrap();
+                let array_iter = indices
+                    .map(|i| lazy_vec.get_opt(i))
+                    .map(|e| e.map(|m| SerdeMap(m)));
+
+                let struct_array = struct_array_from_props(&dt, |sm| *sm, array_iter);
+
+                Some(Arc::new(struct_array))
+            }
             _ => None, //todo!("Unsupported column type"),
         }
     }
@@ -234,7 +247,7 @@ impl Properties {
         }
     }
 
-    pub(crate) fn t_len(&self) -> usize {
+    pub fn t_len(&self) -> usize {
         self.t_properties.len()
     }
 

db4-storage/src/properties/mod.rs

@@ -37,6 +37,7 @@ sorted_vector_map = { workspace = true }
 arrow-array = { workspace = true, optional = true }
 arrow-ipc = { workspace = true, optional = true }
 arrow-schema = { workspace = true, optional = true }
+serde_arrow = { workspace = true, optional = true }
 itertools = { workspace = true }
 iter-enum = { workspace = true }
 minijinja = { workspace = true, optional = true }
@@ -46,7 +47,7 @@ display-error-chain = { workspace = true, optional = true }
 proptest.workspace = true
 
 [features]
-default = []
+default = ["arrow"]
 # Enables generating the pyo3 python bindings
 python = [
     "dep:pyo3", "dep:pyo3-arrow", "dep:display-error-chain"
@@ -55,6 +56,6 @@ python = [
 proto = []
 vectors = []
 template = ["dep:minijinja"]
-arrow = ["dep:arrow-array", "dep:arrow-ipc", "dep:arrow-schema"]
+arrow = ["dep:arrow-array", "dep:arrow-ipc", "dep:arrow-schema", "dep:serde_arrow"]
 search = []
 io = ["dep:serde_json"]