Initial commit

Author: grtlr

Cargo.lock

@@ -190,7 +190,7 @@ ahash = "0.8"
 anyhow = { version = "1.0", default-features = false }
 argh = "0.1.12"
 array-init = "2.1"
-arrow = { version = "55.2", default-features = false }
+arrow = { version = "56.1", default-features = false }
 async-stream = "0.3"
 axum = "0.8.4"
 backtrace = "0.3"
@@ -219,10 +219,10 @@ convert_case = "0.6"
 criterion = "0.5"
 cros-codecs = "0.0.6"
 crossbeam = "0.8"
-datafusion = { version = "47", default-features = false, features = [
+datafusion = { version = "50", default-features = false, features = [
   "nested_expressions",
 ] }
-datafusion-ffi = "47"
+datafusion-ffi = "49.0.2"
 directories = "5"
 document-features = "0.2.8"
 econtext = "0.2" # Prints error contexts on crashes
@@ -275,15 +275,15 @@ nohash-hasher = "0.2"
 notify = { version = "6.1.1", features = ["macos_kqueue"] }
 num-derive = "0.4"
 num-traits = "0.2"
-numpy = "0.24"
+numpy = "0.25"
 objc2-app-kit = "0.3"
 opentelemetry = { version = "0.30", features = ["metrics"] }
 opentelemetry-appender-tracing = "0.30"
 opentelemetry-otlp = "0.30"
 opentelemetry_sdk = { version = "0.30", features = ["rt-tokio"] }
 ordered-float = "4.3.0"
 parking_lot = "0.12.3"
-parquet = { version = "55.1", default-features = false }
+parquet = { version = "56.1", default-features = false }
 paste = "1.0"
 pathdiff = "0.2"
 percent-encoding = "2.3"
@@ -301,8 +301,8 @@ prost-types = "0.13.3"
 prost-reflect = "0.15.3"
 puffin = "0.19.1"
 puffin_http = "0.16"
-pyo3 = "0.24.1"
-pyo3-build-config = "0.24.1"
+pyo3 = "0.25.1"
+pyo3-build-config = "0.25.1"
 quote = "1.0"
 rand = { version = "0.8", default-features = false, features = ["small_rng"] }
 rand_distr = { version = "0.4", default-features = false }

Cargo.toml

@@ -223,8 +223,8 @@ impl DataframeQueryTableProvider {
             }
             Expr::BinaryExpr(binary) => {
                 if binary.op == Operator::NotEq
-                    && let (Expr::Column(col), Expr::Literal(sv))
-                    | (Expr::Literal(sv), Expr::Column(col)) =
+                    && let (Expr::Column(col), Expr::Literal(sv, _))
+                    | (Expr::Literal(sv, _), Expr::Column(col)) =
                         (binary.left.as_ref(), binary.right.as_ref())
                     && sv.is_null()
                 {

crates/store/re_datafusion/src/dataframe_query_common.rs

@@ -233,13 +233,16 @@ impl PartitionStreamExec {
                     SortOptions::new(false, true),
                 ));
             }
-            vec![LexOrdering::new(physical_ordering)]
+            vec![
+                LexOrdering::new(physical_ordering)
+                    .expect("LexOrdering should return Some since input is not empty"),
+            ]
         } else {
             vec![]
         };
 
         let eq_properties =
-            EquivalenceProperties::new_with_orderings(Arc::clone(&projected_schema), &orderings);
+            EquivalenceProperties::new_with_orderings(Arc::clone(&projected_schema), orderings);
 
         let partition_in_output_schema = projection.map(|p| p.contains(&0)).unwrap_or(false);
 

crates/store/re_datafusion/src/dataframe_query_provider.rs

@@ -232,10 +232,13 @@ impl PartitionStreamExec {
             ));
         }
 
-        let orderings = vec![LexOrdering::new(physical_ordering)];
+        let orderings = vec![
+            LexOrdering::new(physical_ordering)
+                .expect("LexOrdering should return Some when non-empty vec is passed"),
+        ];
 
         let eq_properties =
-            EquivalenceProperties::new_with_orderings(Arc::clone(&projected_schema), &orderings);
+            EquivalenceProperties::new_with_orderings(Arc::clone(&projected_schema), orderings);
 
         let partition_in_output_schema = projection.map(|p| p.contains(&0)).unwrap_or(false);
 

crates/store/re_datafusion/src/dataframe_query_provider_wasm.rs

@@ -224,6 +224,8 @@ pub fn generic_placeholder_for_datatype(
             ))
         }
 
+        DataType::Decimal32(_, _) => Arc::new(array::Decimal32Array::from_iter([0])),
+        DataType::Decimal64(_, _) => Arc::new(array::Decimal64Array::from_iter([0])),
         DataType::Decimal128(_, _) => Arc::new(array::Decimal128Array::from_iter([0])),
 
         DataType::Decimal256(_, _) => Arc::new(array::Decimal256Array::from_iter([

crates/store/re_types_core/src/reflection.rs

@@ -110,6 +110,8 @@ impl std::fmt::Display for DisplayDatatype<'_> {
             DataType::Dictionary(key, value) => {
                 return write!(f, "Dictionary{{{}: {}}}", Self(key), Self(value));
             }
+            DataType::Decimal32(_, _) => "Decimal32",
+            DataType::Decimal64(_, _) => "Decimal64",
             DataType::Decimal128(_, _) => "Decimal128",
             DataType::Decimal256(_, _) => "Decimal256",
             DataType::BinaryView => "BinaryView",

crates/utils/re_arrow_util/src/format_data_type.rs

@@ -106,6 +106,8 @@ impl SizeBytes for DataType {
             | Self::Utf8
             | Self::LargeUtf8
             | Self::BinaryView
+            | Self::Decimal32(_, _)
+            | Self::Decimal64(_, _)
             | Self::Decimal128(_, _)
             | Self::Decimal256(_, _)
             | Self::FixedSizeBinary(_)

crates/utils/re_byte_size/src/arrow_sizes.rs

@@ -160,7 +160,7 @@ fn make_ui<'a>(
         | DataType::Utf8View | DataType::BinaryView
         | DataType::Date32 | DataType::Date64 | DataType::Time32(_) | DataType::Time64(_)
         | DataType::Timestamp(_, _) | DataType::Duration(_) | DataType::Interval(_)
-        | DataType::Decimal128(_, _) | DataType::Decimal256(_, _)
+        | DataType::Decimal32(_,_) | DataType::Decimal64(_,_) | DataType::Decimal128(_, _) | DataType::Decimal256(_, _)
         => {
             show_arrow_builtin(array, options)
         }

crates/viewer/re_arrow_ui/src/show_index.rs

@@ -29,7 +29,6 @@ all-features = true
 [advisories]
 version = 2
 ignore = [
-  "RUSTSEC-2024-0384", # Waiting for https://github.com/console-rs/indicatif/pull/666
   "RUSTSEC-2024-0436", # https://rustsec.org/advisories/RUSTSEC-2024-0436 - paste is unmaintained - https://github.com/dtolnay/paste
   "RUSTSEC-2024-0014", # https://rustsec.org/advisories/RUSTSEC-2024-0014 - generational-arena is unmaintained
 ]
@@ -54,6 +53,7 @@ deny = [
 skip = [
   { name = "base64" },           # Too popular
   { name = "block2" },           # Old version via rfd
+  { name = "bzip2" },            # Remove after https://github.com/apache/datafusion/pull/17509 closes
   { name = "cargo-platform" },   # Older version used by ply-rs. It's build-time only!
   { name = "cargo_metadata" },   # Older version used by ply-rs. It's small, and it's build-time only!
   { name = "core-foundation" },  # Currently, e.g. `webbrowser` and `winit` use different versions.
@@ -63,7 +63,7 @@ skip = [
   { name = "objc2-foundation" }, # `accesskit_macos` uses a different version than `arboard`
   { name = "objc2" },            # `accesskit_macos` uses a different version than `arboard`
   { name = "ordered-float" },    # Old version being used by parquet, but super small!
-  { name = "pollster" },         # rfd is still on 0.3
+  { name = "petgraph" },         # Remove after https://github.com/tokio-rs/prost/pull/1268 resolves
   { name = "pulldown-cmark" },   # Build-dependency via `ply-rs` (!). TODO(emilk): use a better crate for .ply parsing
   { name = "redox_syscall" },    # Plenty of versions in the wild
   { name = "rustc-hash" },       # numpy with compatible pyo3 requires different version than wgpu
@@ -95,8 +95,8 @@ allow = [
   "OFL-1.1",                        # https://spdx.org/licenses/OFL-1.1.html
   "Ubuntu-font-1.0",                # https://ubuntu.com/legal/font-licence
   "Unicode-3.0",                    # https://www.unicode.org/license.txt
-  "Unicode-DFS-2016",               # https://spdx.org/licenses/Unicode-DFS-2016.html
   "Zlib",                           # https://tldrlegal.com/license/zlib-libpng-license-(zlib)
+  "bzip2-1.0.6",                    # https://github.com/trifectatechfoundation/libbzip2-rs/blob/v0.2.2/COPYING
 ]
 exceptions = []
 
@@ -110,7 +110,6 @@ name = "ring"
 expression = "MIT AND ISC AND OpenSSL"
 license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
 
-
 [sources]
 unknown-registry = "deny"
 unknown-git = "deny"