chore: update to latest influxdb3_core (#26429)

* chore: update to latest core

* chore: allow CDLA permissive 2 license

* chore: update insta snapshot for new internal df tables

* test: update assertion in flightsql test

* fix: object store size hinting workaround in clap_blocks

Applied a workaround from upstream to strip size hinting from the object
store get request options. See:

influxdata/influxdb_iox#13771

* fix: query_executor tests use object store size hinting workaround

* fix: insta snapshot test for show system summary command

* chore: update windows- crates for advisories

* chore: update to latest sha on influxdb3_core branch

* chore: update to latest influxdb3_core rev

* refactor: pr feedback

* refactor: do not use object store size hint layer

Instead of using the ObjectStoreStripSizeHint layer, just provide the
configuration to datafusion to disable the use of size hinting from
iox_query.

This is used in IOx and not relevant to Monolith.

* fix: use parquet cache for get_opts requests

* test: that the parquet cache is being hit from write buffer

Author: hiltontj

Cargo.lock

@@ -80,8 +80,8 @@ crossbeam-channel = "0.5.11"
 csv = "1.3.0"
 # Use DataFusion fork
 # See https://github.com/influxdata/arrow-datafusion/pull/49 for contents
-datafusion = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "ae0a57b05895ccf4d2febb9c91bbb0956cf7e863" }
-datafusion-proto = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "ae0a57b05895ccf4d2febb9c91bbb0956cf7e863" }
+datafusion = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "1c10b8b635831e87cb043a1e3fa8eb89be430d54" }
+datafusion-proto = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "1c10b8b635831e87cb043a1e3fa8eb89be430d54" }
 dashmap = "6.1.0"
 dotenvy = "0.15.7"
 flate2 = "1.0.27"
@@ -137,7 +137,7 @@ sysinfo = "0.30.8"
 tempfile = "3.14.0"
 test-log = { version = "0.2.16", features = ["trace"] }
 thiserror = "1.0"
-tokio = { version = "1.43.1", features = ["full"] }
+tokio = { version = "1.45", features = ["full"] }
 tokio-util = { version = "0.7.13", features = ["rt"] }
 tonic = { version = "0.11.0", features = ["tls", "tls-roots"] }
 tonic-build = "0.11.0"
@@ -148,41 +148,40 @@ twox-hash = "2.1.0"
 unicode-segmentation = "1.11.0"
 url = "2.5.0"
 urlencoding = "1.1"
-uuid = { version = "1", features = ["v4"] }
+uuid = { version = "1", features = ["v4", "v7", "serde"] }
 num = { version = "0.4.3" }
 
 # Core.git crates we depend on
-arrow_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-authz = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-data_types = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-datafusion_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-executor = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-influxdb-line-protocol = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f"}
-influxdb_influxql_parser = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-influxdb_iox_client = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-iox_catalog = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-iox_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-iox_query = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-iox_query_params = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-iox_query_influxql = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-iox_system_tables = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-iox_time = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-metric = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-metric_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-object_store_metrics = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-observability_deps = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-panic_logging = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-parquet_file = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-schema = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f", features = ["v3"]}
-service_common = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-service_grpc_flight = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-test_helpers = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-tokio_metrics_bridge = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-trace = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-trace_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-trace_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-tracker = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f" }
-trogging = { git = "https://github.com/influxdata/influxdb3_core", rev = "26a30bf8d6e2b6b3f1dd905c4ec27e3db6e20d5f", features = ["clap"] }
+arrow_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+authz = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+data_types = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+datafusion_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+executor = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+influxdb-line-protocol = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb", features = ["v3"]}
+influxdb_influxql_parser = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+influxdb_iox_client = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+iox_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+iox_query = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+iox_query_params = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+iox_query_influxql = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+iox_system_tables = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+iox_time = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+metric = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+metric_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+object_store_metrics = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+observability_deps = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+panic_logging = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+parquet_file = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+schema = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb", features = ["v3"]}
+service_common = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+service_grpc_flight = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+test_helpers = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+tokio_metrics_bridge = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+trace = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+trace_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+trace_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+tracker = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb" }
+trogging = { git = "https://github.com/influxdata/influxdb3_core", rev = "fd0e474a6c0af5ba867399d753f5df18f59907cb", features = ["clap"] }
 
 [workspace.lints.rust]
 missing_copy_implementations = "deny"
@@ -268,3 +267,4 @@ arrow-string = { git = "https://github.com/influxdata/arrow-rs.git", rev = "eae1
 arrow-ord = { git = "https://github.com/influxdata/arrow-rs.git", rev = "eae176c21b1ef915227294e8a8a201b6f266031a" }
 arrow-flight = { git = "https://github.com/influxdata/arrow-rs.git", rev = "eae176c21b1ef915227294e8a8a201b6f266031a" }
 parquet = { git = "https://github.com/influxdata/arrow-rs.git", rev = "eae176c21b1ef915227294e8a8a201b6f266031a" }
+object_store = { git = "https://github.com/influxdata/arrow-rs.git", rev = "c946cd81fa12e6588a3be33be08e3d8e9a2770e7" }

Cargo.toml

@@ -5,9 +5,6 @@
 version = 2
 yanked = "deny"
 ignore = [
-    # dependent on datafusion-common moving away from instant
-    # https://github.com/apache/datafusion/pull/13355
-    "RUSTSEC-2024-0384",
     # paste crate is no longer maintained, but it is past 1.0
     # Keep this here until our transisent dependencies no longer
     # need it
@@ -24,6 +21,7 @@ allow = [
     "BSD-2-Clause",
     "BSD-3-Clause",
     "CC0-1.0",
+    "CDLA-Permissive-2.0",
     "ISC",
     "MIT",
     "Unicode-3.0",

deny.toml

@@ -49,7 +49,7 @@ use influxdb3_write::{
         persisted_files::PersistedFiles,
     },
 };
-use iox_query::exec::{DedicatedExecutor, Executor, ExecutorConfig};
+use iox_query::exec::{DedicatedExecutor, Executor, ExecutorConfig, PerQueryMemoryPoolConfig};
 use iox_time::SystemProvider;
 use object_store::ObjectStore;
 use object_store_metrics::ObjectStoreMetrics;
@@ -561,6 +561,7 @@ pub async fn command(config: Config) -> Result<()> {
         Arc::clone(&time_provider) as _,
         "main",
         &metrics,
+        config.object_store_config.bucket.as_ref(),
     ));
 
     // setup cached object store:
@@ -603,6 +604,9 @@ pub async fn command(config: Config) -> Result<()> {
                 .collect(),
             metric_registry: Arc::clone(&metrics),
             mem_pool_size: config.exec_mem_pool_bytes.as_num_bytes(),
+            // TODO: need to make these configurable?
+            per_query_mem_pool_config: PerQueryMemoryPoolConfig::Disabled,
+            heap_memory_limit: None,
         },
         DedicatedExecutor::new(
             "datafusion",
@@ -735,6 +739,7 @@ pub async fn command(config: Config) -> Result<()> {
         sys_events_store: Arc::clone(&sys_events_store),
         // convert to positive here so that we can avoid double negatives downstream
         started_with_auth: !config.without_auth,
+        time_provider: Arc::clone(&time_provider) as _,
     }));
 
     let listener = TcpListener::bind(*config.http_bind_address)

influxdb3/src/commands/serve.rs

@@ -1,22 +1,16 @@
 ---
 source: influxdb3/tests/cli/mod.rs
-expression: output
+expression: summary_output
 ---
 distinct_caches summary:
-+-------+------+------------+--------------+-----------------+-----------------+
-| table | name | column_ids | column_names | max_cardinality | max_age_seconds |
-+-------+------+------------+--------------+-----------------+-----------------+
-+-------+------+------------+--------------+-----------------+-----------------+
+++
+++
 last_caches summary:
-+-------+------+----------------+------------------+------------------+--------------------+-------+-----+
-| table | name | key_column_ids | key_column_names | value_column_ids | value_column_names | count | ttl |
-+-------+------+----------------+------------------+------------------+--------------------+-------+-----+
-+-------+------+----------------+------------------+------------------+--------------------+-------+-----+
+++
+++
 parquet_files summary:
-+------------+------+------------+-----------+----------+----------+
-| table_name | path | size_bytes | row_count | min_time | max_time |
-+------------+------+------------+-----------+----------+----------+
-+------------+------+------------+-----------+----------+----------+
+++
+++
 processing_engine_logs summary:
 ++
 ++

influxdb3/tests/cli/snapshots/lib__cli__summary_should_show_up_to_ten_entries_from_each_table.snap

@@ -117,6 +117,8 @@ async fn flight() -> Result<(), influxdb3_client::Error> {
                 "+--------------+--------------------+----------------------------+------------+",
                 "| public       | information_schema | columns                    | VIEW       |",
                 "| public       | information_schema | df_settings                | VIEW       |",
+                "| public       | information_schema | parameters                 | VIEW       |",
+                "| public       | information_schema | routines                   | VIEW       |",
                 "| public       | information_schema | schemata                   | VIEW       |",
                 "| public       | information_schema | tables                     | VIEW       |",
                 "| public       | information_schema | views                      | VIEW       |",

influxdb3/tests/server/flight.rs

@@ -276,7 +276,7 @@ impl TestServer {
         //
         // The file is deleted when it goes out of scope (the end of this method) by the TempDir type.
         let tmp_dir = TempDir::new().unwrap();
-        let tmp_dir_path = tmp_dir.into_path();
+        let tmp_dir_path = tmp_dir.keep();
         let tcp_addr_file = tmp_dir_path.join("tcp-listener");
         let mut command = Command::cargo_bin("influxdb3").expect("create the influxdb3 command");
         let command = command

influxdb3/tests/server/mod.rs

@@ -23,4 +23,6 @@ expression: output
 | public        | information_schema | columns                    | VIEW       |
 | public        | information_schema | df_settings                | VIEW       |
 | public        | information_schema | schemata                   | VIEW       |
+| public        | information_schema | routines                   | VIEW       |
+| public        | information_schema | parameters                 | VIEW       |
 +---------------+--------------------+----------------------------+------------+

influxdb3/tests/server/snapshots/lib__server__query__api_v3_query_show_tables_ordering.snap

@@ -1,7 +1,9 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use authz::{Authorizer as IoxAuthorizer, Error as IoxError, Permission as IoxPermission};
+use authz::{
+    Authorization, Authorizer as IoxAuthorizer, Error as IoxError, Permission as IoxPermission,
+};
 use influxdb3_id::{DbId, TokenId};
 use iox_time::{Time, TimeProvider};
 use observability_deps::tracing::{debug, trace};
@@ -168,20 +170,16 @@ impl AuthProvider for TokenAuthenticator {
 
 #[async_trait]
 impl IoxAuthorizer for TokenAuthenticator {
-    async fn permissions(
+    async fn authorize(
         &self,
         token: Option<Vec<u8>>,
         perms: &[IoxPermission],
-    ) -> Result<Vec<IoxPermission>, IoxError> {
-        match self.authenticate(token).await {
-            Ok(_) => {
-                return Ok(perms.to_vec());
-            }
-            Err(err) => {
-                let iox_err = err.into();
-                return Err(iox_err);
-            }
-        }
+    ) -> Result<Authorization, IoxError> {
+        let _token_id = self.authenticate(token).await?;
+        // (trevor) The "subject" may just use the `token_id` as a string, but we withheld doing so
+        // in order to decided and ensure that we make use of the subject consistently throughout
+        // the system. See: https://github.com/influxdata/influxdb/issues/26440
+        Ok(Authorization::new(None, perms.to_vec()))
     }
 }
 
@@ -190,12 +188,12 @@ pub struct NoAuthAuthenticator;
 
 #[async_trait]
 impl IoxAuthorizer for NoAuthAuthenticator {
-    async fn permissions(
+    async fn authorize(
         &self,
         _token: Option<Vec<u8>>,
         perms: &[IoxPermission],
-    ) -> Result<Vec<IoxPermission>, IoxError> {
-        Ok(perms.to_vec())
+    ) -> Result<Authorization, IoxError> {
+        Ok(Authorization::new(None, perms.to_vec()))
     }
 
     async fn probe(&self) -> Result<(), IoxError> {

influxdb3_authz/src/lib.rs

@@ -3,9 +3,9 @@ use std::{any::Any, sync::Arc};
 use arrow::{array::RecordBatch, datatypes::SchemaRef};
 use async_trait::async_trait;
 use datafusion::{
-    catalog::{Session, TableProvider},
+    catalog::{Session, TableFunctionImpl, TableProvider},
     common::{DFSchema, Result, internal_err, plan_err},
-    datasource::{TableType, function::TableFunctionImpl},
+    datasource::TableType,
     execution::context::ExecutionProps,
     logical_expr::TableProviderFilterPushDown,
     physical_expr::{