From 0863367b896959cd064ea60b11a3f06c857926a9 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 01:20:34 +0200 Subject: [PATCH 01/17] feat(cubestore): Upgrade rust to nightly-2025-08-01 --- rust/cubestore/CLAUDE.md | 2 +- rust/cubestore/cubestore-sql-tests/src/lib.rs | 1 - .../cubestore/src/cachestore/cache_rocksstore.rs | 12 ++++++++---- rust/cubestore/cubestore/src/http/mod.rs | 4 ++-- rust/cubestore/cubestore/src/metastore/mod.rs | 4 ++-- rust/cubestore/rust-toolchain.toml | 2 +- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/rust/cubestore/CLAUDE.md b/rust/cubestore/CLAUDE.md index caeffbc45d3ea..33e98dea368ae 100644 --- a/rust/cubestore/CLAUDE.md +++ b/rust/cubestore/CLAUDE.md @@ -133,7 +133,7 @@ The codebase uses a custom dependency injection system defined in `config/inject ## Important Notes -- This is a Rust nightly project (see `rust-toolchain.toml`) +- **Rust Nightly**: Uses nightly-2025-08-01 (see `rust-toolchain.toml`) - Uses custom forks of Arrow/DataFusion and sqlparser-rs for Cube-specific features - Distributed mode involves router and worker nodes communicating via RPC - Heavy use of async/await patterns with Tokio runtime diff --git a/rust/cubestore/cubestore-sql-tests/src/lib.rs b/rust/cubestore/cubestore-sql-tests/src/lib.rs index 1197586664468..3b8f995f005eb 100644 --- a/rust/cubestore/cubestore-sql-tests/src/lib.rs +++ b/rust/cubestore/cubestore-sql-tests/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(async_closure)] #![feature(test)] pub use crate::benches::cubestore_benches; diff --git a/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs b/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs index e82e22803c1d1..4e5165ddb372b 100644 --- a/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs +++ b/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs @@ -273,8 +273,10 @@ impl RocksCacheStore { .upload_loop .process( cachestore.clone(), - async move |_| Ok(Delay::new(Duration::from_secs(upload_interval)).await), - async move |m, _| m.store.run_upload().await, + move |_| async move { + Ok(Delay::new(Duration::from_secs(upload_interval)).await) + }, + move |m, _| async move { m.store.run_upload().await }, ) .await; @@ -292,8 +294,10 @@ impl RocksCacheStore { .metrics_loop .process( cachestore.clone(), - async move |_| Ok(Delay::new(Duration::from_secs(metrics_interval)).await), - async move |m, _| { + move |_| async move { + Ok(Delay::new(Duration::from_secs(metrics_interval)).await) + }, + move |m, _| async move { if let Err(err) = m.submit_metrics().await { log::error!("Error while submitting cachestore metrics: {}", err) }; diff --git a/rust/cubestore/cubestore/src/http/mod.rs b/rust/cubestore/cubestore/src/http/mod.rs index e03fe51d0b425..d19b1ec9008df 100644 --- a/rust/cubestore/cubestore/src/http/mod.rs +++ b/rust/cubestore/cubestore/src/http/mod.rs @@ -403,8 +403,8 @@ impl HttpServer { let drop_processing_messages_after = self.drop_processing_messages_after.clone(); let drop_orphaned_messages_loop = self.drop_orphaned_messages_loop.process( messages_state, - async move |_| Ok(Delay::new(check_orphaned_messages_interval.clone()).await), - async move |messages_state, _| { + move |_| async move { Ok(Delay::new(check_orphaned_messages_interval.clone()).await) }, + move |messages_state, _| async move { let mut messages_state = messages_state.lock().await; let mut keys_to_remove = Vec::new(); let mut orphaned_complete_results = 0; diff --git a/rust/cubestore/cubestore/src/metastore/mod.rs b/rust/cubestore/cubestore/src/metastore/mod.rs index a94ff04a346e7..68f17951f77f2 100644 --- a/rust/cubestore/cubestore/src/metastore/mod.rs +++ b/rust/cubestore/cubestore/src/metastore/mod.rs @@ -1428,8 +1428,8 @@ impl RocksMetaStore { self.upload_loop .process( self.clone(), - async move |_| Ok(Delay::new(Duration::from_secs(upload_interval)).await), - async move |m, _| m.store.run_upload().await, + move |_| async move { Ok(Delay::new(Duration::from_secs(upload_interval)).await) }, + move |m, _| async move { m.store.run_upload().await }, ) .await; } diff --git a/rust/cubestore/rust-toolchain.toml b/rust/cubestore/rust-toolchain.toml index ff511a5586793..ad8132da3e1bc 100644 --- a/rust/cubestore/rust-toolchain.toml +++ b/rust/cubestore/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2024-01-29" +channel = "nightly-2025-08-01" components = ["rustfmt", "clippy"] profile = "minimal" From 306ff823a53c9827fcda4dbe8bc5568db2bd07d5 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 01:22:11 +0200 Subject: [PATCH 02/17] feat(cubestore): Upgrade rust to nightly-2025-08-01 --- rust/cubestore/cubestore/src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/rust/cubestore/cubestore/src/lib.rs b/rust/cubestore/cubestore/src/lib.rs index 2eeafc495cdf1..63ae36317de3b 100644 --- a/rust/cubestore/cubestore/src/lib.rs +++ b/rust/cubestore/cubestore/src/lib.rs @@ -1,10 +1,7 @@ #![feature(test)] -#![feature(async_closure)] #![feature(box_patterns)] #![feature(vec_into_raw_parts)] #![feature(hash_set_entry)] -#![feature(is_sorted)] -#![feature(result_flattening)] // #![feature(trace_macros)] // trace_macros!(true); From c7cc66bfa9512b63e661fe3a50e80d113264f974 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 01:31:39 +0200 Subject: [PATCH 03/17] chore: warnings --- rust/cubeshared/src/codegen/mod.rs | 1 + rust/cubestore/cubehll/src/instance.rs | 2 +- rust/cubestore/cubestore/src/cachestore/lazy.rs | 6 +++--- .../cubestore/src/cachestore/queue_item_payload.rs | 2 ++ rust/cubestore/cubestore/src/import/mod.rs | 2 +- rust/cubestore/cubestore/src/metastore/rocks_table.rs | 8 ++++---- rust/cubestore/cubestore/src/queryplanner/topk/plan.rs | 2 +- rust/cubestore/cubestore/src/remotefs/mod.rs | 2 +- rust/cubestore/cubestore/src/remotefs/queue.rs | 2 +- rust/cubestore/cubestore/src/remotefs/s3.rs | 2 +- rust/cubestore/cubestore/src/util/time_span.rs | 2 +- rust/cubestore/cubezetasketch/src/difference_encoding.rs | 4 ++-- rust/cubestore/cubezetasketch/src/sparse.rs | 2 +- 13 files changed, 20 insertions(+), 17 deletions(-) diff --git a/rust/cubeshared/src/codegen/mod.rs b/rust/cubeshared/src/codegen/mod.rs index fa722ae71b1e7..6bf8e6a8d0a2d 100644 --- a/rust/cubeshared/src/codegen/mod.rs +++ b/rust/cubeshared/src/codegen/mod.rs @@ -1,4 +1,5 @@ #[allow(unused_imports)] +#[allow(mismatched_lifetime_syntaxes)] mod http_message_generated; pub use http_message_generated::*; diff --git a/rust/cubestore/cubehll/src/instance.rs b/rust/cubestore/cubehll/src/instance.rs index d561cb1f0fa68..e2b7626aa617c 100644 --- a/rust/cubestore/cubehll/src/instance.rs +++ b/rust/cubestore/cubehll/src/instance.rs @@ -1217,7 +1217,7 @@ struct BitCursor<'a> { } impl BitCursor<'_> { - pub fn new(input: &[u8]) -> BitCursor { + pub fn new(input: &[u8]) -> BitCursor<'_> { BitCursor { input, pos: 0, diff --git a/rust/cubestore/cubestore/src/cachestore/lazy.rs b/rust/cubestore/cubestore/src/cachestore/lazy.rs index f22c3692601ad..1d8de4c015b58 100644 --- a/rust/cubestore/cubestore/src/cachestore/lazy.rs +++ b/rust/cubestore/cubestore/src/cachestore/lazy.rs @@ -24,7 +24,7 @@ pub enum LazyRocksCacheStoreState { metastore_fs: Arc, config: Arc, listeners: Vec>, - init_flag: Sender, + _init_flag: Sender, }, Closed {}, Initialized { @@ -72,7 +72,7 @@ impl LazyRocksCacheStore { metastore_fs, config, listeners, - init_flag, + _init_flag: init_flag, }), })) } @@ -101,7 +101,7 @@ impl LazyRocksCacheStore { config, listeners, // receiver will be closed on drop - init_flag: _, + _init_flag: _, } => { let store = RocksCacheStore::load_from_remote(&path, metastore_fs.clone(), config.clone()) diff --git a/rust/cubestore/cubestore/src/cachestore/queue_item_payload.rs b/rust/cubestore/cubestore/src/cachestore/queue_item_payload.rs index a9d401ed0ea9f..062dc1f93f191 100644 --- a/rust/cubestore/cubestore/src/cachestore/queue_item_payload.rs +++ b/rust/cubestore/cubestore/src/cachestore/queue_item_payload.rs @@ -42,6 +42,7 @@ impl QueueItemPayload { } #[derive(Clone, Copy, Debug)] +#[allow(dead_code)] pub(crate) enum QueueItemPayloadRocksIndex {} pub struct QueueItemPayloadRocksTable<'a> { @@ -80,6 +81,7 @@ rocks_table_new!( ); #[derive(Hash, Clone, Debug)] +#[allow(dead_code)] pub enum QueueItemPayloadIndexKey {} base_rocks_secondary_index!(QueueItemPayload, QueueItemPayloadRocksIndex); diff --git a/rust/cubestore/cubestore/src/import/mod.rs b/rust/cubestore/cubestore/src/import/mod.rs index 8d1db1a845f97..b91b067f7b843 100644 --- a/rust/cubestore/cubestore/src/import/mod.rs +++ b/rust/cubestore/cubestore/src/import/mod.rs @@ -323,7 +323,7 @@ impl<'a> CsvLineParser<'a> { } } - fn next_value(&mut self) -> Result { + fn next_value(&mut self) -> Result, CubeError> { Ok( if let Some(b'"') = self.remaining.as_bytes().iter().nth(0) { let mut closing_index = None; diff --git a/rust/cubestore/cubestore/src/metastore/rocks_table.rs b/rust/cubestore/cubestore/src/metastore/rocks_table.rs index 55b90680df53f..0cddd89f3eba7 100644 --- a/rust/cubestore/cubestore/src/metastore/rocks_table.rs +++ b/rust/cubestore/cubestore/src/metastore/rocks_table.rs @@ -61,7 +61,7 @@ macro_rules! rocks_table_new { self.db.db } - fn snapshot(&self) -> &cuberockstore::rocksdb::Snapshot { + fn snapshot(&self) -> &cuberockstore::rocksdb::Snapshot<'_> { self.db.snapshot } @@ -69,7 +69,7 @@ macro_rules! rocks_table_new { &self.db.mem_seq } - fn table_ref(&self) -> &crate::metastore::DbTableRef { + fn table_ref(&self) -> &crate::metastore::DbTableRef<'_> { &self.db } @@ -448,8 +448,8 @@ pub trait RocksTable: BaseRocksTable + Debug + Send + Sync { fn delete_event(&self, row: IdRow) -> MetaStoreEvent; fn update_event(&self, old_row: IdRow, new_row: IdRow) -> MetaStoreEvent; fn db(&self) -> &DB; - fn table_ref(&self) -> &DbTableRef; - fn snapshot(&self) -> &Snapshot; + fn table_ref(&self) -> &DbTableRef<'_>; + fn snapshot(&self) -> &Snapshot<'_>; fn mem_seq(&self) -> &MemorySequence; fn index_id(index_num: IndexId) -> IndexId; fn table_id() -> TableId; diff --git a/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs b/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs index 5af391cc7daad..13c772383d78f 100644 --- a/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs +++ b/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs @@ -207,7 +207,7 @@ fn extract_having(p: &Arc) -> (Option, &Arc) { } } -fn extract_projection_and_having(p: &LogicalPlan) -> Option { +fn extract_projection_and_having(p: &LogicalPlan) -> Option> { match p { LogicalPlan::Projection { expr, diff --git a/rust/cubestore/cubestore/src/remotefs/mod.rs b/rust/cubestore/cubestore/src/remotefs/mod.rs index cba78cd47aecc..e484f88265882 100644 --- a/rust/cubestore/cubestore/src/remotefs/mod.rs +++ b/rust/cubestore/cubestore/src/remotefs/mod.rs @@ -91,7 +91,7 @@ pub trait ExtendedRemoteFs: DIService + RemoteFs { async fn list_by_page( &self, remote_prefix: String, - ) -> Result, CubeError>>, CubeError> { + ) -> Result, CubeError>>, CubeError> { // Note, this implementation doesn't actually paginate. let list: Vec = self.list(remote_prefix).await?; diff --git a/rust/cubestore/cubestore/src/remotefs/queue.rs b/rust/cubestore/cubestore/src/remotefs/queue.rs index fb6388ee61d6f..30f0f1f4070d1 100644 --- a/rust/cubestore/cubestore/src/remotefs/queue.rs +++ b/rust/cubestore/cubestore/src/remotefs/queue.rs @@ -346,7 +346,7 @@ impl ExtendedRemoteFs for QueueRemoteFs { async fn list_by_page( &self, remote_prefix: String, - ) -> Result, CubeError>>, CubeError> { + ) -> Result, CubeError>>, CubeError> { self.remote_fs.list_by_page(remote_prefix).await } } diff --git a/rust/cubestore/cubestore/src/remotefs/s3.rs b/rust/cubestore/cubestore/src/remotefs/s3.rs index 32a0bb37a20bd..1f1d033bdbbf1 100644 --- a/rust/cubestore/cubestore/src/remotefs/s3.rs +++ b/rust/cubestore/cubestore/src/remotefs/s3.rs @@ -346,7 +346,7 @@ impl ExtendedRemoteFs for S3RemoteFs { async fn list_by_page( &self, remote_prefix: String, - ) -> Result, CubeError>>, CubeError> { + ) -> Result, CubeError>>, CubeError> { let path = self.s3_path(&remote_prefix); let bucket = self.bucket.load(); let leading_subpath = self.leading_subpath_regex(); diff --git a/rust/cubestore/cubestore/src/util/time_span.rs b/rust/cubestore/cubestore/src/util/time_span.rs index df74ad295e0ac..8e819c923784e 100644 --- a/rust/cubestore/cubestore/src/util/time_span.rs +++ b/rust/cubestore/cubestore/src/util/time_span.rs @@ -4,7 +4,7 @@ use std::time::{Duration, SystemTime}; /// The returned object will [log::warn] if it is alive longer than [timeout]. /// Be cautious when interpreting results in async code, this function looks at wall time. So future /// that is not running will add to the time. -pub fn warn_long(name: &str, timeout: Duration) -> ShortSpan { +pub fn warn_long(name: &str, timeout: Duration) -> ShortSpan<'_> { ShortSpan { name, timeout, diff --git a/rust/cubestore/cubezetasketch/src/difference_encoding.rs b/rust/cubestore/cubezetasketch/src/difference_encoding.rs index 7516732e90ba7..298def2958ce5 100644 --- a/rust/cubestore/cubezetasketch/src/difference_encoding.rs +++ b/rust/cubestore/cubezetasketch/src/difference_encoding.rs @@ -29,7 +29,7 @@ pub struct DifferenceEncoder<'l> { } impl DifferenceEncoder<'_> { - pub fn new(buf: &mut Vec) -> DifferenceEncoder { + pub fn new(buf: &mut Vec) -> DifferenceEncoder<'_> { return DifferenceEncoder { buf, last: 0 }; } @@ -88,7 +88,7 @@ pub struct DifferenceDecoder<'l> { } impl DifferenceDecoder<'_> { - pub fn new(data: &[u8]) -> DifferenceDecoder { + pub fn new(data: &[u8]) -> DifferenceDecoder<'_> { return DifferenceDecoder { data, last: 0 }; } } diff --git a/rust/cubestore/cubezetasketch/src/sparse.rs b/rust/cubestore/cubezetasketch/src/sparse.rs index 4531b5c2912ca..1ab1b70a94dba 100644 --- a/rust/cubestore/cubezetasketch/src/sparse.rs +++ b/rust/cubestore/cubezetasketch/src/sparse.rs @@ -327,7 +327,7 @@ impl SparseRepresentation { Ok(()) } - pub(crate) fn sorted_iterator(sparse_data: Option<&[u8]>) -> DifferenceDecoder { + pub(crate) fn sorted_iterator(sparse_data: Option<&[u8]>) -> DifferenceDecoder<'_> { return DifferenceDecoder::new(sparse_data.unwrap_or(&[])); } From a5058bdd8de4ddfecd4fa0099d5c8df4043b040d Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 01:35:10 +0200 Subject: [PATCH 04/17] chore: warnings --- rust/cubestore/cubestore/src/remotefs/mod.rs | 2 +- rust/cubestore/cubestore/src/remotefs/queue.rs | 2 +- rust/cubestore/cubestore/src/remotefs/s3.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/cubestore/cubestore/src/remotefs/mod.rs b/rust/cubestore/cubestore/src/remotefs/mod.rs index e484f88265882..c7092b0c11e6c 100644 --- a/rust/cubestore/cubestore/src/remotefs/mod.rs +++ b/rust/cubestore/cubestore/src/remotefs/mod.rs @@ -91,7 +91,7 @@ pub trait ExtendedRemoteFs: DIService + RemoteFs { async fn list_by_page( &self, remote_prefix: String, - ) -> Result, CubeError>>, CubeError> { + ) -> Result, CubeError>>, CubeError> { // Note, this implementation doesn't actually paginate. let list: Vec = self.list(remote_prefix).await?; diff --git a/rust/cubestore/cubestore/src/remotefs/queue.rs b/rust/cubestore/cubestore/src/remotefs/queue.rs index 30f0f1f4070d1..bb5beb585f87a 100644 --- a/rust/cubestore/cubestore/src/remotefs/queue.rs +++ b/rust/cubestore/cubestore/src/remotefs/queue.rs @@ -346,7 +346,7 @@ impl ExtendedRemoteFs for QueueRemoteFs { async fn list_by_page( &self, remote_prefix: String, - ) -> Result, CubeError>>, CubeError> { + ) -> Result, CubeError>>, CubeError> { self.remote_fs.list_by_page(remote_prefix).await } } diff --git a/rust/cubestore/cubestore/src/remotefs/s3.rs b/rust/cubestore/cubestore/src/remotefs/s3.rs index 1f1d033bdbbf1..2dc5f943b4bb3 100644 --- a/rust/cubestore/cubestore/src/remotefs/s3.rs +++ b/rust/cubestore/cubestore/src/remotefs/s3.rs @@ -346,7 +346,7 @@ impl ExtendedRemoteFs for S3RemoteFs { async fn list_by_page( &self, remote_prefix: String, - ) -> Result, CubeError>>, CubeError> { + ) -> Result, CubeError>>, CubeError> { let path = self.s3_path(&remote_prefix); let bucket = self.bucket.load(); let leading_subpath = self.leading_subpath_regex(); From 8f50b7a8c0b8579355e01fc8ff4cb6c8fc3d5e26 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 01:37:17 +0200 Subject: [PATCH 05/17] chore: sync --- .github/workflows/rust-cubestore.yml | 2 +- rust/cubestore/builder.Dockerfile | 4 ++-- rust/cubestore/cubestore/src/metastore/mod.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust-cubestore.yml b/.github/workflows/rust-cubestore.yml index a07569688256c..b7353631a4bb3 100644 --- a/.github/workflows/rust-cubestore.yml +++ b/.github/workflows/rust-cubestore.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - rust: [nightly-2024-01-29] + rust: [nightly-2025-08-01] container: image: cubejs/rust-builder:bookworm-llvm-18 env: diff --git a/rust/cubestore/builder.Dockerfile b/rust/cubestore/builder.Dockerfile index 9860ed5303765..91b99d7a55e9a 100644 --- a/rust/cubestore/builder.Dockerfile +++ b/rust/cubestore/builder.Dockerfile @@ -5,8 +5,8 @@ FROM rust:$OS_NAME ARG LLVM_VERSION=18 RUN rustup update && \ - rustup default nightly-2024-01-29 && \ - rustup component add --toolchain nightly-2024-01-29 rustfmt clippy; + rustup default nightly-2025-08-01 && \ + rustup component add --toolchain nightly-2025-08-01 rustfmt clippy; RUN apt update \ && apt upgrade -y \ diff --git a/rust/cubestore/cubestore/src/metastore/mod.rs b/rust/cubestore/cubestore/src/metastore/mod.rs index 68f17951f77f2..d1dff96e1d93f 100644 --- a/rust/cubestore/cubestore/src/metastore/mod.rs +++ b/rust/cubestore/cubestore/src/metastore/mod.rs @@ -140,7 +140,7 @@ macro_rules! data_frame_from { } #[macro_export] -macro_rules! base_rocks_secondary_index { +macro_rules! anbase_rocks_secondary_index { ($table: ty, $index: ty) => { impl crate::metastore::BaseRocksSecondaryIndex<$table> for $index { fn index_value(&self, row: &$table) -> Vec { From b1e9d84d47898d95a60f7530c5c764e6454c40ca Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 01:37:42 +0200 Subject: [PATCH 06/17] chore: sync --- .github/workflows/rust-cubestore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-cubestore.yml b/.github/workflows/rust-cubestore.yml index b7353631a4bb3..f16f5cd2a17d9 100644 --- a/.github/workflows/rust-cubestore.yml +++ b/.github/workflows/rust-cubestore.yml @@ -229,7 +229,7 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 target: ${{ matrix.target }} # override: true # this is by default on rustflags: "" From 48c143f0a464bcdfc3ec2363c0e37b85259b7ca5 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 01:38:10 +0200 Subject: [PATCH 07/17] chore: sync --- rust/cubestore/CLAUDE.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/rust/cubestore/CLAUDE.md b/rust/cubestore/CLAUDE.md index 33e98dea368ae..e407c8aaaac8e 100644 --- a/rust/cubestore/CLAUDE.md +++ b/rust/cubestore/CLAUDE.md @@ -137,4 +137,17 @@ The codebase uses a custom dependency injection system defined in `config/inject - Uses custom forks of Arrow/DataFusion and sqlparser-rs for Cube-specific features - Distributed mode involves router and worker nodes communicating via RPC - Heavy use of async/await patterns with Tokio runtime -- Parquet files are the primary storage format for data \ No newline at end of file +- Parquet files are the primary storage format for data + +## Docker Configuration + +The project includes Docker configurations for building and deploying CubeStore: + +- **`builder.Dockerfile`**: Defines the base build image with Rust nightly-2025-08-01, LLVM 18, and build dependencies +- **`Dockerfile`**: Production Dockerfile that uses `cubejs/rust-builder:bookworm-llvm-18` base image +- **GitHub Actions**: CI/CD workflow in `.github/workflows/rust-cubestore.yml` uses the same Rust version + +When updating the Rust version, ensure all these files are kept in sync: +1. `rust-toolchain.toml` - Primary source of truth +2. `builder.Dockerfile` - Builder image definition +3. `.github/workflows/rust-cubestore.yml` - CI configuration \ No newline at end of file From 90eedb9580e68e800a78cb9216abc37d87ac5ab2 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 01:38:52 +0200 Subject: [PATCH 08/17] chore: sync --- rust/cubestore/cubestore/src/metastore/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/cubestore/cubestore/src/metastore/mod.rs b/rust/cubestore/cubestore/src/metastore/mod.rs index d1dff96e1d93f..4412769597ed0 100644 --- a/rust/cubestore/cubestore/src/metastore/mod.rs +++ b/rust/cubestore/cubestore/src/metastore/mod.rs @@ -140,7 +140,7 @@ macro_rules! data_frame_from { } #[macro_export] -macro_rules! anbase_rocks_secondary_index { +macro_rules! anbase_rocks_secondary_index { ($table: ty, $index: ty) => { impl crate::metastore::BaseRocksSecondaryIndex<$table> for $index { fn index_value(&self, row: &$table) -> Vec { From 0eb3b652308e5b0eb2784fe4ea41036254f0a80c Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 01:39:29 +0200 Subject: [PATCH 09/17] chore: sync --- rust/cubestore/cubestore/src/metastore/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/cubestore/cubestore/src/metastore/mod.rs b/rust/cubestore/cubestore/src/metastore/mod.rs index 4412769597ed0..68f17951f77f2 100644 --- a/rust/cubestore/cubestore/src/metastore/mod.rs +++ b/rust/cubestore/cubestore/src/metastore/mod.rs @@ -140,7 +140,7 @@ macro_rules! data_frame_from { } #[macro_export] -macro_rules! anbase_rocks_secondary_index { +macro_rules! base_rocks_secondary_index { ($table: ty, $index: ty) => { impl crate::metastore::BaseRocksSecondaryIndex<$table> for $index { fn index_value(&self, row: &$table) -> Vec { From fd2e3595fcb7eb7f46858940d7ce47e24c3b39c4 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 11:04:06 +0200 Subject: [PATCH 10/17] chore: fix --- rust/cubestore/cubestore/src/metastore/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/cubestore/cubestore/src/metastore/mod.rs b/rust/cubestore/cubestore/src/metastore/mod.rs index 68f17951f77f2..439cb5dc15d2c 100644 --- a/rust/cubestore/cubestore/src/metastore/mod.rs +++ b/rust/cubestore/cubestore/src/metastore/mod.rs @@ -4956,7 +4956,7 @@ mod tests { #[test] fn test_structures_size() { - assert_eq!(std::mem::size_of::(), 680); + assert_eq!(std::mem::size_of::(), 672); } #[tokio::test] From 5abeca33aa623c1b3c75f7df454f3b01f1913182 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 11:11:33 +0200 Subject: [PATCH 11/17] chore: fix --- rust/cubeshared/src/codegen/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/cubeshared/src/codegen/mod.rs b/rust/cubeshared/src/codegen/mod.rs index 6bf8e6a8d0a2d..fa722ae71b1e7 100644 --- a/rust/cubeshared/src/codegen/mod.rs +++ b/rust/cubeshared/src/codegen/mod.rs @@ -1,5 +1,4 @@ #[allow(unused_imports)] -#[allow(mismatched_lifetime_syntaxes)] mod http_message_generated; pub use http_message_generated::*; From b6ffdfff51f58cc985397aaae0657ea2c97f2f72 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 11:14:43 +0200 Subject: [PATCH 12/17] chore: fix --- .github/workflows/publish.yml | 4 ++-- .github/workflows/push.yml | 2 +- .github/workflows/rust-cubestore-master.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a4dd872146684..72915644b357e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -618,7 +618,7 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 target: ${{ matrix.target }} # override: true # this is by default on rustflags: "" @@ -705,7 +705,7 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 target: ${{ matrix.target }} # override: true # this is by default on rustflags: "" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ff5ce81f0d187..bafd552b8404a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -237,7 +237,7 @@ jobs: - name: Install Rust uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 # override: true # this is by default on rustflags: "" components: rustfmt diff --git a/.github/workflows/rust-cubestore-master.yml b/.github/workflows/rust-cubestore-master.yml index dcfb4f9901734..c78f58828d865 100644 --- a/.github/workflows/rust-cubestore-master.yml +++ b/.github/workflows/rust-cubestore-master.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - rust: [nightly-2024-01-29] + rust: [nightly-2025-08-01] env: RUST: ${{ matrix.rust }} steps: @@ -290,7 +290,7 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 target: ${{ matrix.target }} # override: true # this is by default on rustflags: "" From acff2950e8d4bd7e131fa972e0b312b8898a24d4 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 11:19:23 +0200 Subject: [PATCH 13/17] chore: fix --- rust/cubeshared/src/codegen/http_message_generated.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/cubeshared/src/codegen/http_message_generated.rs b/rust/cubeshared/src/codegen/http_message_generated.rs index f1afd7c51c527..62dcb3467b810 100644 --- a/rust/cubeshared/src/codegen/http_message_generated.rs +++ b/rust/cubeshared/src/codegen/http_message_generated.rs @@ -1246,7 +1246,7 @@ impl core::fmt::Debug for HttpColumnValue<'_> { /// catch every error, or be maximally performant. For the /// previous, unchecked, behavior use /// `root_as_http_message_unchecked`. -pub fn root_as_http_message(buf: &[u8]) -> Result { +pub fn root_as_http_message(buf: &[u8]) -> Result, flatbuffers::InvalidFlatbuffer> { flatbuffers::root::(buf) } #[inline] @@ -1258,7 +1258,7 @@ pub fn root_as_http_message(buf: &[u8]) -> Result Result { +) -> Result, flatbuffers::InvalidFlatbuffer> { flatbuffers::size_prefixed_root::(buf) } #[inline] @@ -1291,14 +1291,14 @@ pub fn size_prefixed_root_as_http_message_with_opts<'b, 'o>( /// Assumes, without verification, that a buffer of bytes contains a HttpMessage and returns it. /// # Safety /// Callers must trust the given bytes do indeed contain a valid `HttpMessage`. -pub unsafe fn root_as_http_message_unchecked(buf: &[u8]) -> HttpMessage { +pub unsafe fn root_as_http_message_unchecked(buf: &[u8]) -> HttpMessage<'_> { flatbuffers::root_unchecked::(buf) } #[inline] /// Assumes, without verification, that a buffer of bytes contains a size prefixed HttpMessage and returns it. /// # Safety /// Callers must trust the given bytes do indeed contain a valid size prefixed `HttpMessage`. -pub unsafe fn size_prefixed_root_as_http_message_unchecked(buf: &[u8]) -> HttpMessage { +pub unsafe fn size_prefixed_root_as_http_message_unchecked(buf: &[u8]) -> HttpMessage<'_> { flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] From 21dde01db091c6ae44a5d1f47a7673fcbeb33054 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 11:28:44 +0200 Subject: [PATCH 14/17] chore: fix --- rust/cubestore/Cargo.lock | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/rust/cubestore/Cargo.lock b/rust/cubestore/Cargo.lock index ef5841f430c43..e577b038d8a70 100644 --- a/rust/cubestore/Cargo.lock +++ b/rust/cubestore/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -650,7 +650,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -2894,13 +2894,13 @@ dependencies = [ [[package]] name = "msql-srv" version = "0.9.2" -source = "git+https://github.com/cube-js/msql-srv#e3010bd6680a89feff53bcd3f74afa21f5403f70" +source = "git+https://github.com/cube-js/msql-srv#82bca052e6bd7facefe759a1971702cc9aac142e" dependencies = [ "async-trait", "byteorder", "chrono", "mysql_common", - "nom 5.1.2", + "nom", "rand 0.8.4", "time 0.2.7", "tokio", @@ -3012,17 +3012,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "nom" -version = "5.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" -dependencies = [ - "lexical-core", - "memchr", - "version_check", -] - [[package]] name = "nom" version = "7.1.3" @@ -5531,8 +5520,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", - "rand 0.8.4", + "cfg-if 1.0.0", + "rand 0.6.5", "static_assertions", ] From 48a1d376e5c534e2995dc86b876cd97809a75750 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 11:37:15 +0200 Subject: [PATCH 15/17] chore: fix --- rust/cubestore/Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/cubestore/Cargo.lock b/rust/cubestore/Cargo.lock index e577b038d8a70..e277b366773e5 100644 --- a/rust/cubestore/Cargo.lock +++ b/rust/cubestore/Cargo.lock @@ -2894,7 +2894,7 @@ dependencies = [ [[package]] name = "msql-srv" version = "0.9.2" -source = "git+https://github.com/cube-js/msql-srv#82bca052e6bd7facefe759a1971702cc9aac142e" +source = "git+https://github.com/cube-js/msql-srv#b560de0db9c4dfc57ec511f88027d157785f3952" dependencies = [ "async-trait", "byteorder", @@ -5520,8 +5520,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 1.0.0", - "rand 0.6.5", + "cfg-if 0.1.10", + "rand 0.8.4", "static_assertions", ] From 004a999de404cd0ab8a668aa90a749e8a48fe520 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 11:39:45 +0200 Subject: [PATCH 16/17] chore: fix --- rust/cubestore/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/cubestore/Dockerfile b/rust/cubestore/Dockerfile index 482ebe30d8dc4..4014111dee4db 100644 --- a/rust/cubestore/Dockerfile +++ b/rust/cubestore/Dockerfile @@ -4,6 +4,7 @@ WORKDIR /build/cubestore COPY cubeshared /build/cubeshared +COPY cubestore/rust-toolchain.toml . COPY cubestore/Cargo.toml . COPY cubestore/Cargo.lock . COPY cubestore/cuberockstore cuberockstore From 8cd8eac6860280c4a945d0ab4f0b28a503ff798a Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 7 Aug 2025 11:50:01 +0200 Subject: [PATCH 17/17] chore: fix --- rust/cubestore/CLAUDE.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/rust/cubestore/CLAUDE.md b/rust/cubestore/CLAUDE.md index e407c8aaaac8e..8ed13d4deeaf0 100644 --- a/rust/cubestore/CLAUDE.md +++ b/rust/cubestore/CLAUDE.md @@ -144,10 +144,16 @@ The codebase uses a custom dependency injection system defined in `config/inject The project includes Docker configurations for building and deploying CubeStore: - **`builder.Dockerfile`**: Defines the base build image with Rust nightly-2025-08-01, LLVM 18, and build dependencies -- **`Dockerfile`**: Production Dockerfile that uses `cubejs/rust-builder:bookworm-llvm-18` base image -- **GitHub Actions**: CI/CD workflow in `.github/workflows/rust-cubestore.yml` uses the same Rust version +- **`Dockerfile`**: Production Dockerfile that uses `cubejs/rust-builder:bookworm-llvm-18` base image and copies rust-toolchain.toml +- **GitHub Actions**: Multiple CI/CD workflows use the same Rust version -When updating the Rust version, ensure all these files are kept in sync: -1. `rust-toolchain.toml` - Primary source of truth -2. `builder.Dockerfile` - Builder image definition -3. `.github/workflows/rust-cubestore.yml` - CI configuration \ No newline at end of file +## Updating Rust Version + +When updating the Rust version, ensure ALL these files are kept in sync: + +1. **`rust-toolchain.toml`** - Primary source of truth for local development +2. **`builder.Dockerfile`** - Update the rustup default command with the new nightly version +3. **`Dockerfile`** - Copies rust-toolchain.toml (no manual update needed if builder image is updated) +4. **GitHub Workflows** - Update all occurrences of the Rust nightly version in `.github/workflows/` directory + +**Note**: The `cubejs/rust-builder:bookworm-llvm-18` Docker image tag may also need updating if the builder.Dockerfile changes significantly.