diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a4dd872146684..72915644b357e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -618,7 +618,7 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 target: ${{ matrix.target }} # override: true # this is by default on rustflags: "" @@ -705,7 +705,7 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 target: ${{ matrix.target }} # override: true # this is by default on rustflags: "" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ff5ce81f0d187..bafd552b8404a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -237,7 +237,7 @@ jobs: - name: Install Rust uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 # override: true # this is by default on rustflags: "" components: rustfmt diff --git a/.github/workflows/rust-cubestore-master.yml b/.github/workflows/rust-cubestore-master.yml index dcfb4f9901734..c78f58828d865 100644 --- a/.github/workflows/rust-cubestore-master.yml +++ b/.github/workflows/rust-cubestore-master.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - rust: [nightly-2024-01-29] + rust: [nightly-2025-08-01] env: RUST: ${{ matrix.rust }} steps: @@ -290,7 +290,7 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 target: ${{ matrix.target }} # override: true # this is by default on rustflags: "" diff --git a/.github/workflows/rust-cubestore.yml b/.github/workflows/rust-cubestore.yml index a07569688256c..f16f5cd2a17d9 100644 --- a/.github/workflows/rust-cubestore.yml +++ b/.github/workflows/rust-cubestore.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - rust: [nightly-2024-01-29] + rust: [nightly-2025-08-01] container: image: cubejs/rust-builder:bookworm-llvm-18 env: @@ -229,7 +229,7 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: - toolchain: nightly-2024-01-29 + toolchain: nightly-2025-08-01 target: ${{ matrix.target }} # override: true # this is by default on rustflags: "" diff --git a/rust/cubeshared/src/codegen/http_message_generated.rs b/rust/cubeshared/src/codegen/http_message_generated.rs index f1afd7c51c527..62dcb3467b810 100644 --- a/rust/cubeshared/src/codegen/http_message_generated.rs +++ b/rust/cubeshared/src/codegen/http_message_generated.rs @@ -1246,7 +1246,7 @@ impl core::fmt::Debug for HttpColumnValue<'_> { /// catch every error, or be maximally performant. For the /// previous, unchecked, behavior use /// `root_as_http_message_unchecked`. -pub fn root_as_http_message(buf: &[u8]) -> Result { +pub fn root_as_http_message(buf: &[u8]) -> Result, flatbuffers::InvalidFlatbuffer> { flatbuffers::root::(buf) } #[inline] @@ -1258,7 +1258,7 @@ pub fn root_as_http_message(buf: &[u8]) -> Result Result { +) -> Result, flatbuffers::InvalidFlatbuffer> { flatbuffers::size_prefixed_root::(buf) } #[inline] @@ -1291,14 +1291,14 @@ pub fn size_prefixed_root_as_http_message_with_opts<'b, 'o>( /// Assumes, without verification, that a buffer of bytes contains a HttpMessage and returns it. /// # Safety /// Callers must trust the given bytes do indeed contain a valid `HttpMessage`. -pub unsafe fn root_as_http_message_unchecked(buf: &[u8]) -> HttpMessage { +pub unsafe fn root_as_http_message_unchecked(buf: &[u8]) -> HttpMessage<'_> { flatbuffers::root_unchecked::(buf) } #[inline] /// Assumes, without verification, that a buffer of bytes contains a size prefixed HttpMessage and returns it. /// # Safety /// Callers must trust the given bytes do indeed contain a valid size prefixed `HttpMessage`. -pub unsafe fn size_prefixed_root_as_http_message_unchecked(buf: &[u8]) -> HttpMessage { +pub unsafe fn size_prefixed_root_as_http_message_unchecked(buf: &[u8]) -> HttpMessage<'_> { flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] diff --git a/rust/cubestore/CLAUDE.md b/rust/cubestore/CLAUDE.md index caeffbc45d3ea..8ed13d4deeaf0 100644 --- a/rust/cubestore/CLAUDE.md +++ b/rust/cubestore/CLAUDE.md @@ -133,8 +133,27 @@ The codebase uses a custom dependency injection system defined in `config/inject ## Important Notes -- This is a Rust nightly project (see `rust-toolchain.toml`) +- **Rust Nightly**: Uses nightly-2025-08-01 (see `rust-toolchain.toml`) - Uses custom forks of Arrow/DataFusion and sqlparser-rs for Cube-specific features - Distributed mode involves router and worker nodes communicating via RPC - Heavy use of async/await patterns with Tokio runtime -- Parquet files are the primary storage format for data \ No newline at end of file +- Parquet files are the primary storage format for data + +## Docker Configuration + +The project includes Docker configurations for building and deploying CubeStore: + +- **`builder.Dockerfile`**: Defines the base build image with Rust nightly-2025-08-01, LLVM 18, and build dependencies +- **`Dockerfile`**: Production Dockerfile that uses `cubejs/rust-builder:bookworm-llvm-18` base image and copies rust-toolchain.toml +- **GitHub Actions**: Multiple CI/CD workflows use the same Rust version + +## Updating Rust Version + +When updating the Rust version, ensure ALL these files are kept in sync: + +1. **`rust-toolchain.toml`** - Primary source of truth for local development +2. **`builder.Dockerfile`** - Update the rustup default command with the new nightly version +3. **`Dockerfile`** - Copies rust-toolchain.toml (no manual update needed if builder image is updated) +4. **GitHub Workflows** - Update all occurrences of the Rust nightly version in `.github/workflows/` directory + +**Note**: The `cubejs/rust-builder:bookworm-llvm-18` Docker image tag may also need updating if the builder.Dockerfile changes significantly. diff --git a/rust/cubestore/Cargo.lock b/rust/cubestore/Cargo.lock index ef5841f430c43..e277b366773e5 100644 --- a/rust/cubestore/Cargo.lock +++ b/rust/cubestore/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -650,7 +650,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -2894,13 +2894,13 @@ dependencies = [ [[package]] name = "msql-srv" version = "0.9.2" -source = "git+https://github.com/cube-js/msql-srv#e3010bd6680a89feff53bcd3f74afa21f5403f70" +source = "git+https://github.com/cube-js/msql-srv#b560de0db9c4dfc57ec511f88027d157785f3952" dependencies = [ "async-trait", "byteorder", "chrono", "mysql_common", - "nom 5.1.2", + "nom", "rand 0.8.4", "time 0.2.7", "tokio", @@ -3012,17 +3012,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "nom" -version = "5.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" -dependencies = [ - "lexical-core", - "memchr", - "version_check", -] - [[package]] name = "nom" version = "7.1.3" diff --git a/rust/cubestore/Dockerfile b/rust/cubestore/Dockerfile index 482ebe30d8dc4..4014111dee4db 100644 --- a/rust/cubestore/Dockerfile +++ b/rust/cubestore/Dockerfile @@ -4,6 +4,7 @@ WORKDIR /build/cubestore COPY cubeshared /build/cubeshared +COPY cubestore/rust-toolchain.toml . COPY cubestore/Cargo.toml . COPY cubestore/Cargo.lock . COPY cubestore/cuberockstore cuberockstore diff --git a/rust/cubestore/builder.Dockerfile b/rust/cubestore/builder.Dockerfile index 9860ed5303765..91b99d7a55e9a 100644 --- a/rust/cubestore/builder.Dockerfile +++ b/rust/cubestore/builder.Dockerfile @@ -5,8 +5,8 @@ FROM rust:$OS_NAME ARG LLVM_VERSION=18 RUN rustup update && \ - rustup default nightly-2024-01-29 && \ - rustup component add --toolchain nightly-2024-01-29 rustfmt clippy; + rustup default nightly-2025-08-01 && \ + rustup component add --toolchain nightly-2025-08-01 rustfmt clippy; RUN apt update \ && apt upgrade -y \ diff --git a/rust/cubestore/cubehll/src/instance.rs b/rust/cubestore/cubehll/src/instance.rs index d561cb1f0fa68..e2b7626aa617c 100644 --- a/rust/cubestore/cubehll/src/instance.rs +++ b/rust/cubestore/cubehll/src/instance.rs @@ -1217,7 +1217,7 @@ struct BitCursor<'a> { } impl BitCursor<'_> { - pub fn new(input: &[u8]) -> BitCursor { + pub fn new(input: &[u8]) -> BitCursor<'_> { BitCursor { input, pos: 0, diff --git a/rust/cubestore/cubestore-sql-tests/src/lib.rs b/rust/cubestore/cubestore-sql-tests/src/lib.rs index 1197586664468..3b8f995f005eb 100644 --- a/rust/cubestore/cubestore-sql-tests/src/lib.rs +++ b/rust/cubestore/cubestore-sql-tests/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(async_closure)] #![feature(test)] pub use crate::benches::cubestore_benches; diff --git a/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs b/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs index e82e22803c1d1..4e5165ddb372b 100644 --- a/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs +++ b/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs @@ -273,8 +273,10 @@ impl RocksCacheStore { .upload_loop .process( cachestore.clone(), - async move |_| Ok(Delay::new(Duration::from_secs(upload_interval)).await), - async move |m, _| m.store.run_upload().await, + move |_| async move { + Ok(Delay::new(Duration::from_secs(upload_interval)).await) + }, + move |m, _| async move { m.store.run_upload().await }, ) .await; @@ -292,8 +294,10 @@ impl RocksCacheStore { .metrics_loop .process( cachestore.clone(), - async move |_| Ok(Delay::new(Duration::from_secs(metrics_interval)).await), - async move |m, _| { + move |_| async move { + Ok(Delay::new(Duration::from_secs(metrics_interval)).await) + }, + move |m, _| async move { if let Err(err) = m.submit_metrics().await { log::error!("Error while submitting cachestore metrics: {}", err) }; diff --git a/rust/cubestore/cubestore/src/cachestore/lazy.rs b/rust/cubestore/cubestore/src/cachestore/lazy.rs index f22c3692601ad..1d8de4c015b58 100644 --- a/rust/cubestore/cubestore/src/cachestore/lazy.rs +++ b/rust/cubestore/cubestore/src/cachestore/lazy.rs @@ -24,7 +24,7 @@ pub enum LazyRocksCacheStoreState { metastore_fs: Arc, config: Arc, listeners: Vec>, - init_flag: Sender, + _init_flag: Sender, }, Closed {}, Initialized { @@ -72,7 +72,7 @@ impl LazyRocksCacheStore { metastore_fs, config, listeners, - init_flag, + _init_flag: init_flag, }), })) } @@ -101,7 +101,7 @@ impl LazyRocksCacheStore { config, listeners, // receiver will be closed on drop - init_flag: _, + _init_flag: _, } => { let store = RocksCacheStore::load_from_remote(&path, metastore_fs.clone(), config.clone()) diff --git a/rust/cubestore/cubestore/src/cachestore/queue_item_payload.rs b/rust/cubestore/cubestore/src/cachestore/queue_item_payload.rs index a9d401ed0ea9f..062dc1f93f191 100644 --- a/rust/cubestore/cubestore/src/cachestore/queue_item_payload.rs +++ b/rust/cubestore/cubestore/src/cachestore/queue_item_payload.rs @@ -42,6 +42,7 @@ impl QueueItemPayload { } #[derive(Clone, Copy, Debug)] +#[allow(dead_code)] pub(crate) enum QueueItemPayloadRocksIndex {} pub struct QueueItemPayloadRocksTable<'a> { @@ -80,6 +81,7 @@ rocks_table_new!( ); #[derive(Hash, Clone, Debug)] +#[allow(dead_code)] pub enum QueueItemPayloadIndexKey {} base_rocks_secondary_index!(QueueItemPayload, QueueItemPayloadRocksIndex); diff --git a/rust/cubestore/cubestore/src/http/mod.rs b/rust/cubestore/cubestore/src/http/mod.rs index e03fe51d0b425..d19b1ec9008df 100644 --- a/rust/cubestore/cubestore/src/http/mod.rs +++ b/rust/cubestore/cubestore/src/http/mod.rs @@ -403,8 +403,8 @@ impl HttpServer { let drop_processing_messages_after = self.drop_processing_messages_after.clone(); let drop_orphaned_messages_loop = self.drop_orphaned_messages_loop.process( messages_state, - async move |_| Ok(Delay::new(check_orphaned_messages_interval.clone()).await), - async move |messages_state, _| { + move |_| async move { Ok(Delay::new(check_orphaned_messages_interval.clone()).await) }, + move |messages_state, _| async move { let mut messages_state = messages_state.lock().await; let mut keys_to_remove = Vec::new(); let mut orphaned_complete_results = 0; diff --git a/rust/cubestore/cubestore/src/import/mod.rs b/rust/cubestore/cubestore/src/import/mod.rs index 8d1db1a845f97..b91b067f7b843 100644 --- a/rust/cubestore/cubestore/src/import/mod.rs +++ b/rust/cubestore/cubestore/src/import/mod.rs @@ -323,7 +323,7 @@ impl<'a> CsvLineParser<'a> { } } - fn next_value(&mut self) -> Result { + fn next_value(&mut self) -> Result, CubeError> { Ok( if let Some(b'"') = self.remaining.as_bytes().iter().nth(0) { let mut closing_index = None; diff --git a/rust/cubestore/cubestore/src/lib.rs b/rust/cubestore/cubestore/src/lib.rs index 2eeafc495cdf1..63ae36317de3b 100644 --- a/rust/cubestore/cubestore/src/lib.rs +++ b/rust/cubestore/cubestore/src/lib.rs @@ -1,10 +1,7 @@ #![feature(test)] -#![feature(async_closure)] #![feature(box_patterns)] #![feature(vec_into_raw_parts)] #![feature(hash_set_entry)] -#![feature(is_sorted)] -#![feature(result_flattening)] // #![feature(trace_macros)] // trace_macros!(true); diff --git a/rust/cubestore/cubestore/src/metastore/mod.rs b/rust/cubestore/cubestore/src/metastore/mod.rs index a94ff04a346e7..439cb5dc15d2c 100644 --- a/rust/cubestore/cubestore/src/metastore/mod.rs +++ b/rust/cubestore/cubestore/src/metastore/mod.rs @@ -1428,8 +1428,8 @@ impl RocksMetaStore { self.upload_loop .process( self.clone(), - async move |_| Ok(Delay::new(Duration::from_secs(upload_interval)).await), - async move |m, _| m.store.run_upload().await, + move |_| async move { Ok(Delay::new(Duration::from_secs(upload_interval)).await) }, + move |m, _| async move { m.store.run_upload().await }, ) .await; } @@ -4956,7 +4956,7 @@ mod tests { #[test] fn test_structures_size() { - assert_eq!(std::mem::size_of::(), 680); + assert_eq!(std::mem::size_of::(), 672); } #[tokio::test] diff --git a/rust/cubestore/cubestore/src/metastore/rocks_table.rs b/rust/cubestore/cubestore/src/metastore/rocks_table.rs index 55b90680df53f..0cddd89f3eba7 100644 --- a/rust/cubestore/cubestore/src/metastore/rocks_table.rs +++ b/rust/cubestore/cubestore/src/metastore/rocks_table.rs @@ -61,7 +61,7 @@ macro_rules! rocks_table_new { self.db.db } - fn snapshot(&self) -> &cuberockstore::rocksdb::Snapshot { + fn snapshot(&self) -> &cuberockstore::rocksdb::Snapshot<'_> { self.db.snapshot } @@ -69,7 +69,7 @@ macro_rules! rocks_table_new { &self.db.mem_seq } - fn table_ref(&self) -> &crate::metastore::DbTableRef { + fn table_ref(&self) -> &crate::metastore::DbTableRef<'_> { &self.db } @@ -448,8 +448,8 @@ pub trait RocksTable: BaseRocksTable + Debug + Send + Sync { fn delete_event(&self, row: IdRow) -> MetaStoreEvent; fn update_event(&self, old_row: IdRow, new_row: IdRow) -> MetaStoreEvent; fn db(&self) -> &DB; - fn table_ref(&self) -> &DbTableRef; - fn snapshot(&self) -> &Snapshot; + fn table_ref(&self) -> &DbTableRef<'_>; + fn snapshot(&self) -> &Snapshot<'_>; fn mem_seq(&self) -> &MemorySequence; fn index_id(index_num: IndexId) -> IndexId; fn table_id() -> TableId; diff --git a/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs b/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs index 5af391cc7daad..13c772383d78f 100644 --- a/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs +++ b/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs @@ -207,7 +207,7 @@ fn extract_having(p: &Arc) -> (Option, &Arc) { } } -fn extract_projection_and_having(p: &LogicalPlan) -> Option { +fn extract_projection_and_having(p: &LogicalPlan) -> Option> { match p { LogicalPlan::Projection { expr, diff --git a/rust/cubestore/cubestore/src/remotefs/mod.rs b/rust/cubestore/cubestore/src/remotefs/mod.rs index cba78cd47aecc..c7092b0c11e6c 100644 --- a/rust/cubestore/cubestore/src/remotefs/mod.rs +++ b/rust/cubestore/cubestore/src/remotefs/mod.rs @@ -91,7 +91,7 @@ pub trait ExtendedRemoteFs: DIService + RemoteFs { async fn list_by_page( &self, remote_prefix: String, - ) -> Result, CubeError>>, CubeError> { + ) -> Result, CubeError>>, CubeError> { // Note, this implementation doesn't actually paginate. let list: Vec = self.list(remote_prefix).await?; diff --git a/rust/cubestore/cubestore/src/remotefs/queue.rs b/rust/cubestore/cubestore/src/remotefs/queue.rs index fb6388ee61d6f..bb5beb585f87a 100644 --- a/rust/cubestore/cubestore/src/remotefs/queue.rs +++ b/rust/cubestore/cubestore/src/remotefs/queue.rs @@ -346,7 +346,7 @@ impl ExtendedRemoteFs for QueueRemoteFs { async fn list_by_page( &self, remote_prefix: String, - ) -> Result, CubeError>>, CubeError> { + ) -> Result, CubeError>>, CubeError> { self.remote_fs.list_by_page(remote_prefix).await } } diff --git a/rust/cubestore/cubestore/src/remotefs/s3.rs b/rust/cubestore/cubestore/src/remotefs/s3.rs index 32a0bb37a20bd..2dc5f943b4bb3 100644 --- a/rust/cubestore/cubestore/src/remotefs/s3.rs +++ b/rust/cubestore/cubestore/src/remotefs/s3.rs @@ -346,7 +346,7 @@ impl ExtendedRemoteFs for S3RemoteFs { async fn list_by_page( &self, remote_prefix: String, - ) -> Result, CubeError>>, CubeError> { + ) -> Result, CubeError>>, CubeError> { let path = self.s3_path(&remote_prefix); let bucket = self.bucket.load(); let leading_subpath = self.leading_subpath_regex(); diff --git a/rust/cubestore/cubestore/src/util/time_span.rs b/rust/cubestore/cubestore/src/util/time_span.rs index df74ad295e0ac..8e819c923784e 100644 --- a/rust/cubestore/cubestore/src/util/time_span.rs +++ b/rust/cubestore/cubestore/src/util/time_span.rs @@ -4,7 +4,7 @@ use std::time::{Duration, SystemTime}; /// The returned object will [log::warn] if it is alive longer than [timeout]. /// Be cautious when interpreting results in async code, this function looks at wall time. So future /// that is not running will add to the time. -pub fn warn_long(name: &str, timeout: Duration) -> ShortSpan { +pub fn warn_long(name: &str, timeout: Duration) -> ShortSpan<'_> { ShortSpan { name, timeout, diff --git a/rust/cubestore/cubezetasketch/src/difference_encoding.rs b/rust/cubestore/cubezetasketch/src/difference_encoding.rs index 7516732e90ba7..298def2958ce5 100644 --- a/rust/cubestore/cubezetasketch/src/difference_encoding.rs +++ b/rust/cubestore/cubezetasketch/src/difference_encoding.rs @@ -29,7 +29,7 @@ pub struct DifferenceEncoder<'l> { } impl DifferenceEncoder<'_> { - pub fn new(buf: &mut Vec) -> DifferenceEncoder { + pub fn new(buf: &mut Vec) -> DifferenceEncoder<'_> { return DifferenceEncoder { buf, last: 0 }; } @@ -88,7 +88,7 @@ pub struct DifferenceDecoder<'l> { } impl DifferenceDecoder<'_> { - pub fn new(data: &[u8]) -> DifferenceDecoder { + pub fn new(data: &[u8]) -> DifferenceDecoder<'_> { return DifferenceDecoder { data, last: 0 }; } } diff --git a/rust/cubestore/cubezetasketch/src/sparse.rs b/rust/cubestore/cubezetasketch/src/sparse.rs index 4531b5c2912ca..1ab1b70a94dba 100644 --- a/rust/cubestore/cubezetasketch/src/sparse.rs +++ b/rust/cubestore/cubezetasketch/src/sparse.rs @@ -327,7 +327,7 @@ impl SparseRepresentation { Ok(()) } - pub(crate) fn sorted_iterator(sparse_data: Option<&[u8]>) -> DifferenceDecoder { + pub(crate) fn sorted_iterator(sparse_data: Option<&[u8]>) -> DifferenceDecoder<'_> { return DifferenceDecoder::new(sparse_data.unwrap_or(&[])); } diff --git a/rust/cubestore/rust-toolchain.toml b/rust/cubestore/rust-toolchain.toml index ff511a5586793..ad8132da3e1bc 100644 --- a/rust/cubestore/rust-toolchain.toml +++ b/rust/cubestore/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2024-01-29" +channel = "nightly-2025-08-01" components = ["rustfmt", "clippy"] profile = "minimal"