Unblock release by publishing GPU crates (#5293)

AdamGS · robert3005 · web-flow · commit b4545084635b · 2025-11-13T02:10:38.000Z
Signed-off-by: Adam Gutglick &lt;adam@spiraldb.com&gt;
Signed-off-by: Robert Kruszewski &lt;github@robertk.io&gt;
Co-authored-by: Robert Kruszewski &lt;github@robertk.io&gt;
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -140,9 +140,9 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Docs
         run: |
-          RUSTDOCFLAGS="-D warnings" cargo doc --no-deps
+          RUSTFLAGS="--cfg gpu_unstable" RUSTDOCFLAGS="-D warnings" cargo doc --no-deps
           # nextest doesn't support doc tests, so we run it here
-          cargo test --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast
+          RUSTFLAGS="--cfg gpu_unstable" cargo test --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast
 
   build-rust:
     name: "Rust build (${{matrix.config.name}})"
@@ -156,7 +156,7 @@ jobs:
       - tag=${{ matrix.config.name }}
     env:
       # disable lints for build, they will be caught in Rust lint job.
-      RUSTFLAGS: "-A warnings"
+      RUSTFLAGS: "-A warnings --cfg gpu_unstable"
     strategy:
       fail-fast: false
       matrix:
@@ -199,6 +199,8 @@ jobs:
   check-min-deps:
     name: "Check build with minimal dependencies"
     timeout-minutes: 120
+    env:
+      RUSTFLAGS: "--cfg gpu_unstable"
     runs-on:
       - runs-on=${{ github.run_id }}
       - family=m7i+m7i-flex+m7a
@@ -221,6 +223,8 @@ jobs:
   rust-lint:
     name: "Rust (lint)"
     timeout-minutes: 120
+    env:
+      RUSTFLAGS: "--cfg gpu_unstable"
     runs-on:
       - runs-on=${{ github.run_id }}
       - family=m7i+m7i-flex+m7a
@@ -249,6 +253,8 @@ jobs:
   rust-lint-no-default:
     name: "Rust (lint, no default)"
     timeout-minutes: 120
+    env:
+      RUSTFLAGS: "--cfg gpu_unstable"
     runs-on:
       - runs-on=${{ github.run_id }}
       - family=m7i+m7i-flex+m7a
@@ -415,6 +421,7 @@ jobs:
       # Keep frame pointers for better stack traces
       CARGO_PROFILE_DEV_DEBUG: "true"
       CARGO_PROFILE_TEST_DEBUG: "true"
+      RUSTFLAGS: "--cfg gpu_unstable"
     steps:
       - uses: runs-on/action@v2
         with:
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -34,14 +34,7 @@ jobs:
 
       - name: Release
         run: |
-          cargo +nightly publish -Zpublish-timeout --no-verify --allow-dirty --workspace \
-          --exclude bench-vortex \
-          --exclude vortex-python \
-          --exclude vortex-duckdb \
-          --exclude vortex-ffi \
-          --exclude vortex-fuzz \
-          --exclude vortex-jni \
-          --exclude xtask
+          cargo +nightly publish -Zpublish-timeout --no-verify --allow-dirty --workspace
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -281,6 +281,7 @@ unexpected_cfgs = { level = "deny", check-cfg = [
     "cfg(codspeed)",
     "cfg(disable_loom)",
     "cfg(vortex_nightly)",
+    "cfg(gpu_unstable)",
 ] }
 warnings = "warn"
 
diff --git a/vortex-buffer/Cargo.toml b/vortex-buffer/Cargo.toml
@@ -18,7 +18,6 @@ all-features = true
 
 [features]
 arrow = []
-cuda = ["dep:cudarc"]
 memmap2 = ["dep:memmap2"]
 serde = ["dep:serde"]
 warn-copy = ["dep:log"]
@@ -27,7 +26,6 @@ warn-copy = ["dep:log"]
 arrow-buffer = { workspace = true }
 bitvec = { workspace = true }
 bytes = { workspace = true }
-cudarc = { workspace = true, optional = true }
 itertools = { workspace = true }
 log = { workspace = true, optional = true }
 memmap2 = { workspace = true, optional = true }
@@ -36,6 +34,9 @@ serde = { workspace = true, optional = true }
 simdutf8 = { workspace = true }
 vortex-error = { workspace = true }
 
+[target.'cfg(gpu_unstable)'.dependencies]
+cudarc = { workspace = true }
+
 [lints]
 workspace = true
 
diff --git a/vortex-buffer/src/lib.rs b/vortex-buffer/src/lib.rs
@@ -64,7 +64,7 @@ mod buffer;
 mod buffer_mut;
 mod bytes;
 mod r#const;
-#[cfg(feature = "cuda")]
+#[cfg(gpu_unstable)]
 mod cuda;
 mod debug;
 mod macros;
diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml
@@ -19,7 +19,6 @@ all-features = true
 [dependencies]
 async-trait = { workspace = true }
 bytes = { workspace = true }
-cudarc = { workspace = true, optional = true }
 flatbuffers = { workspace = true }
 futures = { workspace = true, features = ["std", "async-await"] }
 # Needed to pickup the "wasm_js" feature for wasm targets from the workspace configuration
@@ -43,7 +42,6 @@ vortex-error = { workspace = true }
 vortex-fastlanes = { workspace = true }
 vortex-flatbuffers = { workspace = true, features = ["file"] }
 vortex-fsst = { workspace = true }
-vortex-gpu = { workspace = true, optional = true }
 vortex-io = { workspace = true }
 vortex-layout = { workspace = true }
 vortex-metrics = { workspace = true }
@@ -77,10 +75,7 @@ tokio = [
     "vortex-scan/tokio",
 ]
 zstd = ["dep:vortex-zstd", "vortex-layout/zstd"]
-gpu = [
-    "dep:cudarc",
-    "dep:vortex-gpu",
-    "vortex-gpu/cuda",
-    "vortex-layout/gpu",
-    "vortex-scan/gpu",
-]
+
+[target.'cfg(gpu_unstable)'.dependencies]
+cudarc = { workspace = true }
+vortex-gpu = { workspace = true, features = ["cuda"] }
diff --git a/vortex-file/src/file.rs b/vortex-file/src/file.rs
@@ -96,7 +96,7 @@ impl VortexFile {
         )
     }
 
-    #[cfg(feature = "gpu")]
+    #[cfg(gpu_unstable)]
     pub fn gpu_scan(
         &self,
         ctx: Arc<cudarc::driver::CudaContext>,
diff --git a/vortex-gpu-kernels/Cargo.toml b/vortex-gpu-kernels/Cargo.toml
@@ -12,7 +12,6 @@ readme = { workspace = true }
 repository = { workspace = true }
 rust-version = { workspace = true }
 version = { workspace = true }
-publish = false
 
 [lib]
 name = "vortex_gpu_kernels"
diff --git a/vortex-gpu/Cargo.toml b/vortex-gpu/Cargo.toml
@@ -12,20 +12,16 @@ readme = { workspace = true }
 repository = { workspace = true }
 rust-version = { workspace = true }
 version = { workspace = true }
-publish = false
 
 [dependencies]
 cudarc = { workspace = true }
-itertools = { workspace = true }
-parking_lot = { workspace = true }
 vortex-alp = { workspace = true }
 vortex-array = { workspace = true }
-vortex-buffer = { workspace = true, features = ["cuda"] }
+vortex-buffer = { workspace = true }
 vortex-dtype = { workspace = true }
 vortex-error = { workspace = true }
 vortex-fastlanes = { workspace = true }
 vortex-mask = { workspace = true }
-vortex-utils = { workspace = true }
 
 [dev-dependencies]
 criterion = { version = "0.7", features = ["html_reports"] }
diff --git a/vortex-gpu/src/lib.rs b/vortex-gpu/src/lib.rs
@@ -1,6 +1,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+//! This crate contains support for GPU and CUDA accelerated compute for Vortex.
+//!
+//! This crate is currently considered unstable, and much of its code is behind a the `gpu_unstable` config.
+//! If you wish to use it, you should build your code with:
+//! ```shell
+//! RUSTFLAGS="--cfg gpu_unstable" cargo build -p ...
+//! ```
+
 pub mod bit_unpack;
 pub mod for_;
 mod for_bp;
diff --git a/vortex-layout/Cargo.toml b/vortex-layout/Cargo.toml
@@ -21,7 +21,6 @@ arcref = { workspace = true }
 arrow-buffer = { workspace = true }
 async-stream = { workspace = true }
 async-trait = { workspace = true }
-cudarc = { workspace = true, optional = true }
 flatbuffers = { workspace = true }
 futures = { workspace = true, features = ["alloc", "async-await", "executor"] }
 itertools = { workspace = true }
@@ -46,7 +45,6 @@ vortex-decimal-byte-parts = { workspace = true }
 vortex-dtype = { workspace = true }
 vortex-error = { workspace = true }
 vortex-flatbuffers = { workspace = true, features = ["layout"] }
-vortex-gpu = { workspace = true, optional = true }
 vortex-io = { workspace = true }
 vortex-mask = { workspace = true }
 vortex-metrics = { workspace = true }
@@ -57,6 +55,10 @@ vortex-session = { workspace = true }
 vortex-utils = { workspace = true, features = ["dashmap"] }
 vortex-zstd = { workspace = true, optional = true }
 
+[target.'cfg(gpu_unstable)'.dependencies]
+cudarc = { workspace = true }
+vortex-gpu = { workspace = true, features = ["cuda"] }
+
 [dev-dependencies]
 futures = { workspace = true, features = ["executor"] }
 rstest = { workspace = true }
@@ -68,8 +70,6 @@ vortex-io = { path = "../vortex-io", features = ["tokio"] }
 test-harness = []
 tokio = ["dep:tokio", "vortex-error/tokio"]
 zstd = ["dep:vortex-zstd"]
-gpu = ["cuda", "dep:vortex-gpu"]
-cuda = ["dep:cudarc", "vortex-gpu/cuda"]
 
 [lints]
 workspace = true
diff --git a/vortex-layout/src/layout.rs b/vortex-layout/src/layout.rs
@@ -51,7 +51,7 @@ pub trait Layout: 'static + Send + Sync + Debug + private::Sealed {
     /// Get the segment IDs for this layout.
     fn segment_ids(&self) -> Vec<SegmentId>;
 
-    #[cfg(feature = "gpu")]
+    #[cfg(gpu_unstable)]
     fn new_gpu_reader(
         &self,
         name: Arc<str>,
@@ -259,7 +259,7 @@ impl<V: VTable> Layout for LayoutAdapter<V> {
         V::segment_ids(&self.0)
     }
 
-    #[cfg(feature = "gpu")]
+    #[cfg(gpu_unstable)]
     fn new_gpu_reader(
         &self,
         name: Arc<str>,
diff --git a/vortex-layout/src/layouts/chunked/mod.rs b/vortex-layout/src/layouts/chunked/mod.rs
@@ -72,7 +72,7 @@ impl VTable for ChunkedVTable {
         )))
     }
 
-    #[cfg(feature = "gpu")]
+    #[cfg(gpu_unstable)]
     fn new_gpu_reader(
         layout: &Self::Layout,
         name: Arc<str>,
diff --git a/vortex-layout/src/layouts/dict/mod.rs b/vortex-layout/src/layouts/dict/mod.rs
@@ -83,7 +83,7 @@ impl VTable for DictVTable {
         )?))
     }
 
-    #[cfg(feature = "gpu")]
+    #[cfg(gpu_unstable)]
     fn new_gpu_reader(
         _layout: &Self::Layout,
         _name: Arc<str>,
diff --git a/vortex-layout/src/layouts/flat/mod.rs b/vortex-layout/src/layouts/flat/mod.rs
@@ -79,7 +79,7 @@ impl VTable for FlatVTable {
         )))
     }
 
-    #[cfg(feature = "gpu")]
+    #[cfg(gpu_unstable)]
     fn new_gpu_reader(
         layout: &Self::Layout,
         name: Arc<str>,
diff --git a/vortex-layout/src/layouts/struct_/mod.rs b/vortex-layout/src/layouts/struct_/mod.rs
@@ -104,7 +104,7 @@ impl VTable for StructVTable {
         )?))
     }
 
-    #[cfg(feature = "gpu")]
+    #[cfg(gpu_unstable)]
     fn new_gpu_reader(
         layout: &Self::Layout,
         name: Arc<str>,
diff --git a/vortex-layout/src/layouts/zoned/mod.rs b/vortex-layout/src/layouts/zoned/mod.rs
@@ -91,7 +91,7 @@ impl VTable for ZonedVTable {
         )?))
     }
 
-    #[cfg(feature = "gpu")]
+    #[cfg(gpu_unstable)]
     fn new_gpu_reader(
         layout: &Self::Layout,
         name: Arc<str>,
diff --git a/vortex-layout/src/lib.rs b/vortex-layout/src/lib.rs
@@ -6,7 +6,7 @@ pub mod layouts;
 pub use children::*;
 pub use encoding::*;
 pub use flatbuffers::*;
-#[cfg(feature = "gpu")]
+#[cfg(gpu_unstable)]
 pub use gpu::*;
 pub use layout::*;
 pub use reader::*;
@@ -18,7 +18,7 @@ mod children;
 pub mod display;
 mod encoding;
 mod flatbuffers;
-#[cfg(feature = "gpu")]
+#[cfg(gpu_unstable)]
 pub mod gpu;
 mod layout;
 mod reader;
diff --git a/vortex-layout/src/vtable.rs b/vortex-layout/src/vtable.rs
@@ -55,7 +55,7 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug {
         segment_source: Arc<dyn SegmentSource>,
     ) -> VortexResult<LayoutReaderRef>;
 
-    #[cfg(feature = "gpu")]
+    #[cfg(gpu_unstable)]
     /// Create a new reader for the layout that uses a gpu device
     fn new_gpu_reader(
         layout: &Self::Layout,
diff --git a/vortex-scan/Cargo.toml b/vortex-scan/Cargo.toml
@@ -23,15 +23,13 @@ vortex-array = { workspace = true }
 vortex-buffer = { workspace = true }
 vortex-dtype = { workspace = true }
 vortex-error = { workspace = true }
-vortex-gpu = { workspace = true, optional = true }
 vortex-io = { workspace = true }
 vortex-layout = { workspace = true }
 vortex-mask = { workspace = true }
 vortex-metrics = { workspace = true }
 vortex-session = { workspace = true }
 
 bit-vec = { workspace = true }
-cudarc = { workspace = true, optional = true }
 futures = { workspace = true }
 itertools = { workspace = true }
 log = { workspace = true }
@@ -47,8 +45,10 @@ vortex-layout = { workspace = true, features = ["test-harness"] }
 default = []
 roaring = ["dep:roaring"]
 tokio = ["dep:tokio"]
-gpu = ["cuda", "dep:vortex-gpu", "vortex-layout/gpu"]
-cuda = ["dep:cudarc", "vortex-gpu/cuda"]
+
+[target.'cfg(gpu_unstable)'.dependencies]
+cudarc = { workspace = true }
+vortex-gpu = { workspace = true, features = ["cuda"] }
 
 [lints]
 workspace = true
diff --git a/vortex-scan/src/lib.rs b/vortex-scan/src/lib.rs
@@ -21,7 +21,7 @@ pub use split_by::SplitBy;
 mod scan_builder;
 pub use scan_builder::ScanBuilder;
 
-#[cfg(feature = "gpu")]
+#[cfg(gpu_unstable)]
 pub mod gpu;
 mod repeated_scan;
 

Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ impl VortexFile {`
`96`	`96`	`)`
`97`	`97`	`}`
`98`	`98`
`99`		`- #[cfg(feature = "gpu")]`
	`99`	`+ #[cfg(gpu_unstable)]`
`100`	`100`	`pub fn gpu_scan(`
`101`	`101`	`&self,`
`102`	`102`	`ctx: Arc<cudarc::driver::CudaContext>,`