diff --git a/Cargo.lock b/Cargo.lock index d42db12..cb7e228 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,21 +102,30 @@ name = "cargo-gpu" version = "0.1.0" dependencies = [ "anyhow", - "cargo-util-schemas", + "cargo-gpu-build", "cargo_metadata", "clap", "crossterm", - "directories", "dunce", "env_logger", "log", "relative-path", - "rustc_codegen_spirv-target-specs", - "semver", "serde", "serde_json", "spirv-builder", + "tempfile", "test-log", + "thiserror", +] + +[[package]] +name = "cargo-gpu-build" +version = "0.1.0" +dependencies = [ + "dunce", + "log", + "rustc_codegen_spirv-cache", + "thiserror", ] [[package]] @@ -1026,6 +1035,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_codegen_spirv-cache" +version = "0.1.0" +dependencies = [ + "cargo-util-schemas", + "cargo_metadata", + "clap", + "directories", + "log", + "rustc_codegen_spirv-target-specs", + "serde", + "spirv-builder", + "test-log", + "thiserror", +] + [[package]] name = "rustc_codegen_spirv-target-specs" version = "0.9.0" @@ -1035,7 +1060,7 @@ checksum = "6c89eaf493b3dfc730cda42a77014aad65e03213992c7afe0dff60a9f7d3dd94" [[package]] name = "rustc_codegen_spirv-types" version = "0.9.0" -source = "git+https://github.com/Rust-GPU/rust-gpu?rev=3df836eb9d7b01344f52737bf9a310d1fb5a0c26#3df836eb9d7b01344f52737bf9a310d1fb5a0c26" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=13a80b7ed9fbd5738746ef31ab8ddfee3f403551#13a80b7ed9fbd5738746ef31ab8ddfee3f403551" dependencies = [ "rspirv", "serde", @@ -1215,7 +1240,7 @@ dependencies = [ [[package]] name = "spirv-builder" version = "0.9.0" -source = "git+https://github.com/Rust-GPU/rust-gpu?rev=3df836eb9d7b01344f52737bf9a310d1fb5a0c26#3df836eb9d7b01344f52737bf9a310d1fb5a0c26" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=13a80b7ed9fbd5738746ef31ab8ddfee3f403551#13a80b7ed9fbd5738746ef31ab8ddfee3f403551" dependencies = [ "cargo_metadata", "clap", diff --git a/Cargo.toml b/Cargo.toml index 62ca2ec..4bd5d64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,7 @@ [workspace] members = [ + "crates/rustc_codegen_spirv-cache", + "crates/cargo-gpu-build", "crates/cargo-gpu", "crates/xtask", ] @@ -7,14 +9,22 @@ members = [ exclude = [ # This currently needs to be excluded because it depends on a version of `rust-gpu` that # uses a toolchain whose Cargo version doesn't recognise version 4 of `Cargo.lock`. - "crates/shader-crate-template" + "crates/shader-crate-template", ] resolver = "2" +[workspace.package] +version = "0.1.0" +edition = "2021" +repository = "https://github.com/Rust-GPU/cargo-gpu" +keywords = ["gpu", "compiler", "rust-gpu"] +license = "MIT OR Apache-2.0" + [workspace.dependencies] -spirv-builder = { git = "https://github.com/Rust-GPU/rust-gpu", rev = "3df836eb9d7b01344f52737bf9a310d1fb5a0c26", default-features = false } +spirv-builder = { git = "https://github.com/Rust-GPU/rust-gpu", rev = "13a80b7ed9fbd5738746ef31ab8ddfee3f403551", default-features = false } anyhow = "1.0.98" +thiserror = "2.0.12" clap = { version = "4.5.41", features = ["derive"] } crossterm = "0.29.0" directories = "6.0.0" @@ -28,7 +38,6 @@ tempfile = "3.22" test-log = "0.2.18" cargo_metadata = "0.21.0" cargo-util-schemas = "0.8.2" -semver = "1.0.26" dunce = "1.0.5" # This crate MUST NEVER be upgraded, we need this particular "first" version to support old rust-gpu builds diff --git a/crates/cargo-gpu-build/Cargo.toml b/crates/cargo-gpu-build/Cargo.toml new file mode 100644 index 0000000..8312e93 --- /dev/null +++ b/crates/cargo-gpu-build/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "cargo-gpu-build" +description = "Builder of `rust-gpu` shader crates" +version.workspace = true +edition.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true + +[dependencies] +rustc_codegen_spirv-cache = { path = "../rustc_codegen_spirv-cache" } +dunce.workspace = true +thiserror.workspace = true +log.workspace = true + +[features] +# Rebuilds target shader crate upon changes +watch = ["rustc_codegen_spirv-cache/watch"] +# Enables `clap` support for public structs +clap = ["rustc_codegen_spirv-cache/clap"] + +[lints] +workspace = true diff --git a/crates/cargo-gpu-build/src/build.rs b/crates/cargo-gpu-build/src/build.rs new file mode 100644 index 0000000..e0d15e0 --- /dev/null +++ b/crates/cargo-gpu-build/src/build.rs @@ -0,0 +1,339 @@ +//! This module provides a `rust-gpu` shader crate builder +//! usable inside of build scripts or as a part of CLI. + +use std::{io, process::Stdio}; + +use crate::{ + lockfile::{LockfileMismatchError, LockfileMismatchHandler}, + spirv_builder::{CompileResult, SpirvBuilder, SpirvBuilderError}, + spirv_cache::{ + backend::{ + SpirvCodegenBackend, SpirvCodegenBackendInstallError, SpirvCodegenBackendInstallParams, + SpirvCodegenBackendInstaller, + }, + command::CommandExecError, + toolchain::{ + HaltToolchainInstallation, InheritStderr, InheritStdout, NoopOnComponentsInstall, + NoopOnToolchainInstall, StdioCfg, + }, + user_output, + }, +}; + +#[cfg(feature = "watch")] +use crate::spirv_builder::SpirvWatcher; + +/// Parameters for [`CargoGpuBuilder::new()`]. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct CargoGpuBuilderParams { + /// Parameters of the shader crate build. + pub build: SpirvBuilder, + /// Parameters of the codegen backend installation for the shader crate. + pub install: SpirvCodegenBackendInstaller, + /// There is a tricky situation where a shader crate that depends on workspace config can have + /// a different `Cargo.lock` lockfile version from the the workspace's `Cargo.lock`. This can + /// prevent builds when an old Rust toolchain doesn't recognise the newer lockfile version. + /// + /// The ideal way to resolve this would be to match the shader crate's toolchain with the + /// workspace's toolchain. However, that is not always possible. Another solution is to + /// `exclude = [...]` the problematic shader crate from the workspace. This also may not be a + /// suitable solution if there are a number of shader crates all sharing similar config and + /// you don't want to have to copy/paste and maintain that config across all the shaders. + /// + /// So a somewhat hacky workaround is to overwrite lockfile versions. Enabling this flag + /// will only come into effect if there are a mix of v3/v4 lockfiles. It will also + /// only overwrite versions for the duration of a build. It will attempt to return the versions + /// to their original values once the build is finished. However, of course, unexpected errors + /// can occur and the overwritten values can remain. Hence why this behaviour is not enabled by + /// default. + /// + /// This hack is possible because the change from v3 to v4 only involves a minor change to the + /// way source URLs are encoded. See these PRs for more details: + /// * + /// * + pub force_overwrite_lockfiles_v4_to_v3: bool, + /// Writer of user output. + pub writer: W, + /// Callbacks to halt toolchain installation. + pub halt: HaltToolchainInstallation, + /// Configuration of [`Stdio`] for commands run during installation. + pub stdio_cfg: StdioCfg, +} + +impl CargoGpuBuilderParams { + /// Replaces build parameters of the shader crate. + #[inline] + #[must_use] + pub fn build(self, build: SpirvBuilder) -> Self { + Self { build, ..self } + } + + /// Replaces codegen backend installation parameters of the shader crate. + #[inline] + #[must_use] + pub fn install(self, install: SpirvCodegenBackendInstaller) -> Self { + Self { install, ..self } + } + + /// Sets whether to force overwriting lockfiles from v4 to v3. + #[inline] + #[must_use] + pub fn force_overwrite_lockfiles_v4_to_v3( + self, + force_overwrite_lockfiles_v4_to_v3: bool, + ) -> Self { + Self { + force_overwrite_lockfiles_v4_to_v3, + ..self + } + } + + /// Replaces the writer of user output. + #[inline] + #[must_use] + pub fn writer(self, writer: NW) -> CargoGpuBuilderParams { + CargoGpuBuilderParams { + build: self.build, + install: self.install, + force_overwrite_lockfiles_v4_to_v3: self.force_overwrite_lockfiles_v4_to_v3, + writer, + halt: self.halt, + stdio_cfg: self.stdio_cfg, + } + } + + /// Replaces the callbacks to halt toolchain installation. + #[inline] + #[must_use] + pub fn halt( + self, + halt: HaltToolchainInstallation, + ) -> CargoGpuBuilderParams { + CargoGpuBuilderParams { + build: self.build, + install: self.install, + force_overwrite_lockfiles_v4_to_v3: self.force_overwrite_lockfiles_v4_to_v3, + writer: self.writer, + halt, + stdio_cfg: self.stdio_cfg, + } + } + + /// Replaces the [`Stdio`] configuration for commands run during installation. + #[inline] + #[must_use] + pub fn stdio_cfg( + self, + stdio_cfg: StdioCfg, + ) -> CargoGpuBuilderParams { + CargoGpuBuilderParams { + build: self.build, + install: self.install, + force_overwrite_lockfiles_v4_to_v3: self.force_overwrite_lockfiles_v4_to_v3, + writer: self.writer, + halt: self.halt, + stdio_cfg, + } + } +} + +/// [`Default`] parameters for [`CargoGpuBuilder::new()`]. +pub type DefaultCargoGpuBuilderParams = CargoGpuBuilderParams< + io::Stdout, + NoopOnToolchainInstall, + NoopOnComponentsInstall, + InheritStdout, + InheritStderr, +>; + +impl From for DefaultCargoGpuBuilderParams { + #[inline] + fn from(build: SpirvBuilder) -> Self { + Self { + build, + ..Self::default() + } + } +} + +impl Default for DefaultCargoGpuBuilderParams { + #[inline] + fn default() -> Self { + Self { + build: SpirvBuilder::default(), + install: SpirvCodegenBackendInstaller::default(), + force_overwrite_lockfiles_v4_to_v3: false, + writer: io::stdout(), + halt: HaltToolchainInstallation::noop(), + stdio_cfg: StdioCfg::inherit(), + } + } +} + +/// A builder for compiling a `rust-gpu` shader crate. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct CargoGpuBuilder { + /// The underlying builder for compiling the shader crate. + pub builder: SpirvBuilder, + /// The underlying codegen backend installer for the shader crate. + pub installer: SpirvCodegenBackendInstaller, + /// The installed codegen backend. + pub codegen_backend: SpirvCodegenBackend, + /// The lockfile mismatch handler. + pub lockfile_mismatch_handler: LockfileMismatchHandler, + /// Writer of user output. + pub writer: W, +} + +impl CargoGpuBuilder +where + W: io::Write, +{ + /// Creates shader crate builder, allowing to modify install and build parameters separately. + /// + /// # Errors + /// + /// Returns an error if: + /// + /// * the shader crate path / target was not set, + /// * the shader crate path is not valid, + /// * the backend installation fails, + /// * there is a lockfile version mismatch that cannot be resolved automatically. + #[inline] + pub fn new(params: I) -> Result> + where + I: Into>, + R: From, + T: FnOnce(&str) -> Result<(), R>, + C: FnOnce(&str) -> Result<(), R>, + O: FnMut() -> Stdio, + E: FnMut() -> Stdio, + { + let CargoGpuBuilderParams { + mut build, + install, + force_overwrite_lockfiles_v4_to_v3, + mut writer, + halt, + mut stdio_cfg, + } = params.into(); + + if build.target.is_none() { + return Err(NewCargoGpuBuilderError::MissingTarget); + } + let path_to_crate = build + .path_to_crate + .as_ref() + .ok_or(NewCargoGpuBuilderError::MissingCratePath)?; + let shader_crate = dunce::canonicalize(path_to_crate)?; + build.path_to_crate = Some(shader_crate.clone()); + + let backend_install_params = SpirvCodegenBackendInstallParams::from(&shader_crate) + .writer(&mut writer) + .halt(HaltToolchainInstallation { + on_toolchain_install: |channel: &str| (halt.on_toolchain_install)(channel), + on_components_install: |channel: &str| (halt.on_components_install)(channel), + }) + .stdio_cfg(StdioCfg { + stdout: || (stdio_cfg.stdout)(), + stderr: || (stdio_cfg.stderr)(), + }); + let codegen_backend = install.install(backend_install_params)?; + + let lockfile_mismatch_handler = LockfileMismatchHandler::new( + &shader_crate, + &codegen_backend.toolchain_channel, + force_overwrite_lockfiles_v4_to_v3, + )?; + + #[expect(clippy::unreachable, reason = "target was set")] + codegen_backend + .configure_spirv_builder(&mut build) + .unwrap_or_else(|_| unreachable!("target was set before calling this function")); + + Ok(Self { + builder: build, + installer: install, + codegen_backend, + lockfile_mismatch_handler, + writer, + }) + } + + /// Builds the shader crate using the configured [`SpirvBuilder`]. + /// + /// # Errors + /// + /// Returns an error if building the shader crate failed. + #[inline] + pub fn build(&mut self) -> Result { + let shader_crate = self + .builder + .path_to_crate + .as_ref() + .ok_or(SpirvBuilderError::MissingCratePath)? + .display(); + user_output!(&mut self.writer, "Compiling shaders at {shader_crate}...\n")?; + + let result = self.builder.build()?; + Ok(result) + } + + /// Watches the shader crate for changes using the configured [`SpirvBuilder`]. + /// + /// # Errors + /// + /// Returns an error if watching shader crate for changes failed. + #[cfg(feature = "watch")] + #[inline] + pub fn watch(&mut self) -> Result { + let shader_crate = self + .builder + .path_to_crate + .as_ref() + .ok_or(SpirvBuilderError::MissingCratePath)? + .display(); + user_output!( + &mut self.writer, + "Watching shaders for changes at {shader_crate}...\n" + )?; + + let watcher = self.builder.clone().watch()?; + Ok(watcher) + } +} + +/// An error indicating what went wrong when creating a [`CargoGpuBuilder`]. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum NewCargoGpuBuilderError { + /// Shader crate target is missing from parameters of the build. + #[error("shader crate target must be set, for example `spirv-unknown-vulkan1.2`")] + MissingTarget, + /// Shader path is missing from parameters of the build. + #[error("path to shader crate must be set")] + MissingCratePath, + /// The given shader crate path is not valid. + #[error("shader crate path is not valid: {0}")] + InvalidCratePath(#[from] io::Error), + /// The backend installation failed. + #[error("could not install backend: {0}")] + Install(#[from] SpirvCodegenBackendInstallError), + /// There is a lockfile version mismatch that cannot be resolved automatically. + #[error(transparent)] + LockfileMismatch(#[from] LockfileMismatchError), +} + +/// An error indicating what went wrong when building the shader crate. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum CargoGpuBuildError { + /// Failed to write user output. + #[error("failed to write user output: {0}")] + IoWrite(#[from] io::Error), + /// Failed to build shader crate. + #[error("failed to build shader crate: {0}")] + Build(#[from] SpirvBuilderError), +} diff --git a/crates/cargo-gpu-build/src/lib.rs b/crates/cargo-gpu-build/src/lib.rs new file mode 100644 index 0000000..a3d973c --- /dev/null +++ b/crates/cargo-gpu-build/src/lib.rs @@ -0,0 +1,21 @@ +//! Rust GPU shader crate builder. +//! +//! This library allows you to easily compile your `rust-gpu` shaders, +//! without requiring you to fix your entire project to a specific toolchain. +//! +//! # How it works +//! +//! This library manages installations of `rustc_codegen_spirv` +//! using [`rustc_codegen_spirv-cache`](spirv_cache) crate. +//! +//! Then is uses [`spirv-builder`](spirv_builder) crate +//! to pass the many additional parameters required to configure rustc and our codegen backend, +//! but provide you with a toolchain-agnostic version that you may use from stable rustc. + +#![expect(clippy::pub_use, reason = "pub use for build scripts")] + +pub use rustc_codegen_spirv_cache as spirv_cache; +pub use rustc_codegen_spirv_cache::spirv_builder; + +pub mod build; +pub mod lockfile; diff --git a/crates/cargo-gpu-build/src/lockfile.rs b/crates/cargo-gpu-build/src/lockfile.rs new file mode 100644 index 0000000..1c32947 --- /dev/null +++ b/crates/cargo-gpu-build/src/lockfile.rs @@ -0,0 +1,359 @@ +//! Handles lockfile version conflicts and downgrades. +//! +//! Stable uses lockfile v4, but `rust-gpu` v0.9.0 uses an old toolchain requiring v3 +//! and will refuse to build shader crate with a v4 lockfile being present. +//! This module takes care of warning the user and potentially downgrading the lockfile. + +#![expect(clippy::non_ascii_literal, reason = "'⚠️' character is really needed")] + +use std::{ + fs, + io::{self, Write as _}, + path::{Path, PathBuf}, +}; + +use crate::spirv_cache::{cargo_metadata::semver::Version, spirv_builder::query_rustc_version}; + +/// `Cargo.lock` manifest version 4 became the default in Rust 1.83.0. Conflicting manifest +/// versions between the workspace and the shader crate, can cause problems. +const RUST_VERSION_THAT_USES_V4_CARGO_LOCKS: Version = Version::new(1, 83, 0); + +/// Cargo dependency for `spirv-builder` and the rust toolchain channel. +#[derive(Debug, Clone)] +#[non_exhaustive] +#[expect(clippy::module_name_repetitions, reason = "it is intended")] +pub struct LockfileMismatchHandler { + /// `Cargo.lock`s that have had their manifest versions changed by us and need changing back. + pub cargo_lock_files_with_changed_manifest_versions: Vec, +} + +impl LockfileMismatchHandler { + /// Creates self from the given parameters. + /// + /// # Errors + /// + /// Returns an error if there was a problem checking or changing lockfile manifest versions. + /// See [`LockfileMismatchError`] for details. + #[inline] + pub fn new( + shader_crate_path: &Path, + toolchain_channel: &str, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> Result { + let mut cargo_lock_files_with_changed_manifest_versions = vec![]; + + let maybe_shader_crate_lock = + Self::ensure_workspace_rust_version_does_not_conflict_with_shader( + shader_crate_path, + is_force_overwrite_lockfiles_v4_to_v3, + )?; + + if let Some(shader_crate_lock) = maybe_shader_crate_lock { + cargo_lock_files_with_changed_manifest_versions.push(shader_crate_lock); + } + + let maybe_workspace_crate_lock = + Self::ensure_shader_rust_version_does_not_conflict_with_any_cargo_locks( + shader_crate_path, + toolchain_channel, + is_force_overwrite_lockfiles_v4_to_v3, + )?; + + if let Some(workspace_crate_lock) = maybe_workspace_crate_lock { + cargo_lock_files_with_changed_manifest_versions.push(workspace_crate_lock); + } + + Ok(Self { + cargo_lock_files_with_changed_manifest_versions, + }) + } + + /// See docs for [`force_overwrite_lockfiles_v4_to_v3`](field@crate::build::CargoGpuBuilderParams::force_overwrite_lockfiles_v4_to_v3) + /// flag for why we do this. + fn ensure_workspace_rust_version_does_not_conflict_with_shader( + shader_crate_path: &Path, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> Result, LockfileMismatchError> { + log::debug!("Ensuring no v3/v4 `Cargo.lock` conflicts from workspace Rust..."); + let workspace_rust_version = + query_rustc_version(None).map_err(LockfileMismatchError::QueryRustcVersion)?; + if workspace_rust_version >= RUST_VERSION_THAT_USES_V4_CARGO_LOCKS { + log::debug!( + "user's Rust is v{workspace_rust_version}, so no v3/v4 conflicts possible." + ); + return Ok(None); + } + + Self::handle_conflicting_cargo_lock_v4( + shader_crate_path, + is_force_overwrite_lockfiles_v4_to_v3, + )?; + + if is_force_overwrite_lockfiles_v4_to_v3 { + Ok(Some(shader_crate_path.join("Cargo.lock"))) + } else { + Ok(None) + } + } + + /// See docs for [`force_overwrite_lockfiles_v4_to_v3`](field@crate::build::CargoGpuBuilderParams::force_overwrite_lockfiles_v4_to_v3) + /// flag for why we do this. + fn ensure_shader_rust_version_does_not_conflict_with_any_cargo_locks( + shader_crate_path: &Path, + channel: &str, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> Result, LockfileMismatchError> { + log::debug!("Ensuring no v3/v4 `Cargo.lock` conflicts from shader's Rust..."); + let shader_rust_version = + query_rustc_version(Some(channel)).map_err(LockfileMismatchError::QueryRustcVersion)?; + if shader_rust_version >= RUST_VERSION_THAT_USES_V4_CARGO_LOCKS { + log::debug!("shader's Rust is v{shader_rust_version}, so no v3/v4 conflicts possible."); + return Ok(None); + } + + log::debug!( + "shader's Rust is v{shader_rust_version}, so checking both shader and workspace `Cargo.lock` manifest versions..." + ); + + if shader_crate_path.join("Cargo.lock").exists() { + // Note that we don't return the `Cargo.lock` here (so that it's marked for reversion + // after the build), because we can be sure that updating it now is actually updating it + // to the state it should have been all along. Therefore it doesn't need reverting once + // fixed. + Self::handle_conflicting_cargo_lock_v4( + shader_crate_path, + is_force_overwrite_lockfiles_v4_to_v3, + )?; + } + + if let Some(workspace_root) = Self::get_workspace_root(shader_crate_path)? { + Self::handle_conflicting_cargo_lock_v4( + workspace_root, + is_force_overwrite_lockfiles_v4_to_v3, + )?; + return Ok(Some(workspace_root.join("Cargo.lock"))); + } + + Ok(None) + } + + /// Get the path to the shader crate's workspace, if it has one. We can't use the traditional + /// `cargo metadata` because if the workspace has a conflicting `Cargo.lock` manifest version + /// then that command won't work. Instead we do an old school recursive file tree walk. + fn get_workspace_root( + shader_crate_path: &Path, + ) -> Result, LockfileMismatchError> { + let shader_cargo_toml_path = shader_crate_path.join("Cargo.toml"); + let shader_cargo_toml = match fs::read_to_string(shader_cargo_toml_path) { + Ok(contents) => contents, + Err(source) => { + let file = shader_crate_path.join("Cargo.toml"); + return Err(LockfileMismatchError::ReadFile { file, source }); + } + }; + if !shader_cargo_toml.contains("workspace = true") { + return Ok(None); + } + + let mut current_path = shader_crate_path; + #[expect(clippy::default_numeric_fallback, reason = "It's just a loop")] + for _ in 0..15 { + if let Some(parent_path) = current_path.parent() { + if parent_path.join("Cargo.lock").exists() { + return Ok(Some(parent_path)); + } + current_path = parent_path; + } else { + break; + } + } + + Ok(None) + } + + /// When Rust < 1.83.0 is being used an error will occur if it tries to parse `Cargo.lock` + /// files that use lockfile manifest version 4. Here we check and handle that. + fn handle_conflicting_cargo_lock_v4( + folder: &Path, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> Result<(), LockfileMismatchError> { + let shader_cargo_lock_path = folder.join("Cargo.lock"); + let shader_cargo_lock = match fs::read_to_string(&shader_cargo_lock_path) { + Ok(contents) => contents, + Err(source) => { + let file = shader_cargo_lock_path; + return Err(LockfileMismatchError::ReadFile { file, source }); + } + }; + + let Some(third_line) = shader_cargo_lock.lines().nth(2) else { + let file = shader_cargo_lock_path; + return Err(LockfileMismatchError::TooFewLinesInLockfile { file }); + }; + if third_line.contains("version = 4") { + Self::handle_v3v4_conflict( + &shader_cargo_lock_path, + is_force_overwrite_lockfiles_v4_to_v3, + )?; + return Ok(()); + } + if third_line.contains("version = 3") { + return Ok(()); + } + + let file = shader_cargo_lock_path; + let version_line = third_line.to_owned(); + Err(LockfileMismatchError::UnrecognizedLockfileVersion { file, version_line }) + } + + /// Handle conflicting `Cargo.lock` manifest versions by either overwriting the manifest + /// version or exiting with advice on how to handle the conflict. + fn handle_v3v4_conflict( + offending_cargo_lock: &Path, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> Result<(), LockfileMismatchError> { + if !is_force_overwrite_lockfiles_v4_to_v3 { + return Err(LockfileMismatchError::ConflictingVersions); + } + + Self::replace_cargo_lock_manifest_version(offending_cargo_lock, "4", "3") + } + + /// Once all install and builds have completed put their manifest versions + /// back to how they were. + /// + /// # Errors + /// + /// Returns an error if there was a problem reverting any of the lockfiles. + /// See [`LockfileMismatchError`] for details. + #[inline] + pub fn revert_cargo_lock_manifest_versions(&mut self) -> Result<(), LockfileMismatchError> { + for offending_cargo_lock in &self.cargo_lock_files_with_changed_manifest_versions { + log::debug!("Reverting: {}", offending_cargo_lock.display()); + Self::replace_cargo_lock_manifest_version(offending_cargo_lock, "3", "4")?; + } + Ok(()) + } + + /// Replace the manifest version, eg `version = 4`, in a `Cargo.lock` file. + fn replace_cargo_lock_manifest_version( + offending_cargo_lock: &Path, + from_version: &str, + to_version: &str, + ) -> Result<(), LockfileMismatchError> { + log::warn!( + "Replacing manifest version 'version = {from_version}' with 'version = {to_version}' in: {}", + offending_cargo_lock.display() + ); + let old_contents = match fs::read_to_string(offending_cargo_lock) { + Ok(contents) => contents, + Err(source) => { + let file = offending_cargo_lock.to_path_buf(); + return Err(LockfileMismatchError::ReadFile { file, source }); + } + }; + let new_contents = old_contents.replace( + &format!("\nversion = {from_version}\n"), + &format!("\nversion = {to_version}\n"), + ); + + if let Err(source) = fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(offending_cargo_lock) + .and_then(|mut file| file.write_all(new_contents.as_bytes())) + { + let err = LockfileMismatchError::RewriteLockfile { + file: offending_cargo_lock.to_path_buf(), + from_version: from_version.to_owned(), + to_version: to_version.to_owned(), + source, + }; + return Err(err); + } + + Ok(()) + } +} + +impl Drop for LockfileMismatchHandler { + #[inline] + fn drop(&mut self) { + let result = self.revert_cargo_lock_manifest_versions(); + if let Err(error) = result { + log::error!("could not revert some or all of the shader `Cargo.lock` files ({error})"); + } + } +} + +/// An error indicating a problem occurred +/// while handling lockfile manifest version mismatches. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +#[expect(clippy::module_name_repetitions, reason = "it is intended")] +pub enum LockfileMismatchError { + /// Could not query current rustc version. + #[error("could not query rustc version: {0}")] + QueryRustcVersion(#[source] io::Error), + /// Could not read contents of the file. + #[error("could not read file {file}: {source}")] + ReadFile { + /// Path to the file that couldn't be read. + file: PathBuf, + /// Source of the error. + source: io::Error, + }, + /// Could not rewrite the lockfile with new manifest version. + #[error( + "could not rewrite lockfile {file} from version {from_version} to {to_version}: {source}" + )] + RewriteLockfile { + /// Path to the file that couldn't be rewritten. + file: PathBuf, + /// Old manifest version we were changing from. + from_version: String, + /// New manifest version we were changing to. + to_version: String, + /// Source of the error. + source: io::Error, + }, + /// Lockfile has too few lines to determine manifest version. + #[error("lockfile at {file} has too few lines to determine manifest version")] + TooFewLinesInLockfile { + /// Path to the lockfile that contains too few lines. + file: PathBuf, + }, + /// Lockfile manifest version could not be recognized. + #[error("unrecognized lockfile {file} manifest version at \"{version_line}\"")] + UnrecognizedLockfileVersion { + /// Path to the lockfile that contains the unrecognized version line. + file: PathBuf, + /// The unrecognized version line. + version_line: String, + }, + /// Conflicting lockfile manifest versions detected, with advice on how to resolve them + /// by setting the [`force_overwrite_lockfiles_v4_to_v3`] flag. + /// + /// [`force_overwrite_lockfiles_v4_to_v3`]: field@crate::build::CargoGpuBuilderParams::force_overwrite_lockfiles_v4_to_v3 + #[error( + r#"conflicting `Cargo.lock` versions detected ⚠️ + +Because a dedicated Rust toolchain for compiling shaders is being used, +it's possible that the `Cargo.lock` manifest version of the shader crate +does not match the `Cargo.lock` manifest version of the workspace. +This is due to a change in the defaults introduced in Rust 1.83.0. + +One way to resolve this is to force the workspace to use the same version +of Rust as required by the shader. However, that is not often ideal or even +possible. Another way is to exclude the shader from the workspace. This is +also not ideal if you have many shaders sharing config from the workspace. + +Therefore, `cargo gpu build/install` offers a workaround with the argument: + --force-overwrite-lockfiles-v4-to-v3 + +which corresponds to the `force_overwrite_lockfiles_v4_to_v3` flag of `InstallParams`. + +See `cargo gpu build --help` or flag docs for more information."# + )] + ConflictingVersions, +} diff --git a/crates/cargo-gpu/Cargo.toml b/crates/cargo-gpu/Cargo.toml index 8de68b2..a3a14cd 100644 --- a/crates/cargo-gpu/Cargo.toml +++ b/crates/cargo-gpu/Cargo.toml @@ -1,35 +1,32 @@ [package] name = "cargo-gpu" -version = "0.1.0" -edition = "2021" description = "Generates shader .spv files from rust-gpu shader crates" -repository = "https://github.com/Rust-GPU/cargo-gpu" readme = "../../README.md" -keywords = ["gpu", "compiler", "rust-gpu"] -license = "MIT OR Apache-2.0" +version.workspace = true +edition.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true default-run = "cargo-gpu" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -cargo_metadata.workspace = true -anyhow.workspace = true +cargo-gpu-build = { path = "../cargo-gpu-build", features = ["clap", "watch"] } spirv-builder = { workspace = true, features = ["clap", "watch"] } -legacy_target_specs.workspace = true +anyhow.workspace = true +thiserror.workspace = true clap.workspace = true -directories.workspace = true env_logger.workspace = true log.workspace = true relative-path.workspace = true serde.workspace = true serde_json.workspace = true crossterm.workspace = true -semver.workspace = true dunce.workspace = true [dev-dependencies] +tempfile.workspace = true test-log.workspace = true cargo_metadata = { workspace = true, features = ["builder"] } -cargo-util-schemas = "0.8.2" [lints] workspace = true diff --git a/crates/cargo-gpu/src/build.rs b/crates/cargo-gpu/src/build.rs index 2a7f158..d0de943 100644 --- a/crates/cargo-gpu/src/build.rs +++ b/crates/cargo-gpu/src/build.rs @@ -1,17 +1,20 @@ -#![allow(clippy::shadow_reuse, reason = "let's not be silly")] -#![allow(clippy::unwrap_used, reason = "this is basically a test")] //! `cargo gpu build`, analogous to `cargo build` -use crate::install::Install; -use crate::linkage::Linkage; -use crate::lockfile::LockfileMismatchHandler; +use core::convert::Infallible; +use std::{io::Write as _, panic, path::PathBuf}; + use anyhow::Context as _; -use spirv_builder::{CompileResult, ModuleResult, SpirvBuilder}; -use std::io::Write as _; -use std::path::PathBuf; +use cargo_gpu_build::{ + build::{CargoGpuBuilder, CargoGpuBuilderParams}, + spirv_builder::{CompileResult, ModuleResult, SpirvBuilder}, +}; + +use crate::{install::InstallArgs, linkage::Linkage, user_consent::ask_for_user_consent}; -/// Args for just a build -#[derive(clap::Parser, Debug, Clone, serde::Deserialize, serde::Serialize)] +/// Args for just a build. +#[derive(Debug, Clone, clap::Parser, serde::Deserialize, serde::Serialize)] +#[non_exhaustive] +#[expect(clippy::module_name_repetitions, reason = "it is intended")] pub struct BuildArgs { /// Path to the output directory for the compiled shaders. #[clap(long, short, default_value = "./")] @@ -21,12 +24,12 @@ pub struct BuildArgs { #[clap(long, short, action)] pub watch: bool, - /// the flattened [`SpirvBuilder`] + /// The flattened [`SpirvBuilder`]. #[clap(flatten)] #[serde(flatten)] pub spirv_builder: SpirvBuilder, - ///Renames the manifest.json file to the given name + /// Renames the manifest.json file to the given name. #[clap(long, short, default_value = "manifest.json")] pub manifest_file: String, } @@ -43,32 +46,38 @@ impl Default for BuildArgs { } } -/// `cargo build` subcommands -#[derive(Clone, clap::Parser, Debug, serde::Deserialize, serde::Serialize)] +/// `cargo build` subcommands. +#[derive(Clone, Debug, Default, clap::Parser, serde::Deserialize, serde::Serialize)] +#[non_exhaustive] pub struct Build { - /// CLI args for install the `rust-gpu` compiler and components + /// CLI args for install the `rust-gpu` compiler and components. #[clap(flatten)] - pub install: Install, + pub install: InstallArgs, - /// CLI args for configuring the build of the shader + /// CLI args for configuring the build of the shader. #[clap(flatten)] pub build: BuildArgs, } impl Build { - /// Entrypoint + /// Builds the shader crate. + /// + /// # Errors + /// + /// Returns an error if the build process fails somehow. + #[inline] pub fn run(&mut self) -> anyhow::Result<()> { - let installed_backend = self.install.run()?; + self.build.spirv_builder.path_to_crate = Some(self.install.shader_crate.clone()); - let _lockfile_mismatch_handler = LockfileMismatchHandler::new( - &self.install.shader_crate, - &installed_backend.toolchain_channel, - self.install.force_overwrite_lockfiles_v4_to_v3, - )?; + let halt = ask_for_user_consent(self.install.auto_install_rust_toolchain); + let crate_builder_params = CargoGpuBuilderParams::from(self.build.spirv_builder.clone()) + .install(self.install.spirv_installer.clone()) + .force_overwrite_lockfiles_v4_to_v3(self.install.force_overwrite_lockfiles_v4_to_v3) + .halt(halt); + let crate_builder = CargoGpuBuilder::new(crate_builder_params)?; - let builder = &mut self.build.spirv_builder; - builder.path_to_crate = Some(self.install.shader_crate.clone()); - installed_backend.configure_spirv_builder(builder)?; + self.install.spirv_installer = crate_builder.installer.clone(); + self.build.spirv_builder = crate_builder.builder.clone(); // Ensure the shader output dir exists log::debug!( @@ -80,39 +89,48 @@ impl Build { log::debug!("canonicalized output dir: {}", canonicalized.display()); self.build.output_dir = canonicalized; - // Ensure the shader crate exists - self.install.shader_crate = dunce::canonicalize(&self.install.shader_crate)?; - anyhow::ensure!( - self.install.shader_crate.exists(), - "shader crate '{}' does not exist. (Current dir is '{}')", - self.install.shader_crate.display(), - std::env::current_dir()?.display() - ); - if self.build.watch { - let this = self.clone(); - self.build - .spirv_builder - .watch(move |result, accept| { - let result1 = this.parse_compilation_result(&result); - if let Some(accept) = accept { - accept.submit(result1); - } - })? - .context("unreachable")??; - std::thread::park(); - } else { - crate::user_output!( - "Compiling shaders at {}...\n", - self.install.shader_crate.display() - ); - let result = self.build.spirv_builder.build()?; - self.parse_compilation_result(&result)?; + let never = self.watch(crate_builder)?; + match never {} } + self.build(crate_builder) + } + + /// Builds shader crate using [`CargoGpuBuilder`]. + fn build(&self, mut crate_builder: CargoGpuBuilder) -> anyhow::Result<()> { + let result = crate_builder.build()?; + self.parse_compilation_result(&result)?; Ok(()) } - /// Parses compilation result from `SpirvBuilder` and writes it out to a file + /// Watches shader crate for changes using [`CargoGpuBuilder`]. + fn watch(&self, mut crate_builder: CargoGpuBuilder) -> anyhow::Result { + let this = self.clone(); + let mut watcher = crate_builder.watch()?; + let watch_thread = std::thread::spawn(move || -> ! { + loop { + let compile_result = match watcher.recv() { + Ok(compile_result) => compile_result, + Err(err) => { + log::error!("{err}"); + continue; + } + }; + if let Err(err) = this.parse_compilation_result(&compile_result) { + log::error!("{err}"); + } + } + }); + match watch_thread.join() { + Ok(never) => never, + Err(payload) => { + log::error!("watch thread panicked"); + panic::resume_unwind(payload) + } + } + } + + /// Parses compilation result from [`SpirvBuilder`] and writes it out to a file. fn parse_compilation_result(&self, result: &CompileResult) -> anyhow::Result<()> { let shaders = match &result.module { ModuleResult::MultiModule(modules) => { @@ -177,22 +195,25 @@ impl Build { mod test { use clap::Parser as _; - use crate::{Cli, Command}; + use crate::{ + test::{shader_crate_template_path, tests_teardown}, + Cli, Command, + }; #[test_log::test] fn builder_from_params() { - crate::test::tests_teardown(); + tests_teardown(); - let shader_crate_path = crate::test::shader_crate_template_path(); + let shader_crate_path = shader_crate_template_path(); let output_dir = shader_crate_path.join("shaders"); let args = [ - "target/debug/cargo-gpu", - "build", - "--shader-crate", - &format!("{}", shader_crate_path.display()), - "--output-dir", - &format!("{}", output_dir.display()), + "target/debug/cargo-gpu".as_ref(), + "build".as_ref(), + "--shader-crate".as_ref(), + shader_crate_path.as_os_str(), + "--output-dir".as_ref(), + output_dir.as_os_str(), ]; if let Cli { command: Command::Build(build), diff --git a/crates/cargo-gpu/src/config.rs b/crates/cargo-gpu/src/config.rs index f1a67fa..afc00be 100644 --- a/crates/cargo-gpu/src/config.rs +++ b/crates/cargo-gpu/src/config.rs @@ -1,115 +1,59 @@ -//! Manage and merge the various sources of config: shader crate's `Cargo.toml`(s) and CLI args. -use anyhow::Context as _; -use clap::Parser as _; - -/// Config -pub struct Config; - -impl Config { - /// Get all the config defaults as JSON. - pub fn defaults_as_json() -> anyhow::Result { - let defaults_json = Self::cli_args_to_json(vec![String::new()])?; - Ok(defaults_json) - } +//! Manage and merge the various sources of config: +//! shader crate's `Cargo.toml`(s) and provided args. - /// Convert CLI args to their serde JSON representation. - fn cli_args_to_json(env_args: Vec) -> anyhow::Result { - Ok(serde_json::to_value(crate::build::Build::parse_from( - env_args, - ))?) - } +use std::path::Path; - /// Config for the `cargo gpu build` and `cargo gpu install` can be set in the shader crate's - /// `Cargo.toml`, so here we load that config first as the base config, and the CLI arguments can - /// then later override it. - pub fn clap_command_with_cargo_config( - shader_crate_path: &std::path::PathBuf, - mut env_args: Vec, - ) -> anyhow::Result { - let mut config = crate::metadata::Metadata::as_json(shader_crate_path)?; - - env_args.retain(|arg| !(arg == "build" || arg == "install")); - let cli_args_json = Self::cli_args_to_json(env_args)?; - Self::json_merge(&mut config, cli_args_json, None)?; - - let args = serde_json::from_value::(config)?; - Ok(args) - } +use serde::{de::DeserializeOwned, Serialize}; - /// Merge 2 JSON objects. But only if the incoming patch value isn't the default value. - /// Inspired by: - pub fn json_merge( - left_in: &mut serde_json::Value, - right_in: serde_json::Value, - maybe_pointer: Option<&String>, - ) -> anyhow::Result<()> { - let defaults = Self::defaults_as_json()?; - - match (left_in, right_in) { - (left @ &mut serde_json::Value::Object(_), serde_json::Value::Object(right)) => { - let left_as_object = left - .as_object_mut() - .context("Unreachable, we've already proved it's an object")?; - for (key, value) in right { - let new_pointer = maybe_pointer.as_ref().map_or_else( - || format!("/{}", key.clone()), - |pointer| format!("{}/{}", pointer, key.clone()), - ); - Self::json_merge( - left_as_object - .entry(key.clone()) - .or_insert(serde_json::Value::Null), - value, - Some(&new_pointer), - )?; - } - } - (left, right) => { - if let Some(pointer) = maybe_pointer { - let default = defaults.pointer(pointer).context(format!( - "Configuration option with path `{pointer}` was not found in the default configuration, \ - which is:\ndefaults: {defaults:#?}" - ))?; - if &right != default { - // Only overwrite if the new value differs from the defaults. - *left = right; - } - } - } - } +use crate::{merge::merge, metadata::from_cargo_metadata}; - Ok(()) - } +/// Overrides the config options from `Cargo.toml` of the shader crate +/// with options from the provided config. +pub fn from_cargo_metadata_with_config(shader_crate: &Path, config: &C) -> anyhow::Result +where + C: Default + Serialize + DeserializeOwned, +{ + let from_cargo = from_cargo_metadata(shader_crate)?; + let merged = merge(&from_cargo, config)?; + Ok(merged) } #[cfg(test)] mod test { - use super::*; + use std::{io::Write as _, path::PathBuf}; - use std::io::Write as _; + use clap::Parser as _; + use spirv_builder::Capability; + + use crate::{ + build::Build, + test::{overwrite_shader_cargo_toml, shader_crate_test_path}, + }; + + use super::*; #[test_log::test] fn booleans_from_cli() { - let shader_crate_path = crate::test::shader_crate_test_path(); - - let args = Config::clap_command_with_cargo_config( - &shader_crate_path, - vec![ - "gpu".to_owned(), - "build".to_owned(), - "--debug".to_owned(), - "--auto-install-rust-toolchain".to_owned(), - ], - ) - .unwrap(); + let shader_crate_path = shader_crate_test_path(); + let config = Build::parse_from([ + "gpu".as_ref(), + // "build".as_ref(), + "--shader-crate".as_ref(), + shader_crate_path.as_os_str(), + "--debug".as_ref(), + "--auto-install-rust-toolchain".as_ref(), + ]); + + let args = from_cargo_metadata_with_config(&shader_crate_path, &config).unwrap(); assert!(!args.build.spirv_builder.release); assert!(args.install.auto_install_rust_toolchain); } #[test_log::test] fn booleans_from_cargo() { - let shader_crate_path = crate::test::shader_crate_test_path(); - let mut file = crate::test::overwrite_shader_cargo_toml(&shader_crate_path); + let shader_crate_path = shader_crate_test_path(); + + let mut file = overwrite_shader_cargo_toml(&shader_crate_path); file.write_all( [ "[package.metadata.rust-gpu.build]", @@ -122,14 +66,21 @@ mod test { ) .unwrap(); - let args = Config::clap_command_with_cargo_config(&shader_crate_path, vec![]).unwrap(); + let config = Build::parse_from([ + "gpu".as_ref(), + // "build".as_ref(), + "--shader-crate".as_ref(), + shader_crate_path.as_os_str(), + ]); + + let args = from_cargo_metadata_with_config(&shader_crate_path, &config).unwrap(); assert!(!args.build.spirv_builder.release); assert!(args.install.auto_install_rust_toolchain); } - fn update_cargo_output_dir() -> std::path::PathBuf { - let shader_crate_path = crate::test::shader_crate_test_path(); - let mut file = crate::test::overwrite_shader_cargo_toml(&shader_crate_path); + fn update_cargo_output_dir() -> PathBuf { + let shader_crate_path = shader_crate_test_path(); + let mut file = overwrite_shader_cargo_toml(&shader_crate_path); file.write_all( [ "[package.metadata.rust-gpu.build]", @@ -145,35 +96,41 @@ mod test { #[test_log::test] fn string_from_cargo() { let shader_crate_path = update_cargo_output_dir(); - - let args = Config::clap_command_with_cargo_config(&shader_crate_path, vec![]).unwrap(); + let config = Build::parse_from([ + "gpu".as_ref(), + // "build".as_ref(), + "--shader-crate".as_ref(), + shader_crate_path.as_os_str(), + ]); + + let args = from_cargo_metadata_with_config(&shader_crate_path, &config).unwrap(); if cfg!(target_os = "windows") { - assert_eq!(args.build.output_dir, std::path::Path::new("C:/the/moon")); + assert_eq!(args.build.output_dir, Path::new("C:/the/moon")); } else { - assert_eq!(args.build.output_dir, std::path::Path::new("/the/moon")); + assert_eq!(args.build.output_dir, Path::new("/the/moon")); } } #[test_log::test] fn string_from_cargo_overwritten_by_cli() { let shader_crate_path = update_cargo_output_dir(); - - let args = Config::clap_command_with_cargo_config( - &shader_crate_path, - vec![ - "gpu".to_owned(), - "build".to_owned(), - "--output-dir".to_owned(), - "/the/river".to_owned(), - ], - ) - .unwrap(); - assert_eq!(args.build.output_dir, std::path::Path::new("/the/river")); + let config = Build::parse_from([ + "gpu".as_ref(), + // "build".as_ref(), + "--shader-crate".as_ref(), + shader_crate_path.as_os_str(), + "--output-dir".as_ref(), + "/the/river".as_ref(), + ]); + + let args = from_cargo_metadata_with_config(&shader_crate_path, &config).unwrap(); + assert_eq!(args.build.output_dir, Path::new("/the/river")); } #[test_log::test] fn arrays_from_cargo() { let shader_crate_path = crate::test::shader_crate_test_path(); + let mut file = crate::test::overwrite_shader_cargo_toml(&shader_crate_path); file.write_all( [ @@ -185,30 +142,33 @@ mod test { ) .unwrap(); - let args = Config::clap_command_with_cargo_config(&shader_crate_path, vec![]).unwrap(); + let config = Build::parse_from([ + "gpu".as_ref(), + // "build".as_ref(), + "--shader-crate".as_ref(), + shader_crate_path.as_os_str(), + ]); + + let args = from_cargo_metadata_with_config(&shader_crate_path, &config).unwrap(); assert_eq!( args.build.spirv_builder.capabilities, - vec![ - spirv_builder::Capability::AtomicStorage, - spirv_builder::Capability::Matrix - ] + [Capability::AtomicStorage, Capability::Matrix] ); } #[test_log::test] fn rename_manifest_parse() { let shader_crate_path = crate::test::shader_crate_test_path(); - - let args = Config::clap_command_with_cargo_config( - &shader_crate_path, - vec![ - "gpu".to_owned(), - "build".to_owned(), - "--manifest-file".to_owned(), - "mymanifest".to_owned(), - ], - ) - .unwrap(); + let config = Build::parse_from([ + "gpu".as_ref(), + // "build".as_ref(), + "--shader-crate".as_ref(), + shader_crate_path.as_os_str(), + "--manifest-file".as_ref(), + "mymanifest".as_ref(), + ]); + + let args = from_cargo_metadata_with_config(&shader_crate_path, &config).unwrap(); assert_eq!(args.build.manifest_file, "mymanifest".to_owned()); } } diff --git a/crates/cargo-gpu/src/dump_usage.rs b/crates/cargo-gpu/src/dump_usage.rs index d539414..3d4e288 100644 --- a/crates/cargo-gpu/src/dump_usage.rs +++ b/crates/cargo-gpu/src/dump_usage.rs @@ -1,7 +1,9 @@ //! Convenience function for internal use. Dumps all the CLI usage instructions. Useful for //! updating the README. -use crate::{user_output, Cli}; +use cargo_gpu_build::spirv_cache::user_output; + +use crate::Cli; /// main dump usage function pub fn dump_full_usage_for_readme() -> anyhow::Result<()> { @@ -12,7 +14,8 @@ pub fn dump_full_usage_for_readme() -> anyhow::Result<()> { command.build(); write_help(&mut buffer, &mut command, 0)?; - user_output!("{}", String::from_utf8(buffer)?); + let message = String::from_utf8(buffer)?; + user_output!(std::io::stdout(), "{message}")?; Ok(()) } diff --git a/crates/cargo-gpu/src/install.rs b/crates/cargo-gpu/src/install.rs index aa3dfa1..dea4164 100644 --- a/crates/cargo-gpu/src/install.rs +++ b/crates/cargo-gpu/src/install.rs @@ -1,105 +1,28 @@ -//! Install a dedicated per-shader crate that has the `rust-gpu` compiler in it. +//! `cargo gpu install` -use crate::spirv_source::{ - get_channel_from_rustc_codegen_spirv_build_script, query_metadata, FindPackage as _, -}; -use crate::target_specs::update_target_specs_files; -use crate::{cache_dir, spirv_source::SpirvSource}; -use anyhow::Context as _; -use spirv_builder::SpirvBuilder; -use std::path::{Path, PathBuf}; - -/// Represents a functional backend installation, whether it was cached or just installed. -#[derive(Clone, Debug, Default)] -#[non_exhaustive] -pub struct InstalledBackend { - /// path to the `rustc_codegen_spirv` dylib - pub rustc_codegen_spirv_location: PathBuf, - /// toolchain channel name - pub toolchain_channel: String, - /// directory with target-specs json files - pub target_spec_dir: PathBuf, -} +use std::{io, path::PathBuf}; -impl InstalledBackend { - /// Creates a new `SpirvBuilder` configured to use this installed backend. - #[expect( - clippy::unreachable, - reason = "it's unreachable, no need to return a Result" - )] - #[expect(clippy::impl_trait_in_params, reason = "forwarding spirv-builder API")] - #[inline] - pub fn to_spirv_builder( - &self, - path_to_crate: impl AsRef, - target: impl Into, - ) -> SpirvBuilder { - let mut builder = SpirvBuilder::new(path_to_crate, target); - self.configure_spirv_builder(&mut builder) - .unwrap_or_else(|_| unreachable!("we set target before calling this function")); - builder - } +use cargo_gpu_build::spirv_cache::{ + backend::{SpirvCodegenBackendInstallParams, SpirvCodegenBackendInstaller}, + toolchain::StdioCfg, +}; - /// Configures the supplied [`SpirvBuilder`]. `SpirvBuilder.target` must be set and must not change after calling this function. - /// - /// # Errors - /// if `SpirvBuilder.target` is not set - #[inline] - pub fn configure_spirv_builder(&self, builder: &mut SpirvBuilder) -> anyhow::Result<()> { - builder.rustc_codegen_spirv_location = Some(self.rustc_codegen_spirv_location.clone()); - builder.toolchain_overwrite = Some(self.toolchain_channel.clone()); - builder.path_to_target_spec = Some(self.target_spec_dir.join(format!( - "{}.json", - builder.target.as_ref().context("expect target to be set")? - ))); - Ok(()) - } -} +use crate::user_consent::ask_for_user_consent; -/// Args for an install -#[expect( - clippy::struct_excessive_bools, - reason = "cmdline args have many bools" -)] -#[derive(clap::Parser, Debug, Clone, serde::Deserialize, serde::Serialize)] +/// Arguments for just an install. +#[derive(Clone, Debug, clap::Parser, serde::Deserialize, serde::Serialize)] #[non_exhaustive] -pub struct Install { +#[expect(clippy::module_name_repetitions, reason = "it is intended")] +pub struct InstallArgs { /// Directory containing the shader crate to compile. #[clap(long, alias("package"), short_alias('p'), default_value = "./")] #[serde(alias = "package")] pub shader_crate: PathBuf, - #[expect( - clippy::doc_markdown, - reason = "The URL should appear literally like this. But Clippy wants a markdown clickable link" - )] - /// Source of `spirv-builder` dependency - /// Eg: "https://github.com/Rust-GPU/rust-gpu" - #[clap(long)] - pub spirv_builder_source: Option, - - /// Version of `spirv-builder` dependency. - /// * If `--spirv-builder-source` is not set, then this is assumed to be a crates.io semantic - /// version such as "0.9.0". - /// * If `--spirv-builder-source` is set, then this is assumed to be a Git "commitsh", such - /// as a Git commit hash or a Git tag, therefore anything that `git checkout` can resolve. - #[clap(long, verbatim_doc_comment)] - pub spirv_builder_version: Option, - - /// Force `rustc_codegen_spirv` to be rebuilt. - #[clap(long)] - pub rebuild_codegen: bool, - - /// Assume "yes" to "Install Rust toolchain: [y/n]" prompt. - /// - /// Defaults to `false` in cli, `true` in [`Default`] - #[clap(long, action)] - pub auto_install_rust_toolchain: bool, - - /// Clear target dir of `rustc_codegen_spirv` build after a successful build, saves about - /// 200MiB of disk space. - #[clap(long = "no-clear-target", default_value = "true", action = clap::ArgAction::SetFalse)] - pub clear_target: bool, + /// The flattened [`SpirvCodegenBackendInstaller`]. + #[clap(flatten)] + #[serde(flatten)] + pub spirv_installer: SpirvCodegenBackendInstaller, /// There is a tricky situation where a shader crate that depends on workspace config can have /// a different `Cargo.lock` lockfile version from the the workspace's `Cargo.lock`. This can @@ -111,8 +34,8 @@ pub struct Install { /// suitable solution if there are a number of shader crates all sharing similar config and /// you don't want to have to copy/paste and maintain that config across all the shaders. /// - /// So a somewhat hacky workaround is to have `cargo gpu` overwrite lockfile versions. Enabling - /// this flag will only come into effect if there are a mix of v3/v4 lockfiles. It will also + /// So a somewhat hacky workaround is to overwrite lockfile versions. Enabling this flag + /// will only come into effect if there are a mix of v3/v4 lockfiles. It will also /// only overwrite versions for the duration of a build. It will attempt to return the versions /// to their original values once the build is finished. However, of course, unexpected errors /// can occur and the overwritten values can remain. Hence why this behaviour is not enabled by @@ -124,209 +47,48 @@ pub struct Install { /// * #[clap(long, action, verbatim_doc_comment)] pub force_overwrite_lockfiles_v4_to_v3: bool, + + /// Assume "yes" to "Install Rust toolchain: [y/n]" prompt. + #[clap(long, action)] + pub auto_install_rust_toolchain: bool, } -impl Install { - /// Create a default install for a shader crate of some path +impl Default for InstallArgs { #[inline] - #[must_use] - pub const fn from_shader_crate(shader_crate: PathBuf) -> Self { + fn default() -> Self { Self { - shader_crate, - spirv_builder_source: None, - spirv_builder_version: None, - rebuild_codegen: false, - auto_install_rust_toolchain: true, - clear_target: true, + shader_crate: PathBuf::from("./"), + spirv_installer: SpirvCodegenBackendInstaller::default(), force_overwrite_lockfiles_v4_to_v3: false, + auto_install_rust_toolchain: false, } } +} - /// Create the `rustc_codegen_spirv_dummy` crate that depends on `rustc_codegen_spirv` - fn write_source_files(source: &SpirvSource, checkout: &Path) -> anyhow::Result<()> { - // skip writing a dummy project if we use a local rust-gpu checkout - if source.is_path() { - return Ok(()); - } - log::debug!( - "writing `rustc_codegen_spirv_dummy` source files into '{}'", - checkout.display() - ); - - { - log::trace!("writing dummy lib.rs"); - let src = checkout.join("src"); - std::fs::create_dir_all(&src).context("creating 'src' directory")?; - std::fs::File::create(src.join("lib.rs")).context("creating 'src/lib.rs'")?; - }; - - { - log::trace!("writing dummy Cargo.toml"); - let version_spec = match &source { - SpirvSource::CratesIO(version) => { - format!("version = \"{version}\"") - } - SpirvSource::Git { url, rev } => format!("git = \"{url}\"\nrev = \"{rev}\""), - SpirvSource::Path { - rust_gpu_repo_root, - version, - } => { - // this branch is currently unreachable, as we just build `rustc_codegen_spirv` directly, - // since we don't need the `dummy` crate to make cargo download it for us - let mut new_path = rust_gpu_repo_root.to_owned(); - new_path.push("crates/spirv-builder"); - format!("path = \"{new_path}\"\nversion = \"{version}\"") - } - }; - let cargo_toml = format!( - r#" -[package] -name = "rustc_codegen_spirv_dummy" -version = "0.1.0" -edition = "2021" - -[dependencies.spirv-builder] -package = "rustc_codegen_spirv" -{version_spec} - "# - ); - std::fs::write(checkout.join("Cargo.toml"), cargo_toml) - .context("writing 'Cargo.toml'")?; - }; - Ok(()) - } +/// `cargo gpu install` subcommands. +#[derive(Clone, Debug, Default, clap::Parser, serde::Deserialize, serde::Serialize)] +#[non_exhaustive] +pub struct Install { + /// The flattened [`InstallArgs`]. + #[clap(flatten)] + pub install: InstallArgs, +} - /// Install the binary pair and return the [`InstalledBackend`], from which you can create [`SpirvBuilder`] instances. +impl Install { + /// Install the `rust-gpu` codegen backend for the shader crate. /// /// # Errors - /// If the installation somehow fails. + /// + /// Returns an error if the build process fails somehow. #[inline] - #[expect(clippy::too_many_lines, reason = "it's fine")] - pub fn run(&self) -> anyhow::Result { - // Ensure the cache dir exists - let cache_dir = cache_dir()?; - log::info!("cache directory is '{}'", cache_dir.display()); - std::fs::create_dir_all(&cache_dir).with_context(|| { - format!("could not create cache directory '{}'", cache_dir.display()) - })?; - - let source = SpirvSource::new( - &self.shader_crate, - self.spirv_builder_source.as_deref(), - self.spirv_builder_version.as_deref(), - )?; - let install_dir = source.install_dir()?; - - let dylib_filename = format!( - "{}rustc_codegen_spirv{}", - std::env::consts::DLL_PREFIX, - std::env::consts::DLL_SUFFIX - ); - - let dest_dylib_path; - if source.is_path() { - dest_dylib_path = install_dir - .join("target") - .join("release") - .join(&dylib_filename); - } else { - dest_dylib_path = install_dir.join(&dylib_filename); - if dest_dylib_path.is_file() { - log::info!( - "cargo-gpu artifacts are already installed in '{}'", - install_dir.display() - ); - } - } - - // if `source` is a path, always rebuild - let skip_rebuild = !source.is_path() && dest_dylib_path.is_file() && !self.rebuild_codegen; - if skip_rebuild { - log::info!("...and so we are aborting the install step."); - } else { - Self::write_source_files(&source, &install_dir).context("writing source files")?; - } - - // TODO cache toolchain channel in a file? - log::debug!("resolving toolchain version to use"); - let dummy_metadata = query_metadata(&install_dir) - .context("resolving toolchain version: get `rustc_codegen_spirv_dummy` metadata")?; - let rustc_codegen_spirv = dummy_metadata.find_package("rustc_codegen_spirv").context( - "resolving toolchain version: expected a dependency on `rustc_codegen_spirv`", - )?; - let toolchain_channel = - get_channel_from_rustc_codegen_spirv_build_script(rustc_codegen_spirv).context( - "resolving toolchain version: read toolchain from `rustc_codegen_spirv`'s build.rs", - )?; - log::info!("selected toolchain channel `{toolchain_channel:?}`"); - - log::debug!("update_spec_files"); - let target_spec_dir = update_target_specs_files(&source, &dummy_metadata, !skip_rebuild) - .context("writing target spec files")?; - - log::debug!("ensure_toolchain_and_components_exist"); - crate::install_toolchain::ensure_toolchain_and_components_exist( - &toolchain_channel, - self.auto_install_rust_toolchain, - ) - .context("ensuring toolchain and components exist")?; - - if !skip_rebuild { - // to prevent unsupported version errors when using older toolchains - if !source.is_path() { - log::debug!("remove Cargo.lock"); - std::fs::remove_file(install_dir.join("Cargo.lock")) - .context("remove Cargo.lock")?; - } - - crate::user_output!("Compiling `rustc_codegen_spirv` from source {}\n", source); - let mut cargo = spirv_builder::cargo_cmd::CargoCmd::new(); - cargo - .current_dir(&install_dir) - .arg(format!("+{toolchain_channel}")) - .args(["build", "--release"]); - if source.is_path() { - cargo.args(["-p", "rustc_codegen_spirv", "--lib"]); - } - - log::debug!("building artifacts with `{cargo}`"); - cargo - .stdout(std::process::Stdio::inherit()) - .stderr(std::process::Stdio::inherit()) - .output() - .context("getting command output") - .and_then(|output| { - if output.status.success() { - Ok(output) - } else { - Err(anyhow::anyhow!("bad status {:?}", output.status)) - } - }) - .context("running build command")?; - - let target = install_dir.join("target"); - let dylib_path = target.join("release").join(&dylib_filename); - if dylib_path.is_file() { - log::info!("successfully built {}", dylib_path.display()); - if !source.is_path() { - std::fs::rename(&dylib_path, &dest_dylib_path) - .context("renaming dylib path")?; - - if self.clear_target { - log::warn!("clearing target dir {}", target.display()); - std::fs::remove_dir_all(&target).context("clearing target dir")?; - } - } - } else { - log::error!("could not find {}", dylib_path.display()); - anyhow::bail!("`rustc_codegen_spirv` build failed"); - } - } - - Ok(InstalledBackend { - rustc_codegen_spirv_location: dest_dylib_path, - toolchain_channel, - target_spec_dir, - }) + pub fn run(&self) -> anyhow::Result<()> { + let skip_consent = self.install.auto_install_rust_toolchain; + let halt = ask_for_user_consent(skip_consent); + let install_params = SpirvCodegenBackendInstallParams::from(&self.install.shader_crate) + .writer(io::stdout()) + .halt(halt) + .stdio_cfg(StdioCfg::inherit()); + self.install.spirv_installer.install(install_params)?; + Ok(()) } } diff --git a/crates/cargo-gpu/src/install_toolchain.rs b/crates/cargo-gpu/src/install_toolchain.rs deleted file mode 100644 index 98d35d5..0000000 --- a/crates/cargo-gpu/src/install_toolchain.rs +++ /dev/null @@ -1,145 +0,0 @@ -//! toolchain installation logic - -use anyhow::Context as _; -use crossterm::tty::IsTty as _; - -use crate::user_output; - -/// Use `rustup` to install the toolchain and components, if not already installed. -/// -/// Pretty much runs: -/// -/// * rustup toolchain add nightly-2024-04-24 -/// * rustup component add --toolchain nightly-2024-04-24 rust-src rustc-dev llvm-tools -pub fn ensure_toolchain_and_components_exist( - channel: &str, - skip_toolchain_install_consent: bool, -) -> anyhow::Result<()> { - // Check for the required toolchain - let output_toolchain_list = std::process::Command::new("rustup") - .args(["toolchain", "list"]) - .output() - .context("running rustup command")?; - anyhow::ensure!( - output_toolchain_list.status.success(), - "could not list installed toolchains" - ); - let string_toolchain_list = String::from_utf8_lossy(&output_toolchain_list.stdout); - if string_toolchain_list - .split_whitespace() - .any(|toolchain| toolchain.starts_with(channel)) - { - log::debug!("toolchain {channel} is already installed"); - } else { - let message = format!("Rust {channel} with `rustup`"); - get_consent_for_toolchain_install( - format!("Install {message}").as_ref(), - skip_toolchain_install_consent, - )?; - crate::user_output!("Installing {message}\n"); - - let output_toolchain_add = std::process::Command::new("rustup") - .args(["toolchain", "add"]) - .arg(channel) - .stdout(std::process::Stdio::inherit()) - .stderr(std::process::Stdio::inherit()) - .output() - .context("adding toolchain")?; - anyhow::ensure!( - output_toolchain_add.status.success(), - "could not install required toolchain" - ); - } - - // Check for the required components - let output_component_list = std::process::Command::new("rustup") - .args(["component", "list", "--toolchain"]) - .arg(channel) - .output() - .context("getting toolchain list")?; - anyhow::ensure!( - output_component_list.status.success(), - "could not list installed components" - ); - let string_component_list = String::from_utf8_lossy(&output_component_list.stdout); - let required_components = ["rust-src", "rustc-dev", "llvm-tools"]; - let installed_components = string_component_list.lines().collect::>(); - let all_components_installed = required_components.iter().all(|component| { - installed_components.iter().any(|installed_component| { - let is_component = installed_component.starts_with(component); - let is_installed = installed_component.ends_with("(installed)"); - is_component && is_installed - }) - }); - if all_components_installed { - log::debug!("all required components are installed"); - } else { - let message = "toolchain components [rust-src, rustc-dev, llvm-tools] with `rustup`"; - get_consent_for_toolchain_install( - format!("Install {message}").as_ref(), - skip_toolchain_install_consent, - )?; - crate::user_output!("Installing {message}\n"); - - let output_component_add = std::process::Command::new("rustup") - .args(["component", "add", "--toolchain"]) - .arg(channel) - .args(["rust-src", "rustc-dev", "llvm-tools"]) - .stdout(std::process::Stdio::inherit()) - .stderr(std::process::Stdio::inherit()) - .output() - .context("adding rustup component")?; - anyhow::ensure!( - output_component_add.status.success(), - "could not install required components" - ); - } - - Ok(()) -} - -/// Prompt user if they want to install a new Rust toolchain. -fn get_consent_for_toolchain_install( - prompt: &str, - skip_toolchain_install_consent: bool, -) -> anyhow::Result<()> { - if skip_toolchain_install_consent { - return Ok(()); - } - - if !std::io::stdout().is_tty() { - user_output!("No TTY detected so can't ask for consent to install Rust toolchain."); - log::error!("Attempted to ask for consent when there's no TTY"); - #[expect(clippy::exit, reason = "can't ask for user consent if there's no TTY")] - std::process::exit(1); - } - - log::debug!("asking for consent to install the required toolchain"); - crossterm::terminal::enable_raw_mode().context("enabling raw mode")?; - crate::user_output!("{prompt} [y/n]: "); - let mut input = crossterm::event::read().context("reading crossterm event")?; - - if let crossterm::event::Event::Key(crossterm::event::KeyEvent { - code: crossterm::event::KeyCode::Enter, - kind: crossterm::event::KeyEventKind::Release, - .. - }) = input - { - // In Powershell, programs will potentially observe the Enter key release after they started - // (see crossterm#124). If that happens, re-read the input. - input = crossterm::event::read().context("re-reading crossterm event")?; - } - crossterm::terminal::disable_raw_mode().context("disabling raw mode")?; - - if let crossterm::event::Event::Key(crossterm::event::KeyEvent { - code: crossterm::event::KeyCode::Char('y'), - .. - }) = input - { - Ok(()) - } else { - crate::user_output!("Exiting...\n"); - #[expect(clippy::exit, reason = "user requested abort")] - std::process::exit(0); - } -} diff --git a/crates/cargo-gpu/src/lib.rs b/crates/cargo-gpu/src/lib.rs index 1be7abb..9a544ce 100644 --- a/crates/cargo-gpu/src/lib.rs +++ b/crates/cargo-gpu/src/lib.rs @@ -1,102 +1,45 @@ -#![expect(clippy::pub_use, reason = "pub use for build scripts")] - -//! Rust GPU shader crate builder. +//! Command line tool for building Rust shaders using `rust-gpu`. //! -//! This program and library allows you to easily compile your rust-gpu shaders, +//! This program allows you to easily compile your rust-gpu shaders, //! without requiring you to fix your entire project to a specific toolchain. //! -//! # How it works +//! ## Building shader crates //! -//! This program primarily manages installations of `rustc_codegen_spirv`, the -//! codegen backend of rust-gpu to generate SPIR-V shader binaries. The codegen -//! backend builds on internal, ever-changing interfaces of rustc, which requires -//! fixing a version of rust-gpu to a specific version of the rustc compiler. -//! Usually, this would require you to fix your entire project to that specific -//! toolchain, but this project loosens that requirement by managing installations -//! of `rustc_codegen_spirv` and their associated toolchains for you. +//! It takes a path to a shader crate to build, as well as a path to a directory to put +//! the compiled `spv` source files. It also takes a path to an output manifest file +//! where all shader entry points will be mapped to their `spv` source files. +//! This manifest file can be used by build scripts (`build.rs` files) to generate linkage +//! or conduct other post-processing, like converting the `spv` files into `wgsl` files, for example. //! -//! We continue to use rust-gpu's `spirv_builder` crate to pass the many additional -//! parameters required to configure rustc and our codegen backend, but provide you -//! with a toolchain agnostic version that you may use from stable rustc. And a -//! `cargo gpu` cmdline utility to simplify shader building even more. +//! For additional information, see the [`cargo-gpu-build`](cargo_gpu_build) crate documentation. //! //! ## Where the binaries are //! -//! We store our prebuild `rustc_spirv_builder` binaries in the default cache -//! directory of your OS: -//! * Windows: `C:/users//AppData/Local/rust-gpu` -//! * Mac: `~/Library/Caches/rust-gpu` -//! * Linux: `~/.cache/rust-gpu` -//! -//! ## How we build the backend -//! -//! * retrieve the version of rust-gpu you want to use based on the version of the -//! `spirv-std` dependency in your shader crate. -//! * create a dummy project at `/codegen//` that depends on -//! `rustc_codegen_spirv` -//! * use `cargo metadata` to `cargo update` the dummy project, which downloads the -//! `rustc_codegen_spirv` crate into cargo's cache, and retrieve the path to the -//! download location. -//! * search for the required toolchain in `build.rs` of `rustc_codegen_spirv` -//! * build it with the required toolchain version -//! * copy out the binary and clean the target dir -//! -//! ## Building shader crates -//! -//! `cargo-gpu` takes a path to a shader crate to build, as well as a path to a directory -//! to put the compiled `spv` source files. It also takes a path to an output manifest -//! file where all shader entry points will be mapped to their `spv` source files. This -//! manifest file can be used by build scripts (`build.rs` files) to generate linkage or -//! conduct other post-processing, like converting the `spv` files into `wgsl` files, -//! for example. +//! Prebuilt binaries are stored in the [cache directory](spirv_cache::cache::cache_dir()), +//! which path differs by OS you are using. + +#![expect(clippy::pub_use, reason = "part of public API")] + +pub use cargo_gpu_build::{spirv_builder, spirv_cache}; -use anyhow::Context as _; +use self::{ + build::Build, config::from_cargo_metadata_with_config, dump_usage::dump_full_usage_for_readme, + install::Install, show::Show, +}; -use crate::dump_usage::dump_full_usage_for_readme; -use build::Build; -use show::Show; +pub mod build; +pub mod install; +pub mod show; -mod build; mod config; mod dump_usage; -mod install; -mod install_toolchain; mod linkage; -mod lockfile; +mod merge; mod metadata; -mod show; -mod spirv_source; -mod target_specs; mod test; +mod user_consent; -pub use install::*; -pub use spirv_builder; - -/// Central function to write to the user. -#[macro_export] -macro_rules! user_output { - ($($args: tt)*) => { - #[allow( - clippy::allow_attributes, - clippy::useless_attribute, - unused_imports, - reason = "`std::io::Write` is only sometimes called??" - )] - use std::io::Write as _; - - #[expect( - clippy::non_ascii_literal, - reason = "CRAB GOOD. CRAB IMPORTANT." - )] - { - print!("🦀 "); - } - print!($($args)*); - std::io::stdout().flush().unwrap(); - } -} - -/// All of the available subcommands for `cargo gpu` +/// All of the available subcommands for `cargo gpu`. #[derive(clap::Subcommand)] #[non_exhaustive] pub enum Command { @@ -112,40 +55,39 @@ pub enum Command { /// A hidden command that can be used to recursively print out all the subcommand help messages: /// `cargo gpu dump-usage` /// Useful for updating the README. + #[doc(hidden)] #[clap(hide(true))] DumpUsage, } impl Command { - /// Runs the command + /// Runs the command. /// /// # Errors - /// Any errors during execution, usually printed to the user + /// + /// Any errors during execution, usually printed to the user. #[inline] - pub fn run(&self, env_args: Vec) -> anyhow::Result<()> { + pub fn run(&self) -> anyhow::Result<()> { match &self { Self::Install(install) => { - let shader_crate_path = &install.shader_crate; - let command = - config::Config::clap_command_with_cargo_config(shader_crate_path, env_args)?; - log::debug!( - "installing with final merged arguments: {:#?}", - command.install - ); - command.install.run()?; + let shader_crate = &install.install.shader_crate; + let command = from_cargo_metadata_with_config(shader_crate, install.as_ref())?; + log::debug!("installing with final merged arguments: {command:#?}"); + + command.run()?; } Self::Build(build) => { - let shader_crate_path = &build.install.shader_crate; - let mut command = - config::Config::clap_command_with_cargo_config(shader_crate_path, env_args)?; + let shader_crate = &build.install.shader_crate; + let mut command = from_cargo_metadata_with_config(shader_crate, build.as_ref())?; log::debug!("building with final merged arguments: {command:#?}"); + // When watching, do one normal run to setup the `manifest.json` file. if command.build.watch { - // When watching, do one normal run to setup the `manifest.json` file. command.build.watch = false; command.run()?; command.build.watch = true; } + command.run()?; } Self::Show(show) => show.run()?, @@ -156,7 +98,7 @@ impl Command { } } -/// the Cli struct representing the main cli +/// The struct representing the main CLI. #[derive(clap::Parser)] #[clap(author, version, about, subcommand_required = true)] #[non_exhaustive] @@ -165,36 +107,3 @@ pub struct Cli { #[clap(subcommand)] pub command: Command, } - -/// The central cache directory of cargo gpu -/// -/// # Errors -/// may fail if we can't find the user home directory -#[inline] -pub fn cache_dir() -> anyhow::Result { - let dir = directories::BaseDirs::new() - .with_context(|| "could not find the user home directory")? - .cache_dir() - .join("rust-gpu"); - - Ok(if cfg!(test) { - let thread_id = std::thread::current().id(); - let id = format!("{thread_id:?}").replace('(', "-").replace(')', ""); - dir.join("tests").join(id) - } else { - dir - }) -} - -/// Returns a string suitable to use as a directory. -/// -/// Created from the spirv-builder source dep and the rustc channel. -fn to_dirname(text: &str) -> String { - text.replace( - [std::path::MAIN_SEPARATOR, '\\', '/', '.', ':', '@', '='], - "_", - ) - .split(['{', '}', ' ', '\n', '"', '\'']) - .collect::>() - .concat() -} diff --git a/crates/cargo-gpu/src/lockfile.rs b/crates/cargo-gpu/src/lockfile.rs deleted file mode 100644 index bce1487..0000000 --- a/crates/cargo-gpu/src/lockfile.rs +++ /dev/null @@ -1,274 +0,0 @@ -//! Handles lockfile version conflicts and downgrades. Stable uses lockfile v4, but rust-gpu -//! v0.9.0 uses an old toolchain requiring v3 and will refuse to build with a v4 lockfile being -//! present. This module takes care of warning the user and potentially downgrading the lockfile. - -use anyhow::Context as _; -use semver::Version; -use spirv_builder::query_rustc_version; -use std::io::Write as _; - -/// `Cargo.lock` manifest version 4 became the default in Rust 1.83.0. Conflicting manifest -/// versions between the workspace and the shader crate, can cause problems. -const RUST_VERSION_THAT_USES_V4_CARGO_LOCKS: Version = Version::new(1, 83, 0); - -/// Cargo dependency for `spirv-builder` and the rust toolchain channel. -#[derive(Debug, Clone)] -pub struct LockfileMismatchHandler { - /// `Cargo.lock`s that have had their manifest versions changed by us and need changing back. - pub cargo_lock_files_with_changed_manifest_versions: Vec, -} - -impl LockfileMismatchHandler { - /// Create instance - pub fn new( - shader_crate_path: &std::path::Path, - toolchain_channel: &str, - is_force_overwrite_lockfiles_v4_to_v3: bool, - ) -> anyhow::Result { - let mut cargo_lock_files_with_changed_manifest_versions = vec![]; - - let maybe_shader_crate_lock = - Self::ensure_workspace_rust_version_doesnt_conflict_with_shader( - shader_crate_path, - is_force_overwrite_lockfiles_v4_to_v3, - ) - .context("ensure_workspace_rust_version_doesnt_conflict_with_shader")?; - - if let Some(shader_crate_lock) = maybe_shader_crate_lock { - cargo_lock_files_with_changed_manifest_versions.push(shader_crate_lock); - } - - let maybe_workspace_crate_lock = - Self::ensure_shader_rust_version_doesnt_conflict_with_any_cargo_locks( - shader_crate_path, - toolchain_channel, - is_force_overwrite_lockfiles_v4_to_v3, - ) - .context("ensure_shader_rust_version_doesnt_conflict_with_any_cargo_locks")?; - - if let Some(workspace_crate_lock) = maybe_workspace_crate_lock { - cargo_lock_files_with_changed_manifest_versions.push(workspace_crate_lock); - } - - Ok(Self { - cargo_lock_files_with_changed_manifest_versions, - }) - } - - /// See docs for `force_overwrite_lockfiles_v4_to_v3` flag for why we do this. - fn ensure_workspace_rust_version_doesnt_conflict_with_shader( - shader_crate_path: &std::path::Path, - is_force_overwrite_lockfiles_v4_to_v3: bool, - ) -> anyhow::Result> { - log::debug!("Ensuring no v3/v4 `Cargo.lock` conflicts from workspace Rust..."); - let workspace_rust_version = query_rustc_version(None).context("reading rustc version")?; - if workspace_rust_version >= RUST_VERSION_THAT_USES_V4_CARGO_LOCKS { - log::debug!( - "user's Rust is v{workspace_rust_version}, so no v3/v4 conflicts possible." - ); - return Ok(None); - } - - Self::handle_conflicting_cargo_lock_v4( - shader_crate_path, - is_force_overwrite_lockfiles_v4_to_v3, - ) - .context("handling v4/v3 conflict")?; - - if is_force_overwrite_lockfiles_v4_to_v3 { - Ok(Some(shader_crate_path.join("Cargo.lock"))) - } else { - Ok(None) - } - } - - /// See docs for `force_overwrite_lockfiles_v4_to_v3` flag for why we do this. - fn ensure_shader_rust_version_doesnt_conflict_with_any_cargo_locks( - shader_crate_path: &std::path::Path, - channel: &str, - is_force_overwrite_lockfiles_v4_to_v3: bool, - ) -> anyhow::Result> { - log::debug!("Ensuring no v3/v4 `Cargo.lock` conflicts from shader's Rust..."); - let shader_rust_version = - query_rustc_version(Some(channel)).context("getting rustc version")?; - if shader_rust_version >= RUST_VERSION_THAT_USES_V4_CARGO_LOCKS { - log::debug!("shader's Rust is v{shader_rust_version}, so no v3/v4 conflicts possible."); - return Ok(None); - } - - log::debug!( - "shader's Rust is v{shader_rust_version}, so checking both shader and workspace `Cargo.lock` manifest versions..." - ); - - if shader_crate_path.join("Cargo.lock").exists() { - // Note that we don't return the `Cargo.lock` here (so that it's marked for reversion - // after the build), because we can be sure that updating it now is actually updating it - // to the state it should have been all along. Therefore it doesn't need reverting once - // fixed. - Self::handle_conflicting_cargo_lock_v4( - shader_crate_path, - is_force_overwrite_lockfiles_v4_to_v3, - ) - .context("handling v4/v3 conflict")?; - } - - if let Some(workspace_root) = - Self::get_workspace_root(shader_crate_path).context("reading workspace root")? - { - Self::handle_conflicting_cargo_lock_v4( - workspace_root, - is_force_overwrite_lockfiles_v4_to_v3, - ) - .context("handling conflicting cargo v4")?; - return Ok(Some(workspace_root.join("Cargo.lock"))); - } - - Ok(None) - } - - /// Get the path to the shader crate's workspace, if it has one. We can't use the traditional - /// `cargo metadata` because if the workspace has a conflicting `Cargo.lock` manifest version - /// then that command won't work. Instead we do an old school recursive file tree walk. - fn get_workspace_root( - shader_crate_path: &std::path::Path, - ) -> anyhow::Result> { - let shader_cargo_toml = std::fs::read_to_string(shader_crate_path.join("Cargo.toml")) - .with_context(|| format!("reading Cargo.toml at {}", shader_crate_path.display()))?; - if !shader_cargo_toml.contains("workspace = true") { - return Ok(None); - } - - let mut current_path = shader_crate_path; - #[expect(clippy::default_numeric_fallback, reason = "It's just a loop")] - for _ in 0..15 { - if let Some(parent_path) = current_path.parent() { - if parent_path.join("Cargo.lock").exists() { - return Ok(Some(parent_path)); - } - current_path = parent_path; - } else { - break; - } - } - - Ok(None) - } - - /// When Rust < 1.83.0 is being used an error will occur if it tries to parse `Cargo.lock` - /// files that use lockfile manifest version 4. Here we check and handle that. - fn handle_conflicting_cargo_lock_v4( - folder: &std::path::Path, - is_force_overwrite_lockfiles_v4_to_v3: bool, - ) -> anyhow::Result<()> { - let shader_cargo_lock_path = folder.join("Cargo.lock"); - let shader_cargo_lock = std::fs::read_to_string(shader_cargo_lock_path.clone()) - .context("reading shader cargo lock")?; - let third_line = shader_cargo_lock.lines().nth(2).context("no third line")?; - if third_line.contains("version = 4") { - Self::handle_v3v4_conflict( - &shader_cargo_lock_path, - is_force_overwrite_lockfiles_v4_to_v3, - ) - .context("handling v4/v3 conflict")?; - return Ok(()); - } - if third_line.contains("version = 3") { - return Ok(()); - } - anyhow::bail!( - "Unrecognized `Cargo.lock` manifest version at: {}", - folder.display() - ) - } - - /// Handle conflicting `Cargo.lock` manifest versions by either overwriting the manifest - /// version or exiting with advice on how to handle the conflict. - fn handle_v3v4_conflict( - offending_cargo_lock: &std::path::Path, - is_force_overwrite_lockfiles_v4_to_v3: bool, - ) -> anyhow::Result<()> { - if !is_force_overwrite_lockfiles_v4_to_v3 { - Self::exit_with_v3v4_hack_suggestion(); - } - - Self::replace_cargo_lock_manifest_version(offending_cargo_lock, "4", "3") - .context("replacing version 4 -> 3")?; - - Ok(()) - } - - /// Once all install and builds have completed put their manifest versions back to how they - /// were. - pub fn revert_cargo_lock_manifest_versions(&self) -> anyhow::Result<()> { - for offending_cargo_lock in &self.cargo_lock_files_with_changed_manifest_versions { - log::debug!("Reverting: {}", offending_cargo_lock.display()); - Self::replace_cargo_lock_manifest_version(offending_cargo_lock, "3", "4") - .context("replacing version 3 -> 4")?; - } - - Ok(()) - } - - /// Replace the manifest version, eg `version = 4`, in a `Cargo.lock` file. - fn replace_cargo_lock_manifest_version( - offending_cargo_lock: &std::path::Path, - from_version: &str, - to_version: &str, - ) -> anyhow::Result<()> { - log::warn!( - "Replacing manifest version 'version = {}' with 'version = {}' in: {}", - from_version, - to_version, - offending_cargo_lock.display() - ); - let old_contents = std::fs::read_to_string(offending_cargo_lock) - .context("reading offending Cargo.lock")?; - let new_contents = old_contents.replace( - &format!("\nversion = {from_version}\n"), - &format!("\nversion = {to_version}\n"), - ); - - let mut file = std::fs::OpenOptions::new() - .write(true) - .truncate(true) - .open(offending_cargo_lock) - .context("opening offending Cargo.lock")?; - file.write_all(new_contents.as_bytes())?; - - Ok(()) - } - - /// Exit and give the user advice on how to deal with the infamous v3/v4 Cargo lockfile version - /// problem. - #[expect(clippy::non_ascii_literal, reason = "It's CLI output")] - fn exit_with_v3v4_hack_suggestion() { - crate::user_output!( - "Conflicting `Cargo.lock` versions detected ⚠️\n\ - Because `cargo gpu` uses a dedicated Rust toolchain for compiling shaders\n\ - it's possible that the `Cargo.lock` manifest version of the shader crate\n\ - does not match the `Cargo.lock` manifest version of the workspace. This is\n\ - due to a change in the defaults introduced in Rust 1.83.0.\n\ - \n\ - One way to resolve this is to force the workspace to use the same version\n\ - of Rust as required by the shader. However that is not often ideal or even\n\ - possible. Another way is to exlude the shader from the workspace. This is\n\ - also not ideal if you have many shaders sharing config from the workspace.\n\ - \n\ - Therefore `cargo gpu build/install` offers a workaround with the argument:\n\ - --force-overwrite-lockfiles-v4-to-v3\n\ - \n\ - See `cargo gpu build --help` for more information.\n\ - " - ); - std::process::exit(1); - } -} - -impl Drop for LockfileMismatchHandler { - fn drop(&mut self) { - let result = self.revert_cargo_lock_manifest_versions(); - if let Err(error) = result { - log::error!("Couldn't revert some or all of the shader `Cargo.lock` files: {error}"); - } - } -} diff --git a/crates/cargo-gpu/src/main.rs b/crates/cargo-gpu/src/main.rs index f40e3c6..da122cd 100644 --- a/crates/cargo-gpu/src/main.rs +++ b/crates/cargo-gpu/src/main.rs @@ -1,4 +1,5 @@ -//! main executable of cargo gpu +//! The main executable of `cargo-gpu`. + use cargo_gpu::Cli; use clap::Parser as _; @@ -36,6 +37,7 @@ fn run() -> anyhow::Result<()> { }) .collect::>(); log::trace!("CLI args: {env_args:#?}"); - let cli = Cli::parse_from(&env_args); - cli.command.run(env_args) + + let cli = Cli::parse_from(env_args); + cli.command.run() } diff --git a/crates/cargo-gpu/src/merge.rs b/crates/cargo-gpu/src/merge.rs new file mode 100644 index 0000000..83761f0 --- /dev/null +++ b/crates/cargo-gpu/src/merge.rs @@ -0,0 +1,62 @@ +//! Utilities for struct merging. + +use serde::{de::DeserializeOwned, Serialize}; +use serde_json::{from_value, to_value, Result, Value}; + +/// Merges two objects (converting them to JSON in the process), +/// but only if the incoming patch value isn't the default one. +#[inline] +pub fn merge(value: &T, patch: &T) -> Result +where + T: Default + Serialize + DeserializeOwned, +{ + let json_value = to_value(value)?; + let json_patch = to_value(patch)?; + let json_default = to_value(T::default())?; + let json_merged = json_merge(json_value, json_patch, &json_default); + let merged = from_value(json_merged)?; + Ok(merged) +} + +/// Recursively merges two JSON objects, +/// but only if the incoming patch value isn't the default one. +#[inline] +#[must_use] +pub fn json_merge(mut value: Value, patch: Value, default: &Value) -> Value { + json_merge_inner(&mut value, patch, default, None); + value +} + +/// Recursively merges two JSON objects in place, +/// but only if the incoming patch value isn't the default one. +#[inline] +pub fn json_merge_in(value: &mut Value, patch: Value, default: &Value) { + json_merge_inner(value, patch, default, None); +} + +/// Inspired by: +fn json_merge_inner( + old_in: &mut Value, + new_in: Value, + defaults: &Value, + maybe_pointer: Option<&str>, +) { + match (old_in, new_in) { + (Value::Object(old), Value::Object(new)) => { + for (key, new_value) in new { + let pointer = maybe_pointer.unwrap_or(""); + let new_pointer = format!("{pointer}/{key}"); + let old_value = old.entry(key).or_insert(Value::Null); + json_merge_inner(old_value, new_value, defaults, Some(&new_pointer)); + } + } + (old, new) => { + let Some(default) = maybe_pointer.and_then(|pointer| defaults.pointer(pointer)) else { + return; + }; + if new != *default { + *old = new; + } + } + } +} diff --git a/crates/cargo-gpu/src/metadata.rs b/crates/cargo-gpu/src/metadata.rs index 0b330fb..81dca93 100644 --- a/crates/cargo-gpu/src/metadata.rs +++ b/crates/cargo-gpu/src/metadata.rs @@ -1,118 +1,124 @@ -//! Get config from the shader crate's `Cargo.toml` `[*.metadata.rust-gpu.*]` +//! Get config from the shader crate's `Cargo.toml` `[*.metadata.rust-gpu.*]`. +//! +//! `cargo` formally ignores this metadata, +//! so that packages can implement their own behaviour with it. -use cargo_metadata::MetadataCommand; -use serde_json::Value; +use std::{fs, path::Path}; -/// `Metadata` refers to the `[metadata.*]` section of `Cargo.toml` that `cargo` formally -/// ignores so that packages can implement their own behaviour with it. -#[derive(Debug)] -pub struct Metadata; +use cargo_gpu_build::spirv_cache::cargo_metadata::{self, MetadataCommand}; +use serde::{de::DeserializeOwned, Serialize}; +use serde_json::{from_value, json, to_value, Value}; -impl Metadata { - /// Convert `rust-gpu`-specific sections in `Cargo.toml` to `clap`-compatible arguments. - /// The section in question is: `[package.metadata.rust-gpu.*]`. See the `shader-crate-template` - /// for an example. - /// - /// First we generate the CLI arg defaults as JSON. Then on top of those we merge any config - /// from the workspace `Cargo.toml`, then on top of those we merge any config from the shader - /// crate's `Cargo.toml`. - pub fn as_json(path: &std::path::PathBuf) -> anyhow::Result { - let cargo_json = Self::get_cargo_toml_as_json(path)?; - let config = Self::merge_configs(&cargo_json, path)?; - Ok(config) - } +use crate::merge::json_merge_in; - /// Merge the various source of config: defaults, workspace and shader crate. - fn merge_configs( - cargo_json: &cargo_metadata::Metadata, - path: &std::path::Path, - ) -> anyhow::Result { - let mut metadata = crate::config::Config::defaults_as_json()?; - crate::config::Config::json_merge( - &mut metadata, - { - log::debug!("looking for workspace metadata"); - let ws_meta = Self::get_rust_gpu_from_metadata(&cargo_json.workspace_metadata); - log::trace!("workspace_metadata: {ws_meta:#?}"); - ws_meta - }, - None, - )?; - crate::config::Config::json_merge( - &mut metadata, - { - log::debug!("looking for crate metadata"); - let mut crate_meta = Self::get_crate_metadata(cargo_json, path)?; - log::trace!("crate_metadata: {crate_meta:#?}"); - if let Some(output_path) = crate_meta.pointer_mut("/build/output_dir") { - log::debug!("found output-dir path in crate metadata: {output_path:?}"); - if let Some(output_dir) = output_path.clone().as_str() { - let new_output_path = path.join(output_dir); - *output_path = Value::String(format!("{}", new_output_path.display())); - log::debug!( - "setting that to be relative to the Cargo.toml it was found in: {}", - new_output_path.display() - ); - } - } - crate_meta - }, - None, - )?; +/// Converts `rust-gpu`-specific sections from `Cargo.toml` +/// to the value of specified type. +/// +/// The section in question is: `[*.metadata.rust-gpu.*]`. +/// See the `shader-crate-template` for an example. +pub fn from_cargo_metadata(path: &Path) -> anyhow::Result +where + T: Default + Serialize + DeserializeOwned, +{ + let cargo_meta = cargo_metadata(path)?; + let config = merge_configs(&cargo_meta, path)?; + Ok(config) +} - Ok(metadata) - } +/// Retrieves cargo metadata from a `Cargo.toml` by provided path. +fn cargo_metadata(path: &Path) -> anyhow::Result { + let metadata = MetadataCommand::new().current_dir(path).exec()?; + Ok(metadata) +} - /// Convert a `Cargo.toml` to JSON - fn get_cargo_toml_as_json( - path: &std::path::PathBuf, - ) -> anyhow::Result { - Ok(MetadataCommand::new().current_dir(path).exec()?) - } +/// Merges the various sources of config: defaults, workspace and shader crate. +fn merge_configs(cargo_meta: &cargo_metadata::Metadata, path: &Path) -> anyhow::Result +where + T: Default + Serialize + DeserializeOwned, +{ + let default = to_value(T::default())?; + let mut json_metadata = default.clone(); - /// Get any `rust-gpu` metadata set in the crate's `Cargo.toml` - fn get_crate_metadata( - json: &cargo_metadata::Metadata, - path: &std::path::Path, - ) -> anyhow::Result { - let shader_crate_path = std::fs::canonicalize(path)?.join("Cargo.toml"); - - for package in &json.packages { - let manifest_path = std::fs::canonicalize(package.manifest_path.as_std_path())?; - log::debug!( - "Matching shader crate path with manifest path: '{}' == '{}'?", - shader_crate_path.display(), - manifest_path.display() - ); - if manifest_path == shader_crate_path { - log::debug!("...matches! Getting metadata"); - return Ok(Self::get_rust_gpu_from_metadata(&package.metadata)); + json_merge_in( + &mut json_metadata, + { + log::debug!("looking for workspace metadata..."); + let ws_metadata = rust_gpu_metadata(&cargo_meta.workspace_metadata); + log::trace!("found workspace metadata: {ws_metadata:#?}"); + ws_metadata + }, + &default, + ); + + json_merge_in( + &mut json_metadata, + { + log::debug!("looking for crate metadata..."); + let mut crate_meta = crate_metadata(cargo_meta, path)?; + log::trace!("found crate metadata: {crate_meta:#?}"); + if let Some(output_path) = crate_meta.pointer_mut("/build/output_dir") { + log::debug!("found output dir path in crate metadata: {output_path:?}"); + if let Value::String(output_dir) = output_path { + let new_output_dir = path.join(&output_dir).display().to_string(); + log::debug!("setting that to be relative to the Cargo.toml it was found in: {new_output_dir}"); + *output_dir = new_output_dir; + } } - } - Ok(serde_json::json!({})) - } + crate_meta + }, + &default, + ); - /// Get `rust-gpu` value from some metadata - fn get_rust_gpu_from_metadata(metadata: &Value) -> Value { - Self::keys_to_snake_case( - metadata - .pointer("/rust-gpu") - .cloned() - .unwrap_or(Value::Null), - ) + let metadata = from_value(json_metadata)?; + Ok(metadata) +} + +/// Retrieves `rust-gpu` value from some metadata in JSON format. +fn rust_gpu_metadata(metadata: &Value) -> Value { + metadata + .pointer("/rust-gpu") + .cloned() + .unwrap_or(Value::Null) + .keys_to_snake_case() +} + +/// Retrieves any `rust-gpu` metadata set in the crate's `Cargo.toml`. +fn crate_metadata(cargo_meta: &cargo_metadata::Metadata, path: &Path) -> anyhow::Result { + let shader_crate_path = fs::canonicalize(path)?.join("Cargo.toml"); + + for package in &cargo_meta.packages { + let manifest_path = fs::canonicalize(package.manifest_path.as_std_path())?; + log::debug!( + "Matching shader crate path with manifest path: '{}' == '{}'?", + shader_crate_path.display(), + manifest_path.display() + ); + if manifest_path == shader_crate_path { + log::debug!("...matches! Getting metadata"); + return Ok(rust_gpu_metadata(&package.metadata)); + } } + Ok(json!({})) +} - /// Convert JSON keys from kebab case to snake case. Eg: `a-b` to `a_b`. +/// Extension trait for [JSON value](Value). +trait JsonKeysToSnakeCase { + /// Converts JSON keys from kebab case to snake case, e.g. from `a-b` to `a_b`. /// - /// Detection of keys for serde deserialization must match the case in the Rust structs. - /// However clap defaults to detecting CLI args in kebab case. So here we do the conversion. + /// Detection of keys for [`serde`] deserialization must match the case in the Rust structs. + /// However, [`clap`] defaults to detecting CLI args in kebab case. So here we do the conversion. + fn keys_to_snake_case(self) -> Value; +} + +impl JsonKeysToSnakeCase for Value { + #[inline] #[expect(clippy::wildcard_enum_match_arm, reason = "we only want objects")] - fn keys_to_snake_case(json: Value) -> Value { - match json { - Value::Object(object) => Value::Object( + fn keys_to_snake_case(self) -> Value { + match self { + Self::Object(object) => Self::Object( object .into_iter() - .map(|(key, value)| (key.replace('-', "_"), Self::keys_to_snake_case(value))) + .map(|(key, value)| (key.replace('-', "_"), value.keys_to_snake_case())) .collect(), ), other => other, @@ -126,31 +132,29 @@ impl Metadata { )] #[cfg(test)] mod test { - use super::*; use std::path::Path; + use crate::build::Build; + + use super::*; + + const MANIFEST: &str = env!("CARGO_MANIFEST_DIR"); + const PACKAGE: &str = env!("CARGO_PKG_NAME"); + #[test_log::test] fn generates_defaults() { - let mut metadata = MetadataCommand::new() - .current_dir(env!("CARGO_MANIFEST_DIR")) - .exec() - .unwrap(); - metadata.packages.first_mut().unwrap().metadata = serde_json::json!({}); - let configs = Metadata::merge_configs(&metadata, Path::new("./")).unwrap(); - assert_eq!(configs["build"]["release"], Value::Bool(true)); - assert_eq!( - configs["install"]["auto_install_rust_toolchain"], - Value::Bool(false) - ); + let mut metadata = MetadataCommand::new().current_dir(MANIFEST).exec().unwrap(); + metadata.packages.first_mut().unwrap().metadata = json!({}); + + let configs = merge_configs::(&metadata, Path::new("./")).unwrap(); + assert!(configs.build.spirv_builder.release); + assert!(!configs.install.auto_install_rust_toolchain); } #[test_log::test] fn can_override_config_from_workspace_toml() { - let mut metadata = MetadataCommand::new() - .current_dir(env!("CARGO_MANIFEST_DIR")) - .exec() - .unwrap(); - metadata.workspace_metadata = serde_json::json!({ + let mut metadata = MetadataCommand::new().current_dir(MANIFEST).exec().unwrap(); + metadata.workspace_metadata = json!({ "rust-gpu": { "build": { "release": false @@ -160,26 +164,21 @@ mod test { } } }); - let configs = Metadata::merge_configs(&metadata, Path::new("./")).unwrap(); - assert_eq!(configs["build"]["release"], Value::Bool(false)); - assert_eq!( - configs["install"]["auto_install_rust_toolchain"], - Value::Bool(true) - ); + + let configs = merge_configs::(&metadata, Path::new("./")).unwrap(); + assert!(!configs.build.spirv_builder.release); + assert!(configs.install.auto_install_rust_toolchain); } #[test_log::test] fn can_override_config_from_crate_toml() { - let mut metadata = MetadataCommand::new() - .current_dir(env!("CARGO_MANIFEST_DIR")) - .exec() - .unwrap(); + let mut metadata = MetadataCommand::new().current_dir(MANIFEST).exec().unwrap(); let cargo_gpu = metadata .packages .iter_mut() - .find(|package| package.name.contains("cargo-gpu")) + .find(|package| package.name.contains(PACKAGE)) .unwrap(); - cargo_gpu.metadata = serde_json::json!({ + cargo_gpu.metadata = json!({ "rust-gpu": { "build": { "release": false @@ -189,11 +188,9 @@ mod test { } } }); - let configs = Metadata::merge_configs(&metadata, Path::new(".")).unwrap(); - assert_eq!(configs["build"]["release"], Value::Bool(false)); - assert_eq!( - configs["install"]["auto_install_rust_toolchain"], - Value::Bool(true) - ); + + let configs = merge_configs::(&metadata, Path::new(".")).unwrap(); + assert!(!configs.build.spirv_builder.release); + assert!(configs.install.auto_install_rust_toolchain); } } diff --git a/crates/cargo-gpu/src/show.rs b/crates/cargo-gpu/src/show.rs index b7eb19b..9f76b4e 100644 --- a/crates/cargo-gpu/src/show.rs +++ b/crates/cargo-gpu/src/show.rs @@ -1,14 +1,18 @@ //! Display various information about `cargo gpu`, eg its cache directory. -use crate::cache_dir; -use crate::spirv_source::{query_metadata, SpirvSource}; -use crate::target_specs::update_target_specs_files; -use anyhow::bail; -use std::fs; -use std::path::Path; +use std::{fs, path::Path}; + +use cargo_gpu_build::{ + spirv_builder::Capability, + spirv_cache::{ + cache::cache_dir, metadata::query_metadata, spirv_source::SpirvSource, + target_specs::update_target_specs_files, + }, +}; /// Show the computed source of the spirv-std dependency. #[derive(Clone, Debug, clap::Parser)] +#[non_exhaustive] pub struct SpirvSourceDep { /// The location of the shader-crate to inspect to determine its spirv-std dependency. #[clap(long, default_value = "./")] @@ -17,6 +21,7 @@ pub struct SpirvSourceDep { /// Different tidbits of information that can be queried at the command line. #[derive(Clone, Debug, clap::Subcommand)] +#[non_exhaustive] pub enum Info { /// Displays the location of the cache directory CacheDirectory, @@ -26,21 +31,26 @@ pub enum Info { Commitsh, /// All the available SPIR-V capabilities that can be set with `--capabilities` Capabilities, - /// All available SPIR-V targets Targets(SpirvSourceDep), } -/// `cargo gpu show` +/// `cargo gpu show` subcommands. #[derive(clap::Parser)] +#[non_exhaustive] pub struct Show { - /// Display information about rust-gpu + /// Display information about rust-gpu. #[clap(subcommand)] - command: Info, + pub command: Info, } impl Show { - /// Entrypoint + /// Entrypoint of `cargo gpu show` subcommands. + /// + /// # Errors + /// + /// Returns an error if the command fails somehow. + #[inline] pub fn run(&self) -> anyhow::Result<()> { log::info!("{:?}: ", self.command); @@ -83,12 +93,12 @@ impl Show { } /// Iterator over all `Capability` variants. - fn capability_variants_iter() -> impl Iterator { + fn capability_variants_iter() -> impl Iterator { // Since spirv::Capability is repr(u32) we can iterate over // u32s until some maximum #[expect(clippy::as_conversions, reason = "We know all variants are repr(u32)")] - let last_capability = spirv_builder::Capability::CacheControlsINTEL as u32; - (0..=last_capability).filter_map(spirv_builder::Capability::from_u32) + let last_capability = Capability::CacheControlsINTEL as u32; + (0..=last_capability).filter_map(Capability::from_u32) } /// List all available spirv targets, note: the targets from compile time of cargo-gpu and those @@ -99,7 +109,7 @@ impl Show { let source = SpirvSource::new(shader_crate, None, None)?; let install_dir = source.install_dir()?; if !install_dir.is_dir() { - bail!("rust-gpu version {} is not installed", source); + anyhow::bail!("rust-gpu version {} is not installed", source); } let dummy_metadata = query_metadata(&install_dir)?; let target_specs_dir = update_target_specs_files(&source, &dummy_metadata, false)?; diff --git a/crates/cargo-gpu/src/spirv_source.rs b/crates/cargo-gpu/src/spirv_source.rs deleted file mode 100644 index 4c095e0..0000000 --- a/crates/cargo-gpu/src/spirv_source.rs +++ /dev/null @@ -1,332 +0,0 @@ -//! Use the shader that we're compiling as the default source for which version of `rust-gpu` to use. -//! -//! We do this by calling `cargo tree` inside the shader's crate to get the defined `spirv-std` -//! version. Then with that we `git checkout` the `rust-gpu` repo that corresponds to that version. -//! From there we can look at the source code to get the required Rust toolchain. - -use anyhow::Context as _; -use cargo_metadata::camino::{Utf8Path, Utf8PathBuf}; -use cargo_metadata::semver::Version; -use cargo_metadata::{Metadata, MetadataCommand, Package}; -use std::fs; -use std::path::{Path, PathBuf}; - -#[expect( - clippy::doc_markdown, - reason = "The URL should appear literally like this. But Clippy wants a markdown clickable link" -)] -/// The source and version of `rust-gpu`. -/// Eg: -/// * From crates.io with version "0.10.0" -/// * From Git with: -/// - a repo of "https://github.com/Rust-GPU/rust-gpu.git" -/// - a revision of "abc213" -/// * a local Path -#[derive(Eq, PartialEq, Clone, Debug)] -pub enum SpirvSource { - /// If the shader specifies a simple version like `spirv-std = "0.9.0"` then the source of - /// `rust-gpu` is the conventional crates.io version. - CratesIO(Version), - /// If the shader specifies a version like: - /// `spirv-std = { git = "https://github.com..." ... }` - /// then the source of `rust-gpu` is `Git`. - Git { - /// URL of the repository - url: String, - /// Revision or "commitsh" - rev: String, - }, - /// If the shader specifies a version like: - /// `spirv-std = { path = "/path/to/rust-gpu" ... }` - /// then the source of `rust-gpu` is `Path`. - Path { - /// File path of rust-gpu repository - rust_gpu_repo_root: Utf8PathBuf, - /// Version of specified rust-gpu repository - version: Version, - }, -} - -impl core::fmt::Display for SpirvSource { - #[expect( - clippy::min_ident_chars, - reason = "It's a core library trait implementation" - )] - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::CratesIO(version) => version.fmt(f), - Self::Git { url, rev } => { - // shorten rev to 8 chars, prevents windows compile errors due to too long paths... seriously - if let Some(short_rev) = rev.get(..8) { - write!(f, "{url}+{short_rev}") - } else { - write!(f, "{url}+{rev}") - } - } - Self::Path { - rust_gpu_repo_root, - version, - } => write!(f, "{rust_gpu_repo_root}+{version}"), - } - } -} - -impl SpirvSource { - /// Figures out which source of `rust-gpu` to use - pub fn new( - shader_crate_path: &Path, - maybe_rust_gpu_source: Option<&str>, - maybe_rust_gpu_version: Option<&str>, - ) -> anyhow::Result { - let source = if let Some(rust_gpu_version) = maybe_rust_gpu_version { - if let Some(rust_gpu_source) = maybe_rust_gpu_source { - Self::Git { - url: rust_gpu_source.to_owned(), - rev: rust_gpu_version.to_owned(), - } - } else { - Self::CratesIO(Version::parse(rust_gpu_version)?) - } - } else { - Self::get_rust_gpu_deps_from_shader(shader_crate_path).with_context(|| { - format!( - "get spirv-std dependency from shader crate '{}'", - shader_crate_path.display() - ) - })? - }; - Ok(source) - } - - /// Look into the shader crate to get the version of `rust-gpu` it's using. - pub fn get_rust_gpu_deps_from_shader(shader_crate_path: &Path) -> anyhow::Result { - let crate_metadata = query_metadata(shader_crate_path)?; - let spirv_std_package = crate_metadata.find_package("spirv-std")?; - let spirv_source = Self::parse_spirv_std_source_and_version(spirv_std_package)?; - log::debug!( - "Parsed `SpirvSource` from crate `{}`: \ - {spirv_source:?}", - shader_crate_path.display(), - ); - Ok(spirv_source) - } - - /// Convert the `SpirvSource` to a cache directory in which we can build it. - /// It needs to be dynamically created because an end-user might want to swap out the source, - /// maybe using their own fork for example. - pub fn install_dir(&self) -> anyhow::Result { - match self { - Self::Path { - rust_gpu_repo_root, .. - } => Ok(rust_gpu_repo_root.as_std_path().to_owned()), - Self::CratesIO { .. } | Self::Git { .. } => { - let dir = crate::to_dirname(self.to_string().as_ref()); - Ok(crate::cache_dir()?.join("codegen").join(dir)) - } - } - } - - /// Returns true if self is a Path - pub const fn is_path(&self) -> bool { - matches!(self, Self::Path { .. }) - } - - /// Parse a string like: - /// `spirv-std v0.9.0 (https://github.com/Rust-GPU/rust-gpu?rev=54f6978c#54f6978c) (*)` - /// Which would return: - /// `SpirvSource::Git("https://github.com/Rust-GPU/rust-gpu", "54f6978c")` - fn parse_spirv_std_source_and_version(spirv_std_package: &Package) -> anyhow::Result { - log::trace!("parsing spirv-std source and version from package: '{spirv_std_package:?}'"); - - let result = if let Some(source) = &spirv_std_package.source { - let is_git = source.repr.starts_with("git+"); - let is_crates_io = source.is_crates_io(); - - match (is_git, is_crates_io) { - (true, true) => anyhow::bail!("parsed both git and crates.io?"), - (true, false) => { - let parse_git = || { - let link = &source.repr.get(4..)?; - let sharp_index = link.find('#')?; - let url_end = link.find('?').unwrap_or(sharp_index); - let url = link.get(..url_end)?.to_owned(); - let rev = link.get(sharp_index + 1..)?.to_owned(); - Some(Self::Git { url, rev }) - }; - parse_git() - .with_context(|| format!("Failed to parse git url {}", &source.repr))? - } - (false, true) => Self::CratesIO(spirv_std_package.version.clone()), - (false, false) => { - anyhow::bail!("Metadata of spirv-std package uses unknown url format!") - } - } - } else { - let rust_gpu_repo_root = spirv_std_package - .manifest_path // rust-gpu/crates/spirv-std/Cargo.toml - .parent() // rust-gpu/crates/spirv-std - .and_then(Utf8Path::parent) // rust-gpu/crates - .and_then(Utf8Path::parent) // rust-gpu - .context("selecting rust-gpu workspace root dir in local path")? - .to_owned(); - if !rust_gpu_repo_root.is_dir() { - anyhow::bail!("path {rust_gpu_repo_root} is not a directory"); - } - let version = spirv_std_package.version.clone(); - Self::Path { - rust_gpu_repo_root, - version, - } - }; - - log::debug!("Parsed `rust-gpu` source and version: {result:?}"); - - Ok(result) - } -} - -/// get the Package metadata from some crate -pub fn query_metadata(crate_path: &Path) -> anyhow::Result { - log::debug!("Running `cargo metadata` on `{}`", crate_path.display()); - let metadata = MetadataCommand::new() - .current_dir( - &crate_path - .canonicalize() - .context("could not get absolute path to shader crate")?, - ) - .exec()?; - Ok(metadata) -} - -/// implements [`Self::find_package`] -pub trait FindPackage { - /// Search for a package or return a nice error - fn find_package(&self, crate_name: &str) -> anyhow::Result<&Package>; -} - -impl FindPackage for Metadata { - fn find_package(&self, crate_name: &str) -> anyhow::Result<&Package> { - if let Some(package) = self - .packages - .iter() - .find(|package| package.name.as_str() == crate_name) - { - log::trace!(" found `{}` version `{}`", package.name, package.version); - Ok(package) - } else { - anyhow::bail!( - "`{crate_name}` not found in `Cargo.toml` at `{:?}`", - self.workspace_root - ); - } - } -} - -/// Parse the `rust-toolchain.toml` in the working tree of the checked-out version of the `rust-gpu` repo. -pub fn get_channel_from_rustc_codegen_spirv_build_script( - rustc_codegen_spirv_package: &Package, -) -> anyhow::Result { - let path = rustc_codegen_spirv_package - .manifest_path - .parent() - .context("finding `rustc_codegen_spirv` crate root")?; - let build_rs = path.join("build.rs"); - - log::debug!("Parsing `build.rs` at {build_rs:?} for the used toolchain"); - let contents = fs::read_to_string(&build_rs)?; - let channel_start = "channel = \""; - let channel_line = contents - .lines() - .find_map(|line| line.strip_prefix(channel_start)) - .context(format!("Can't find `{channel_start}` line in {build_rs:?}"))?; - let channel = channel_line - .get(..channel_line.find('"').context("ending \" missing")?) - .context("can't slice version")?; - Ok(channel.to_owned()) -} - -#[cfg(test)] -mod test { - use super::*; - use cargo_metadata::{PackageBuilder, PackageId, Source}; - use cargo_util_schemas::manifest::PackageName; - - #[test_log::test] - fn parsing_spirv_std_dep_for_shader_template() { - let shader_template_path = crate::test::shader_crate_template_path(); - let source = SpirvSource::get_rust_gpu_deps_from_shader(&shader_template_path).unwrap(); - assert_eq!( - source, - SpirvSource::Git { - url: "https://github.com/Rust-GPU/rust-gpu".to_owned(), - rev: "86fc48032c4cd4afb74f1d81ae859711d20386a1".to_owned() - } - ); - } - - #[test_log::test] - fn path_sanity() { - let path = std::path::PathBuf::from("./"); - assert!(path.is_relative()); - } - - #[test_log::test] - fn cached_checkout_dir_sanity() { - let shader_template_path = crate::test::shader_crate_template_path(); - let source = SpirvSource::get_rust_gpu_deps_from_shader(&shader_template_path).unwrap(); - let dir = source.install_dir().unwrap(); - let name = dir - .file_name() - .unwrap() - .to_str() - .map(std::string::ToString::to_string) - .unwrap(); - assert_eq!("https___github_com_Rust-GPU_rust-gpu+86fc4803", &name); - } - - #[test_log::test] - fn parse_git_with_rev() { - let source = parse_git( - "git+https://github.com/Rust-GPU/rust-gpu?rev=86fc48032c4cd4afb74f1d81ae859711d20386a1#86fc4803", - ); - assert_eq!( - source, - SpirvSource::Git { - url: "https://github.com/Rust-GPU/rust-gpu".to_owned(), - rev: "86fc4803".to_owned(), - } - ); - } - - #[test_log::test] - fn parse_git_no_question_mark() { - // taken directly from Graphite - let source = parse_git( - "git+https://github.com/Rust-GPU/rust-gpu.git#6e2c84d4fe64e32df4c060c5a7f3e35a32e45421", - ); - assert_eq!( - source, - SpirvSource::Git { - url: "https://github.com/Rust-GPU/rust-gpu.git".to_owned(), - rev: "6e2c84d4fe64e32df4c060c5a7f3e35a32e45421".to_owned(), - } - ); - } - - fn parse_git(source: &str) -> SpirvSource { - let package = PackageBuilder::new( - PackageName::new("spirv-std".to_owned()).unwrap(), - Version::new(0, 9, 0), - PackageId { - repr: String::new(), - }, - "", - ) - .source(Some(Source { - repr: source.to_owned(), - })) - .build() - .unwrap(); - SpirvSource::parse_spirv_std_source_and_version(&package).unwrap() - } -} diff --git a/crates/cargo-gpu/src/target_specs.rs b/crates/cargo-gpu/src/target_specs.rs deleted file mode 100644 index eef8dca..0000000 --- a/crates/cargo-gpu/src/target_specs.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! This module deals with target specs, which are json metadata files that need to be passed to -//! rustc to add foreign targets such as `spirv_unknown_vulkan1.2`. -//! -//! There are 4 version ranges of `rustc_codegen_spirv` and they all need different handling of -//! their target specs: -//! * "ancient" versions such as 0.9.0 or earlier do not need target specs, just passing the target -//! string (`spirv-unknown-vulkan1.2`) directly is sufficient. We still prep target-specs for them -//! like the "legacy" variant below, spirv-builder -//! [will just ignore it](https://github.com/Rust-GPU/rust-gpu/blob/369122e1703c0c32d3d46f46fa11ccf12667af03/crates/spirv-builder/src/lib.rs#L987) -//! * "legacy" versions require target specs to compile, which is a requirement introduced by some -//! rustc version. Back then it was decided that cargo gpu would ship them, as they'd probably -//! never change, right? So now we're stuck with having to ship these "legacy" target specs with -//! cargo gpu *forever*. These are the symbol `legacy_target_specs::TARGET_SPECS`, with -//! `legacy_target_specs` being a **fixed** version of `rustc_codegen_spirv-target-specs`, -//! which must **never** update. -//! * As of [PR 256](https://github.com/Rust-GPU/rust-gpu/pull/256), `rustc_codegen_spirv` now has -//! a direct dependency on `rustc_codegen_spirv-target-specs`, allowing cargo gpu to pull the -//! required target specs directly from that dependency. At this point, the target specs are -//! still the same as the legacy target specs. -//! * The [edition 2024 PR](https://github.com/Rust-GPU/rust-gpu/pull/249) must update the -//! target specs to comply with newly added validation within rustc. This is why the new system -//! was implemented, so we can support both old and new target specs without having to worry -//! which version of cargo gpu you are using. It'll "just work". - -use crate::cache_dir; -use crate::spirv_source::{FindPackage as _, SpirvSource}; -use anyhow::Context as _; -use cargo_metadata::Metadata; -use std::path::{Path, PathBuf}; - -/// Extract legacy target specs from our executable into some directory -pub fn write_legacy_target_specs(target_spec_dir: &Path) -> anyhow::Result<()> { - std::fs::create_dir_all(target_spec_dir)?; - for (filename, contents) in legacy_target_specs::TARGET_SPECS { - let path = target_spec_dir.join(filename); - std::fs::write(&path, contents.as_bytes()) - .with_context(|| format!("writing legacy target spec file at [{}]", path.display()))?; - } - Ok(()) -} - -/// Copy spec files from one dir to another, assuming no subdirectories -fn copy_spec_files(src: &Path, dst: &Path) -> anyhow::Result<()> { - std::fs::create_dir_all(dst)?; - let dir = std::fs::read_dir(src)?; - for dir_entry in dir { - let file = dir_entry?; - let file_path = file.path(); - if file_path.is_file() { - std::fs::copy(file_path, dst.join(file.file_name()))?; - } - } - Ok(()) -} - -/// Computes the `target-specs` directory to use and updates the target spec files, if enabled. -pub fn update_target_specs_files( - source: &SpirvSource, - dummy_metadata: &Metadata, - update_files: bool, -) -> anyhow::Result { - log::info!( - "target-specs: Resolving target specs `{}`", - if update_files { - "and update them" - } else { - "without updating" - } - ); - - let mut target_specs_dst = source.install_dir()?.join("target-specs"); - if let Ok(target_specs) = dummy_metadata.find_package("rustc_codegen_spirv-target-specs") { - log::info!( - "target-specs: found crate `rustc_codegen_spirv-target-specs` with manifest at `{}`", - target_specs.manifest_path - ); - - let target_specs_src = target_specs - .manifest_path - .as_std_path() - .parent() - .and_then(|root| { - let src = root.join("target-specs"); - src.is_dir().then_some(src) - }) - .context("Could not find `target-specs` directory within `rustc_codegen_spirv-target-specs` dependency")?; - log::info!( - "target-specs: found `rustc_codegen_spirv-target-specs` with `target-specs` directory `{}`", - target_specs_dst.display() - ); - - if source.is_path() { - // skip copy - log::info!( - "target-specs resolution: source is local path, use target-specs directly from `{}`", - target_specs_dst.display() - ); - target_specs_dst = target_specs_src; - } else { - // copy over the target-specs - log::info!( - "target-specs resolution: copying target-specs from `{}`{}", - target_specs_dst.display(), - if update_files { "" } else { " was skipped" } - ); - if update_files { - copy_spec_files(&target_specs_src, &target_specs_dst) - .context("copying target-specs json files")?; - } - } - } else { - // use legacy target specs bundled with cargo gpu - if source.is_path() { - // This is a stupid situation: - // * We can't be certain that there are `target-specs` in the local checkout (there may be some in `spirv-builder`) - // * We can't dump our legacy ones into the `install_dir`, as that would modify the local rust-gpu checkout - // -> do what the old cargo gpu did, one global dir for all target specs - // and hope parallel runs don't shred each other - target_specs_dst = cache_dir()?.join("legacy-target-specs-for-local-checkout"); - } - log::info!( - "target-specs resolution: legacy target specs in directory `{}`", - target_specs_dst.display() - ); - if update_files { - log::info!( - "target-specs: writing legacy target specs into `{}`", - target_specs_dst.display() - ); - write_legacy_target_specs(&target_specs_dst)?; - } - } - - Ok(target_specs_dst) -} diff --git a/crates/cargo-gpu/src/test.rs b/crates/cargo-gpu/src/test.rs index c9ee93a..bfe63a9 100644 --- a/crates/cargo-gpu/src/test.rs +++ b/crates/cargo-gpu/src/test.rs @@ -1,8 +1,8 @@ //! utilities for tests + #![cfg(test)] -use crate::cache_dir; -use std::io::Write as _; +use std::{cell::RefCell, io::Write as _}; fn copy_dir_all( src: impl AsRef, @@ -26,10 +26,18 @@ pub fn shader_crate_template_path() -> std::path::PathBuf { project_base.join("../shader-crate-template") } +thread_local! { + static TEMPDIR: RefCell> = RefCell::new(Some( + tempfile::TempDir::with_prefix("shader_crate").unwrap(), + )); +} + pub fn shader_crate_test_path() -> std::path::PathBuf { - let shader_crate_path = crate::cache_dir().unwrap().join("shader_crate"); - copy_dir_all(shader_crate_template_path(), shader_crate_path.clone()).unwrap(); - shader_crate_path + TEMPDIR.with_borrow(|tempdir| { + let shader_crate_path = tempdir.as_ref().unwrap().path(); + copy_dir_all(shader_crate_template_path(), shader_crate_path).unwrap(); + shader_crate_path.to_path_buf() + }) } pub fn overwrite_shader_cargo_toml(shader_crate_path: &std::path::Path) -> std::fs::File { @@ -45,9 +53,7 @@ pub fn overwrite_shader_cargo_toml(shader_crate_path: &std::path::Path) -> std:: } pub fn tests_teardown() { - let cache_dir = cache_dir().unwrap(); - if !cache_dir.exists() { - return; - } - std::fs::remove_dir_all(cache_dir).unwrap(); + TEMPDIR.with_borrow_mut(|tempdir| { + tempdir.take().unwrap().close().unwrap(); + }); } diff --git a/crates/cargo-gpu/src/user_consent.rs b/crates/cargo-gpu/src/user_consent.rs new file mode 100644 index 0000000..af2f308 --- /dev/null +++ b/crates/cargo-gpu/src/user_consent.rs @@ -0,0 +1,104 @@ +//! User consent acquiring logic. + +use std::io; + +use cargo_gpu_build::spirv_cache::{ + command::CommandExecError, + toolchain::{HaltToolchainInstallation, REQUIRED_TOOLCHAIN_COMPONENTS}, + user_output, +}; +use crossterm::tty::IsTty as _; + +/// Halts the installation process of toolchain or its required components +/// if the user does not consent to install either of them. +#[expect( + clippy::type_complexity, + reason = "it is impossible to create an alias for now" +)] +pub fn ask_for_user_consent( + skip: bool, +) -> HaltToolchainInstallation< + impl FnOnce(&str) -> Result<(), UserConsentError>, + impl FnOnce(&str) -> Result<(), UserConsentError>, +> { + let on_toolchain_install = move |channel: &str| { + let message = format!("Rust {channel} with `rustup`"); + get_user_consent(format!("Install {message}"), skip)?; + log::debug!("installing toolchain {channel}"); + user_output!(io::stdout(), "Installing {message}\n").map_err(UserConsentError::IoWrite)?; + Ok(()) + }; + let on_components_install = move |channel: &str| { + let message = format!( + "components {REQUIRED_TOOLCHAIN_COMPONENTS:?} for toolchain {channel} with `rustup`" + ); + get_user_consent(format!("Install {message}"), skip)?; + log::debug!("installing required components of toolchain {channel}"); + user_output!(io::stdout(), "Installing {message}\n").map_err(UserConsentError::IoWrite)?; + Ok(()) + }; + + HaltToolchainInstallation { + on_toolchain_install, + on_components_install, + } +} + +/// Prompt user if they want to install a new Rust toolchain or its components. +fn get_user_consent(prompt: impl AsRef, skip: bool) -> Result<(), UserConsentError> { + if skip { + return Ok(()); + } + + if !io::stdout().is_tty() { + log::error!("attempted to ask for consent when there's no TTY"); + return Err(UserConsentError::NoTTY); + } + + log::debug!("asking for consent to install the required toolchain"); + crossterm::terminal::enable_raw_mode().map_err(UserConsentError::IoRead)?; + user_output!(io::stdout(), "{} [y/n]: ", prompt.as_ref()).map_err(UserConsentError::IoWrite)?; + let mut input = crossterm::event::read().map_err(UserConsentError::IoRead)?; + + if let crossterm::event::Event::Key(crossterm::event::KeyEvent { + code: crossterm::event::KeyCode::Enter, + kind: crossterm::event::KeyEventKind::Release, + .. + }) = input + { + // In Powershell, programs will potentially observe the Enter key release after they started + // (see crossterm#124). If that happens, re-read the input. + input = crossterm::event::read().map_err(UserConsentError::IoRead)?; + } + crossterm::terminal::disable_raw_mode().map_err(UserConsentError::IoRead)?; + + if let crossterm::event::Event::Key(crossterm::event::KeyEvent { + code: crossterm::event::KeyCode::Char('y'), + .. + }) = input + { + Ok(()) + } else { + Err(UserConsentError::UserDenied) + } +} + +/// An error indicating that user consent were not acquired. +#[derive(Debug, thiserror::Error)] +pub enum UserConsentError { + /// An error occurred while executing a command. + #[error(transparent)] + CommandExec(#[from] CommandExecError), + /// No TTY detected, so can't ask for consent to install Rust toolchain. + #[error("no TTY detected, so can't ask for consent to install Rust toolchain")] + NoTTY, + /// An I/O error occurred while reading user input. + #[error("failed to read user input: {0}")] + IoRead(#[source] io::Error), + /// An I/O error occurred while writing user output. + #[error("failed to write user output: {0}")] + IoWrite(#[source] io::Error), + /// User denied to install the required toolchain. + #[error("user denied to install the required toolchain")] + UserDenied, +} diff --git a/crates/rustc_codegen_spirv-cache/Cargo.toml b/crates/rustc_codegen_spirv-cache/Cargo.toml new file mode 100644 index 0000000..f117b39 --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "rustc_codegen_spirv-cache" +description = "Cacher of `rust-gpu` codegen for required toolchain" +version.workspace = true +edition.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true + +[dependencies] +spirv-builder.workspace = true +legacy_target_specs.workspace = true +thiserror.workspace = true +directories.workspace = true +cargo_metadata.workspace = true +log.workspace = true +serde.workspace = true +clap = { workspace = true, optional = true } + +[dev-dependencies] +test-log.workspace = true +cargo-util-schemas.workspace = true + +[features] +# Enables `clap` support for public structs +clap = ["dep:clap", "spirv-builder/clap"] +# Enables `watch` feature for `spirv-builder` +watch = ["spirv-builder/watch"] + +[lints] +workspace = true diff --git a/crates/rustc_codegen_spirv-cache/src/backend.rs b/crates/rustc_codegen_spirv-cache/src/backend.rs new file mode 100644 index 0000000..22315a5 --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/src/backend.rs @@ -0,0 +1,535 @@ +//! This module deals with an installation a dedicated per-shader crate +//! that has the `rust-gpu` codegen backend in it. +//! +//! This process could be described as follows: +//! * first retrieve the version of rust-gpu you want to use based on the version of the +//! `spirv-std` dependency in your shader crate, +//! * then create a dummy project at `/codegen//` +//! that depends on `rustc_codegen_spirv`, +//! * use `cargo metadata` to `cargo update` the dummy project, which downloads the +//! `rustc_codegen_spirv` crate into cargo's cache, and retrieve the path to the +//! download location, +//! * search for the required toolchain in `build.rs` of `rustc_codegen_spirv`, +//! * build it with the required toolchain version, +//! * copy out the resulting dylib and clean the target directory. + +use std::{ + fs, io, + path::{Path, PathBuf}, + process::Stdio, +}; + +use crate::{ + cache::{cache_dir, CacheDirError}, + command::{execute_command, CommandExecError}, + metadata::{query_metadata, MetadataExt as _, MissingPackageError, QueryMetadataError}, + spirv_builder::{cargo_cmd::CargoCmd, SpirvBuilder, SpirvBuilderError}, + spirv_source::{ + rust_gpu_toolchain_channel, RustGpuToolchainChannelError, SpirvSource, SpirvSourceError, + }, + target_specs::{update_target_specs_files, UpdateTargetSpecsFilesError}, + toolchain::{ + ensure_toolchain_installation, HaltToolchainInstallation, NoopOnComponentsInstall, + NoopOnToolchainInstall, NullStderr, NullStdout, StdioCfg, + }, + user_output, +}; + +/// Represents a functional backend installation, whether it was cached or just installed. +#[derive(Debug, Default, Clone)] +#[non_exhaustive] +#[expect(clippy::module_name_repetitions, reason = "this is intended")] +pub struct SpirvCodegenBackend { + /// Path to the `rustc_codegen_spirv` dylib. + pub rustc_codegen_spirv_location: PathBuf, + /// Toolchain channel name. + pub toolchain_channel: String, + /// Directory with target specs json files. + pub target_spec_dir: PathBuf, +} + +impl SpirvCodegenBackend { + /// Creates a new [`SpirvBuilder`] configured to use this installed backend. + #[expect( + clippy::unreachable, + reason = "it's unreachable, no need to return a Result" + )] + #[expect(clippy::impl_trait_in_params, reason = "forwarding spirv-builder API")] + #[inline] + pub fn to_spirv_builder( + &self, + path_to_crate: impl AsRef, + target: impl Into, + ) -> SpirvBuilder { + let mut builder = SpirvBuilder::new(path_to_crate, target); + self.configure_spirv_builder(&mut builder) + .unwrap_or_else(|_| unreachable!("we set target before calling this function")); + builder + } + + /// Configures the supplied [`SpirvBuilder`]. + /// [`SpirvBuilder::target`] must be set and must not change after calling this function. + /// + /// # Errors + /// + /// Returns an error if [`SpirvBuilder::target`] is not set. + #[inline] + pub fn configure_spirv_builder( + &self, + builder: &mut SpirvBuilder, + ) -> Result<(), SpirvBuilderError> { + builder.rustc_codegen_spirv_location = Some(self.rustc_codegen_spirv_location.clone()); + builder.toolchain_overwrite = Some(self.toolchain_channel.clone()); + + let target = builder + .target + .as_deref() + .ok_or(SpirvBuilderError::MissingTarget)?; + builder.path_to_target_spec = Some(self.target_spec_dir.join(format!("{target}.json"))); + Ok(()) + } +} + +/// Settings for an installation of the codegen backend. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +#[cfg_attr(feature = "clap", derive(clap::Parser))] +#[non_exhaustive] +pub struct SpirvCodegenBackendInstaller { + #[expect( + rustdoc::bare_urls, + clippy::doc_markdown, + reason = "The URL should appear literally like this. But Clippy & rustdoc want a markdown clickable link" + )] + /// Source of [`spirv-builder`](spirv_builder) dependency. + /// + /// E.g. "https://github.com/Rust-GPU/rust-gpu". + #[cfg_attr(feature = "clap", clap(long))] + pub spirv_builder_source: Option, + + /// Version of [`spirv-builder`](spirv_builder) dependency. + /// + /// * If `--spirv-builder-source` is not set, then this is assumed to be a crates.io semantic + /// version such as "0.9.0". + /// * If `--spirv-builder-source` is set, then this is assumed to be a Git "commitsh", such + /// as a Git commit hash or a Git tag, therefore anything that `git checkout` can resolve. + #[cfg_attr(feature = "clap", clap(long, verbatim_doc_comment))] + pub spirv_builder_version: Option, + + /// Force `rustc_codegen_spirv` to be rebuilt. + #[cfg_attr(feature = "clap", clap(long))] + pub rebuild_codegen: bool, + + /// Clear target dir of `rustc_codegen_spirv` build after a successful build, + /// saves about 200MiB of disk space. + #[cfg_attr(feature = "clap", clap(long = "no-clear-target", default_value = "true", action = clap::ArgAction::SetFalse))] + pub clear_target: bool, +} + +impl Default for SpirvCodegenBackendInstaller { + #[inline] + fn default() -> Self { + Self { + spirv_builder_source: None, + spirv_builder_version: None, + rebuild_codegen: false, + clear_target: true, + } + } +} + +impl SpirvCodegenBackendInstaller { + /// Sets the source of [`spirv-builder`](spirv_builder) dependency. + #[inline] + #[must_use] + pub fn spirv_builder_source(self, spirv_builder_source: I) -> Self + where + I: Into>, + { + Self { + spirv_builder_source: spirv_builder_source.into(), + ..self + } + } + + /// Sets the version of [`spirv-builder`](spirv_builder) dependency. + #[inline] + #[must_use] + pub fn spirv_builder_version(self, spirv_builder_version: I) -> Self + where + I: Into>, + { + Self { + spirv_builder_version: spirv_builder_version.into(), + ..self + } + } + + /// Sets whether to force `rustc_codegen_spirv` to be rebuilt. + #[inline] + #[must_use] + pub fn rebuild_codegen(self, rebuild_codegen: bool) -> Self { + Self { + rebuild_codegen, + ..self + } + } + + /// Sets whether to clear target dir of `rustc_codegen_spirv` build after a successful build. + #[inline] + #[must_use] + pub fn clear_target(self, clear_target: bool) -> Self { + Self { + clear_target, + ..self + } + } + + /// Create the `rustc_codegen_spirv_dummy` crate that depends on `rustc_codegen_spirv` + fn write_source_files( + source: &SpirvSource, + checkout: &Path, + ) -> Result<(), SpirvCodegenBackendInstallError> { + // skip writing a dummy project if we use a local rust-gpu checkout + if source.is_path() { + return Ok(()); + } + + log::debug!( + "writing `rustc_codegen_spirv_dummy` source files into {}", + checkout.display() + ); + + log::trace!("writing dummy lib.rs"); + let src = checkout.join("src"); + fs::create_dir_all(&src).map_err(SpirvCodegenBackendInstallError::CreateDummySrcDir)?; + fs::File::create(src.join("lib.rs")) + .map_err(SpirvCodegenBackendInstallError::CreateDummyLibRs)?; + + log::trace!("writing dummy Cargo.toml"); + fs::write(checkout.join("Cargo.toml"), dummy_cargo_toml(source)) + .map_err(SpirvCodegenBackendInstallError::WriteDummyCargoToml)?; + + Ok(()) + } + + /// Installs the `rust-gpu` [codegen backend](SpirvCodegenBackend) for the shader crate, + /// from which you can create [`SpirvBuilder`] instances. + /// + /// # Errors + /// + /// Returns an error if the installation somehow fails. + /// See [`SpirvCodegenBackendInstallError`] for further details. + #[inline] + pub fn install( + &self, + params: I, + ) -> Result> + where + I: Into>, + W: io::Write, + R: From, + T: FnOnce(&str) -> Result<(), R>, + C: FnOnce(&str) -> Result<(), R>, + O: FnMut() -> Stdio, + E: FnMut() -> Stdio, + { + // Ensure the cache dir exists + let cache_dir = cache_dir()?; + log::info!("cache directory is '{}'", cache_dir.display()); + if let Err(source) = fs::create_dir_all(&cache_dir) { + return Err(SpirvCodegenBackendInstallError::CreateCacheDir { cache_dir, source }); + } + + let SpirvCodegenBackendInstallParams { + shader_crate, + writer, + halt, + stdio_cfg, + } = params.into(); + + let source = SpirvSource::new( + &shader_crate, + self.spirv_builder_source.as_deref(), + self.spirv_builder_version.as_deref(), + )?; + let install_dir = source.install_dir()?; + + let dylib_filename = dylib_filename("rustc_codegen_spirv"); + let dest_dylib_path; + if source.is_path() { + dest_dylib_path = install_dir + .join("target") + .join("release") + .join(&dylib_filename); + } else { + dest_dylib_path = install_dir.join(&dylib_filename); + if dest_dylib_path.is_file() { + log::info!( + "cargo-gpu artifacts are already installed in '{}'", + install_dir.display() + ); + } + } + + // if `source` is a path, always rebuild + let skip_rebuild = !source.is_path() && dest_dylib_path.is_file() && !self.rebuild_codegen; + if skip_rebuild { + log::info!("...and so we are aborting the install step."); + } else { + Self::write_source_files(&source, &install_dir)?; + } + + // TODO cache toolchain channel in a file? + log::debug!("resolving toolchain version to use"); + let dummy_metadata = query_metadata(&install_dir)?; + let rustc_codegen_spirv = dummy_metadata.find_package("rustc_codegen_spirv")?; + let toolchain_channel = rust_gpu_toolchain_channel(rustc_codegen_spirv)?; + log::info!("selected toolchain channel `{toolchain_channel:?}`"); + + log::debug!("Update target specs files"); + let target_spec_dir = update_target_specs_files(&source, &dummy_metadata, !skip_rebuild)?; + + log::debug!("ensure_toolchain_and_components_exist"); + ensure_toolchain_installation(&toolchain_channel, halt, stdio_cfg) + .map_err(SpirvCodegenBackendInstallError::EnsureToolchainInstallation)?; + + if !skip_rebuild { + // to prevent unsupported version errors when using older toolchains + if !source.is_path() { + log::debug!("remove Cargo.lock"); + fs::remove_file(install_dir.join("Cargo.lock")) + .map_err(SpirvCodegenBackendInstallError::RemoveDummyCargoLock)?; + } + + user_output!(writer, "Compiling `rustc_codegen_spirv` from {source}\n") + .map_err(SpirvCodegenBackendInstallError::IoWrite)?; + + let mut cargo = CargoCmd::new(); + cargo + .current_dir(&install_dir) + .arg(format!("+{toolchain_channel}")) + .args(["build", "--release"]); + if source.is_path() { + cargo.args(["-p", "rustc_codegen_spirv", "--lib"]); + } + cargo.stdout(Stdio::inherit()).stderr(Stdio::inherit()); + + log::debug!("building artifacts with `{cargo:?}`"); + execute_command(cargo)?; + + let target = install_dir.join("target"); + let dylib_path = target.join("release").join(&dylib_filename); + if dylib_path.is_file() { + log::info!("successfully built {}", dylib_path.display()); + if !source.is_path() { + fs::rename(&dylib_path, &dest_dylib_path) + .map_err(SpirvCodegenBackendInstallError::MoveRustcCodegenSpirvDylib)?; + + if self.clear_target { + log::warn!("clearing target dir {}", target.display()); + fs::remove_dir_all(&target).map_err( + SpirvCodegenBackendInstallError::RemoveRustcCodegenSpirvTargetDir, + )?; + } + } + } else { + log::error!("could not find {}", dylib_path.display()); + return Err(SpirvCodegenBackendInstallError::RustcCodegenSpirvDylibNotFound); + } + } + + Ok(SpirvCodegenBackend { + rustc_codegen_spirv_location: dest_dylib_path, + toolchain_channel, + target_spec_dir, + }) + } +} + +/// Parameters for [`SpirvCodegenBackendInstaller::install()`]. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct SpirvCodegenBackendInstallParams { + /// Path to the shader crate to install the codegen backend for. + pub shader_crate: PathBuf, + /// Writer of user output. + pub writer: W, + /// Callbacks to halt toolchain installation. + pub halt: HaltToolchainInstallation, + /// Configuration of [`Stdio`] for commands run during installation. + pub stdio_cfg: StdioCfg, +} + +impl SpirvCodegenBackendInstallParams { + /// Replaces path to the shader crate to install the codegen backend for. + #[inline] + #[must_use] + pub fn shader_crate

(self, shader_crate: P) -> Self + where + P: Into, + { + Self { + shader_crate: shader_crate.into(), + ..self + } + } + + /// Replaces the writer of user output. + #[inline] + #[must_use] + pub fn writer(self, writer: NW) -> SpirvCodegenBackendInstallParams { + SpirvCodegenBackendInstallParams { + shader_crate: self.shader_crate, + writer, + halt: self.halt, + stdio_cfg: self.stdio_cfg, + } + } + + /// Replaces the callbacks to halt toolchain installation. + #[inline] + #[must_use] + pub fn halt( + self, + halt: HaltToolchainInstallation, + ) -> SpirvCodegenBackendInstallParams { + SpirvCodegenBackendInstallParams { + shader_crate: self.shader_crate, + writer: self.writer, + halt, + stdio_cfg: self.stdio_cfg, + } + } + + /// Replaces the [`Stdio`] configuration for commands run during installation. + #[inline] + #[must_use] + pub fn stdio_cfg( + self, + stdio_cfg: StdioCfg, + ) -> SpirvCodegenBackendInstallParams { + SpirvCodegenBackendInstallParams { + shader_crate: self.shader_crate, + writer: self.writer, + halt: self.halt, + stdio_cfg, + } + } +} + +/// [`Default`] parameters for [`SpirvCodegenBackendInstaller::install()`]. +pub type DefaultSpirvCodegenBackendInstallParams = SpirvCodegenBackendInstallParams< + io::Empty, + NoopOnToolchainInstall, + NoopOnComponentsInstall, + NullStdout, + NullStderr, +>; + +impl

From

for DefaultSpirvCodegenBackendInstallParams +where + P: Into, +{ + #[inline] + fn from(path_to_crate: P) -> Self { + Self { + shader_crate: path_to_crate.into(), + writer: io::empty(), + halt: HaltToolchainInstallation::noop(), + stdio_cfg: StdioCfg::null(), + } + } +} + +/// Returns the platform-specific filename of the dylib with the given name. +#[inline] +fn dylib_filename(name: impl AsRef) -> String { + use std::env::consts::{DLL_PREFIX, DLL_SUFFIX}; + + let str_name = name.as_ref(); + format!("{DLL_PREFIX}{str_name}{DLL_SUFFIX}") +} + +/// Contents of the `Cargo.toml` file for the local `rustc_codegen_spirv_dummy` crate +/// without the version specification of the `rustc_codegen_spirv` dependency. +const DUMMY_CARGO_TOML_NO_VERSION_SPEC: &str = include_str!("dummy/Cargo.toml"); + +/// Returns the contents of the `Cargo.toml` file for the local `rustc_codegen_spirv_dummy` crate. +fn dummy_cargo_toml(source: &SpirvSource) -> String { + let version_spec = match source { + SpirvSource::CratesIO(version) => format!("version = \"{version}\""), + SpirvSource::Git { url, rev } => format!("git = \"{url}\"\nrev = \"{rev}\""), + SpirvSource::Path { + rust_gpu_repo_root, + version, + } => { + // this branch is currently unreachable, as we just build `rustc_codegen_spirv` directly, + // since we don't need the `dummy` crate to make cargo download it for us + let new_path = rust_gpu_repo_root.join("crates").join("spirv-builder"); + format!("path = \"{new_path}\"\nversion = \"{version}\"") + } + }; + format!("{DUMMY_CARGO_TOML_NO_VERSION_SPEC}{version_spec}\n") +} + +/// An error indicating codegen `rustc_codegen_spirv` installation failure. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum SpirvCodegenBackendInstallError { + /// Failed to write user output. + #[error("failed to write user output: {0}")] + IoWrite(#[source] io::Error), + /// There is no cache directory available. + #[error(transparent)] + NoCacheDir(#[from] CacheDirError), + /// Failed to create the cache directory. + #[error("failed to create cache directory {cache_dir}: {source}")] + CreateCacheDir { + /// Path to the cache directory we tried to create. + cache_dir: PathBuf, + /// The source of the error. + source: io::Error, + }, + /// Failed to determine the source of `rust-gpu`. + #[error(transparent)] + SpirvSource(#[from] SpirvSourceError), + /// Failed to create `src` directory for local `rustc_codegen_spirv_dummy` crate. + #[error("failed to create `src` directory for `rustc_codegen_spirv_dummy`: {0}")] + CreateDummySrcDir(#[source] io::Error), + /// Failed to create `src/lib.rs` file for local `rustc_codegen_spirv_dummy` crate. + #[error("failed to create `src/lib.rs` file for `rustc_codegen_spirv_dummy`: {0}")] + CreateDummyLibRs(#[source] io::Error), + /// Failed to write `Cargo.toml` file for local `rustc_codegen_spirv_dummy` crate. + #[error("failed to write `Cargo.toml` file for `rustc_codegen_spirv_dummy`: {0}")] + WriteDummyCargoToml(#[source] io::Error), + /// Failed to query cargo metadata of the local `rustc_codegen_spirv_dummy` crate. + #[error(transparent)] + QueryDummyMetadata(#[from] QueryMetadataError), + /// Could not find `rustc_codegen_spirv` dependency in the local `rustc_codegen_spirv_dummy` crate. + #[error(transparent)] + NoRustcCodegenSpirv(#[from] MissingPackageError), + /// Failed to determine the toolchain channel of `rustc_codegen_spirv`. + #[error("could not get toolchain channel of `rustc_codegen_spirv`: {0}")] + RustGpuToolchainChannel(#[from] RustGpuToolchainChannelError), + /// Failed to update target specs files. + #[error(transparent)] + UpdateTargetSpecsFiles(#[from] UpdateTargetSpecsFilesError), + /// Failed to ensure installation of a toolchain and its required components. + #[error("failed to ensure toolchain and components exist: {0}")] + EnsureToolchainInstallation(#[source] E), + /// Failed to remove `Cargo.lock` file for local `rustc_codegen_spirv_dummy` crate. + #[error("failed to remove `Cargo.lock` file for `rustc_codegen_spirv_dummy`: {0}")] + RemoveDummyCargoLock(#[source] io::Error), + /// Failed to move `rustc_codegen_spirv` to its final location. + #[error("failed to move `rustc_codegen_spirv` to final location: {0}")] + MoveRustcCodegenSpirvDylib(#[source] io::Error), + /// Failed to remove target dir from `rustc_codegen_spirv`. + #[error("failed to remove `target` dir from compiled codegen `rustc_codegen_spirv`: {0}")] + RemoveRustcCodegenSpirvTargetDir(#[source] io::Error), + /// Failed to build `rustc_codegen_spirv` by `cargo`. + #[error(transparent)] + RustcCodegenSpirvBuild(#[from] CommandExecError), + /// The `rustc_codegen_spirv` build did not produce the expected dylib. + #[error("`rustc_codegen_spirv` build did not produce the expected dylib")] + RustcCodegenSpirvDylibNotFound, +} diff --git a/crates/rustc_codegen_spirv-cache/src/cache.rs b/crates/rustc_codegen_spirv-cache/src/cache.rs new file mode 100644 index 0000000..4da89d6 --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/src/cache.rs @@ -0,0 +1,29 @@ +//! Defines caching policy of the crate. + +#![expect(clippy::module_name_repetitions, reason = "this is intended")] + +use std::path::PathBuf; + +/// Returns path to the directory where all the cache files are located. +/// +/// Possible values by OS are: +/// * Windows: `C:/users//AppData/Local/rust-gpu` +/// * Mac: `~/Library/Caches/rust-gpu` +/// * Linux: `~/.cache/rust-gpu` +/// +/// # Errors +/// +/// Fails if there is no cache directory available. +#[inline] +pub fn cache_dir() -> Result { + let dir = directories::BaseDirs::new() + .ok_or(CacheDirError(()))? + .cache_dir() + .join("rust-gpu"); + Ok(dir) +} + +/// An error indicating that there is no cache directory available. +#[derive(Debug, Clone, thiserror::Error)] +#[error("could not find cache directory")] +pub struct CacheDirError(()); diff --git a/crates/rustc_codegen_spirv-cache/src/command.rs b/crates/rustc_codegen_spirv-cache/src/command.rs new file mode 100644 index 0000000..9021b9d --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/src/command.rs @@ -0,0 +1,79 @@ +//! Utilities for executing [commands](Command). + +use std::{ + io, + process::{Command, Output}, +}; + +/// An error indicating failure while executing some command. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +#[expect(clippy::module_name_repetitions, reason = "this is intended")] +pub enum CommandExecError { + /// IO error occurred while calling some command. + #[error("IO error occurred while calling `{command:?}`: {source}")] + Io { + /// The command which was called. + command: Box, + /// Source of the error. + source: io::Error, + }, + /// Result of calling some command was not successful. + #[error("calling `{command:?}` was not successful")] + ExecFail { + /// The command which was called. + command: Box, + /// The output of called command. + output: Output, + }, +} + +impl CommandExecError { + /// Creates [`Io`](CommandExecError::Io) variant from given arguments. + fn io(command: impl Into, source: io::Error) -> Self { + Self::Io { + command: Box::new(command.into()), + source, + } + } + + /// Creates [`ExecFail`](CommandExecError::ExecFail) variant from given arguments. + fn exec_fail(command: impl Into, output: Output) -> Self { + Self::ExecFail { + command: Box::new(command.into()), + output, + } + } + + /// Returns the command which was called. + #[inline] + #[expect(clippy::must_use_candidate, reason = "returns a reference")] + pub fn command(&self) -> &Command { + match self { + Self::Io { command, .. } | Self::ExecFail { command, .. } => command.as_ref(), + } + } + + /// Converts self into the command which was called. + #[inline] + #[must_use] + pub fn into_command(self) -> Command { + match self { + Self::Io { command, .. } | Self::ExecFail { command, .. } => *command, + } + } +} + +/// Executes the command, returning its output. +#[expect(clippy::shadow_reuse, reason = "this is intended")] +pub(crate) fn execute_command(command: impl Into) -> Result { + let mut command = command.into(); + let output = match command.output() { + Ok(output) => output, + Err(source) => return Err(CommandExecError::io(command, source)), + }; + if !output.status.success() { + return Err(CommandExecError::exec_fail(command, output)); + } + Ok(output) +} diff --git a/crates/rustc_codegen_spirv-cache/src/dummy/Cargo.toml b/crates/rustc_codegen_spirv-cache/src/dummy/Cargo.toml new file mode 100644 index 0000000..2d34d74 --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/src/dummy/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "rustc_codegen_spirv_dummy" +version = "0.1.0" +edition = "2021" + +[dependencies.spirv-builder] +package = "rustc_codegen_spirv" diff --git a/crates/rustc_codegen_spirv-cache/src/lib.rs b/crates/rustc_codegen_spirv-cache/src/lib.rs new file mode 100644 index 0000000..30b2451 --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/src/lib.rs @@ -0,0 +1,45 @@ +//! Cacher of `rust-gpu` codegen for required toolchain. +//! +//! This library manages installations of `rustc_codegen_spirv`, +//! the codegen backend of rust-gpu to generate SPIR-V shader binaries. +//! +//! # How it works +//! +//! The codegen backend builds on internal, ever-changing interfaces of rustc, +//! which requires fixing a version of rust-gpu to a specific version of the rustc compiler. +//! Usually, this would require you to fix your entire project to that specific +//! toolchain, but this project loosens that requirement by managing installations +//! of `rustc_codegen_spirv` and their associated toolchains for you. + +#![expect(clippy::pub_use, reason = "part of public API")] + +pub use cargo_metadata; +pub use spirv_builder; + +pub mod backend; +pub mod cache; +pub mod command; +pub mod metadata; +pub mod spirv_source; +pub mod target_specs; +pub mod toolchain; + +/// Writes formatted user output into a [writer](std::io::Write). +#[macro_export] +macro_rules! user_output { + ($dst:expr, $($args:tt)*) => {{ + #[allow( + clippy::allow_attributes, + clippy::useless_attribute, + unused_imports, + reason = "`std::io::Write` is only sometimes called??" + )] + use ::std::io::Write as _; + + let mut writer = $dst; + #[expect(clippy::non_ascii_literal, reason = "CRAB GOOD. CRAB IMPORTANT.")] + ::std::write!(writer, "🦀 ") + .and_then(|()| ::std::write!(writer, $($args)*)) + .and_then(|()| ::std::io::Write::flush(&mut writer)) + }}; +} diff --git a/crates/rustc_codegen_spirv-cache/src/metadata.rs b/crates/rustc_codegen_spirv-cache/src/metadata.rs new file mode 100644 index 0000000..4d96c06 --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/src/metadata.rs @@ -0,0 +1,81 @@ +//! Functionality of the crate which is tightly linked +//! with cargo [metadata](Metadata). + +#![expect(clippy::module_name_repetitions, reason = "this is intended")] + +use std::{io, path::Path}; + +use cargo_metadata::{camino::Utf8PathBuf, Metadata, MetadataCommand, Package}; + +/// Get the package metadata from the shader crate located at `crate_path`. +/// +/// # Errors +/// +/// Returns an error if the path does not exist, non-final part of it is not a directory +/// or if `cargo metadata` invocation fails. +#[inline] +pub fn query_metadata(crate_path: &Path) -> Result { + log::debug!("Running `cargo metadata` on `{}`", crate_path.display()); + let path = &crate_path.canonicalize()?; + let metadata = MetadataCommand::new().current_dir(path).exec()?; + Ok(metadata) +} + +/// An error indicating that querying metadata failed. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum QueryMetadataError { + /// Provided shader crate path is invalid. + #[error("failed to get an absolute path to the crate: {0}")] + InvalidPath(#[from] io::Error), + /// Failed to run `cargo metadata` for provided shader crate. + #[error(transparent)] + CargoMetadata(#[from] cargo_metadata::Error), +} + +/// Extension trait for [`Metadata`]. +pub trait MetadataExt { + /// Search for a package by provided name. + /// + /// # Errors + /// + /// If no package with the specified name was found, returns an error. + fn find_package(&self, name: &str) -> Result<&Package, MissingPackageError>; +} + +impl MetadataExt for Metadata { + #[inline] + fn find_package(&self, name: &str) -> Result<&Package, MissingPackageError> { + let Some(package) = self + .packages + .iter() + .find(|package| package.name.as_str() == name) + else { + let workspace_root = self.workspace_root.clone(); + return Err(MissingPackageError::new(name, workspace_root)); + }; + + log::trace!(" found `{}` version `{}`", package.name, package.version); + Ok(package) + } +} + +/// An error indicating that a package with the specified crate name was not found. +#[derive(Debug, Clone, thiserror::Error)] +#[error("`{crate_name}` not found in `Cargo.toml` at `{workspace_root:?}`")] +pub struct MissingPackageError { + /// The crate name that was not found. + crate_name: String, + /// The workspace root of the [`Metadata`]. + workspace_root: Utf8PathBuf, +} + +impl MissingPackageError { + /// Creates self from the given crate name and workspace root. + fn new(crate_name: impl Into, workspace_root: impl Into) -> Self { + Self { + crate_name: crate_name.into(), + workspace_root: workspace_root.into(), + } + } +} diff --git a/crates/rustc_codegen_spirv-cache/src/spirv_source.rs b/crates/rustc_codegen_spirv-cache/src/spirv_source.rs new file mode 100644 index 0000000..fb64f8e --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/src/spirv_source.rs @@ -0,0 +1,471 @@ +//! Use the shader that we're compiling as the default source for which version of `rust-gpu` to use. +//! +//! We do this by calling `cargo tree` inside the shader's crate to get the defined `spirv-std` +//! version. Then with that we `git checkout` the `rust-gpu` repo that corresponds to that version. +//! From there we can look at the source code to get the required Rust toolchain. + +use core::{ + fmt::{self, Display}, + ops::Range, +}; +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +use cargo_metadata::{ + camino::{Utf8Path, Utf8PathBuf}, + semver::Version, + Package, +}; + +use crate::{ + cache::{cache_dir, CacheDirError}, + metadata::{query_metadata, MetadataExt as _, MissingPackageError, QueryMetadataError}, +}; + +#[expect( + rustdoc::bare_urls, + clippy::doc_markdown, + reason = "The URL should appear literally like this. But Clippy & rustdoc want a markdown clickable link" +)] +#[expect(clippy::exhaustive_enums, reason = "It is expected to be exhaustive")] +/// The source and version of `rust-gpu`. +/// Eg: +/// * From crates.io with version "0.10.0" +/// * From Git with: +/// - a repo of "https://github.com/Rust-GPU/rust-gpu.git" +/// - a revision of "abc213" +/// * a local Path +#[derive(Eq, PartialEq, Clone, Debug)] +pub enum SpirvSource { + /// If the shader specifies a simple version like `spirv-std = "0.9.0"` then the source of + /// `rust-gpu` is the conventional crates.io version. + CratesIO(Version), + /// If the shader specifies a version like: + /// `spirv-std = { git = "https://github.com..." ... }` + /// then the source of `rust-gpu` is `Git`. + Git { + /// URL of the repository + url: String, + /// Revision or "commitsh" + rev: String, + }, + /// If the shader specifies a version like: + /// `spirv-std = { path = "/path/to/rust-gpu" ... }` + /// then the source of `rust-gpu` is `Path`. + Path { + /// File path of rust-gpu repository + rust_gpu_repo_root: Utf8PathBuf, + /// Version of specified rust-gpu repository + version: Version, + }, +} + +impl Display for SpirvSource { + #[expect( + clippy::min_ident_chars, + reason = "It's a core library trait implementation" + )] + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CratesIO(version) => version.fmt(f), + Self::Git { url, rev } => { + // shorten rev to 8 chars, prevents windows compile errors due to too long paths... seriously + if let Some(short_rev) = rev.get(..8) { + write!(f, "{url}+{short_rev}") + } else { + write!(f, "{url}+{rev}") + } + } + Self::Path { + rust_gpu_repo_root, + version, + } => write!(f, "{rust_gpu_repo_root}+{version}"), + } + } +} + +impl SpirvSource { + /// Figures out which source of `rust-gpu` to use. + /// + /// # Errors + /// + /// If unable to determine the source of `rust-gpu`, returns an error. + #[inline] + pub fn new( + shader_crate_path: &Path, + maybe_rust_gpu_source: Option<&str>, + maybe_rust_gpu_version: Option<&str>, + ) -> Result { + match (maybe_rust_gpu_source, maybe_rust_gpu_version) { + (Some(rust_gpu_source), Some(rust_gpu_version)) => Ok(Self::Git { + url: rust_gpu_source.to_owned(), + rev: rust_gpu_version.to_owned(), + }), + (None, Some(rust_gpu_version)) => Ok(Self::CratesIO(Version::parse(rust_gpu_version)?)), + _ => Self::get_rust_gpu_deps_from_shader(shader_crate_path), + } + } + + /// Look into the shader crate to get the source and version of `rust-gpu` it's using. + /// + /// # Errors + /// + /// If unable to determine the source and version of `rust-gpu`, returns an error. + #[inline] + pub fn get_rust_gpu_deps_from_shader( + shader_crate_path: &Path, + ) -> Result { + let crate_metadata = query_metadata(shader_crate_path)?; + let spirv_std_package = crate_metadata.find_package("spirv-std")?; + let spirv_source = Self::parse_spirv_std_source_and_version(spirv_std_package)?; + + log::debug!( + "Parsed `SpirvSource` from crate `{}`: {spirv_source:?}", + shader_crate_path.display(), + ); + Ok(spirv_source) + } + + /// Convert self into a cache directory in which we can build it. + /// + /// It needs to be dynamically created because an end-user might want to swap out the source, + /// maybe using their own fork for example. + /// + /// # Errors + /// + /// Returns an error if there is no cache directory available. + #[inline] + pub fn install_dir(&self) -> Result { + let dir = match self { + Self::Path { + rust_gpu_repo_root, .. + } => rust_gpu_repo_root.as_std_path().to_owned(), + Self::CratesIO { .. } | Self::Git { .. } => { + let dir = to_dirname(self.to_string().as_ref()); + cache_dir()?.join("codegen").join(dir) + } + }; + Ok(dir) + } + + /// Returns `true` if self is a [`Path`](SpirvSource::Path). + #[expect( + clippy::must_use_candidate, + reason = "calculations are cheap, `bool` is `Copy`" + )] + #[inline] + pub const fn is_path(&self) -> bool { + matches!(self, Self::Path { .. }) + } + + /// Parse a string like: + /// `spirv-std v0.9.0 (https://github.com/Rust-GPU/rust-gpu?rev=54f6978c#54f6978c) (*)` + /// Which would return: + /// `SpirvSource::Git("https://github.com/Rust-GPU/rust-gpu", "54f6978c")` + fn parse_spirv_std_source_and_version( + spirv_std_package: &Package, + ) -> Result { + log::trace!("parsing spirv-std source and version from package: '{spirv_std_package:?}'"); + + let result = if let Some(source) = spirv_std_package.source.clone() { + let is_git = source.repr.starts_with("git+"); + let is_crates_io = source.is_crates_io(); + + match (is_git, is_crates_io) { + (true, true) => return Err(ParseSourceVersionError::AmbiguousSource(source)), + (true, false) => { + let parse_git = || { + let link = &source.repr.get(4..)?; + let sharp_index = link.find('#')?; + let url_end = link.find('?').unwrap_or(sharp_index); + let url = link.get(..url_end)?.to_owned(); + let rev = link.get(sharp_index + 1..)?.to_owned(); + Some(Self::Git { url, rev }) + }; + parse_git().ok_or(ParseSourceVersionError::InvalidGitSource(source))? + } + (false, true) => Self::CratesIO(spirv_std_package.version.clone()), + (false, false) => return Err(ParseSourceVersionError::UnknownSource(source)), + } + } else { + let manifest_path = spirv_std_package.manifest_path.as_path(); + let Some(rust_gpu_repo_root) = manifest_path // rust-gpu/crates/spirv-std/Cargo.toml + .parent() // rust-gpu/crates/spirv-std + .and_then(Utf8Path::parent) // rust-gpu/crates + .and_then(Utf8Path::parent) // rust-gpu + .filter(|path| path.is_dir()) + .map(ToOwned::to_owned) + else { + let err = ParseSourceVersionError::InvalidManifestPath(manifest_path.to_owned()); + return Err(err); + }; + let version = spirv_std_package.version.clone(); + Self::Path { + rust_gpu_repo_root, + version, + } + }; + + log::debug!("Parsed `rust-gpu` source and version: {result:?}"); + Ok(result) + } +} + +/// An error indicating that construction of [`SpirvSource`] failed. +#[expect(clippy::module_name_repetitions, reason = "this is intended")] +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum SpirvSourceError { + /// Querying metadata failed. + #[error(transparent)] + QueryMetadata(#[from] QueryMetadataError), + /// The package was missing from the metadata. + #[error(transparent)] + MissingPackage(#[from] MissingPackageError), + /// Parsing the source and version of `spirv-std` crate from the package failed. + #[error(transparent)] + ParseSourceVersion(#[from] ParseSourceVersionError), + /// Parsed version of the crate is not valid. + #[error("invalid version: {0}")] + InvalidVersion(#[from] cargo_metadata::semver::Error), +} + +/// An error indicating that parsing the source and version of `rust-gpu` from the package failed. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ParseSourceVersionError { + /// The source was found to be ambiguous. + #[error("both git and crates.io were found at source {0}")] + AmbiguousSource(cargo_metadata::Source), + /// The source was found to be of git format but it is not valid. + #[error("invalid git format of source {0}")] + InvalidGitSource(cargo_metadata::Source), + /// The source has unknown / unsupported format. + #[error("unknown format of source {0}")] + UnknownSource(cargo_metadata::Source), + /// Manifest path of the package is not valid. + #[error("invalid manifest path {0}")] + InvalidManifestPath(Utf8PathBuf), +} + +/// Parse the `rust-toolchain.toml` in the working tree of the checked-out version of the `rust-gpu` repo. +/// +/// # Errors +/// +/// Returns an error if the package is at the root of the filesystem, +/// build script does not exist, or there is no definition of `channel` in it. +#[inline] +pub fn rust_gpu_toolchain_channel( + rustc_codegen_spirv: &Package, +) -> Result { + let path = rustc_codegen_spirv + .manifest_path + .parent() + .ok_or(RustGpuToolchainChannelError::ManifestAtRoot)?; + let build_script = path.join("build.rs"); + + log::debug!("Parsing `build.rs` at {build_script:?} for the used toolchain"); + let contents = match fs::read_to_string(&build_script) { + Ok(contents) => contents, + Err(source) => { + let err = RustGpuToolchainChannelError::InvalidBuildScript { + source, + build_script, + }; + return Err(err); + } + }; + + let channel_start = "channel = \""; + let Some(channel_line) = contents + .lines() + .find(|line| line.starts_with(channel_start)) + else { + let err = RustGpuToolchainChannelError::ChannelStartNotFound { + channel_start: channel_start.to_owned(), + build_script, + }; + return Err(err); + }; + let start = channel_start.len(); + + let channel_end = "\""; + #[expect(clippy::string_slice, reason = "line starts with `channel_start`")] + let Some(end) = channel_line[start..] + .find(channel_end) + .map(|end| end + start) + else { + let err = RustGpuToolchainChannelError::ChannelEndNotFound { + channel_end: channel_end.to_owned(), + channel_line: channel_line.to_owned(), + build_script, + }; + return Err(err); + }; + + let range = start..end; + let Some(channel) = channel_line.get(range.clone()) else { + let err = RustGpuToolchainChannelError::InvalidChannelSlice { + range, + channel_line: channel_line.to_owned(), + build_script, + }; + return Err(err); + }; + Ok(channel.to_owned()) +} + +/// An error indicating that getting the channel of a Rust toolchain from the package failed. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum RustGpuToolchainChannelError { + /// Manifest of the package is located at the root of the file system + /// and cannot have a parent. + #[error("package manifest was located at root")] + ManifestAtRoot, + /// Build script file is not valid or does not exist. + #[error("invalid build script {build_script}: {source}")] + InvalidBuildScript { + /// Source of the error. + source: io::Error, + /// Path to the build script file. + build_script: Utf8PathBuf, + }, + /// There is no line starting with `channel_start` + /// in the build script file contents. + #[error("`{channel_start}` line in {build_script:?} not found")] + ChannelStartNotFound { + /// Start of the channel line. + channel_start: String, + /// Path to the build script file. + build_script: Utf8PathBuf, + }, + /// Channel line does not contain `channel_end` + /// in the build script file contents. + #[error("ending `{channel_end}` of line \"{channel_line}\" in {build_script:?} not found")] + ChannelEndNotFound { + /// End of the channel line. + channel_end: String, + /// The line containing the channel information. + channel_line: String, + /// Path to the build script file. + build_script: Utf8PathBuf, + }, + /// The range to slice the channel line is not valid. + #[error("cannot slice line \"{channel_line}\" of {build_script:?} by range {range:?}")] + InvalidChannelSlice { + /// The invalid range. + range: Range, + /// The line containing the channel information. + channel_line: String, + /// Path to the build script file. + build_script: Utf8PathBuf, + }, +} + +/// Returns a string suitable to use as a directory. +/// +/// Created from the spirv-builder source dep and the rustc channel. +fn to_dirname(text: &str) -> String { + text.replace( + [std::path::MAIN_SEPARATOR, '\\', '/', '.', ':', '@', '='], + "_", + ) + .split(['{', '}', ' ', '\n', '"', '\'']) + .collect::>() + .concat() +} + +#[cfg(test)] +mod test { + use super::*; + use cargo_metadata::{PackageBuilder, PackageId, Source}; + use cargo_util_schemas::manifest::PackageName; + + pub fn shader_crate_template_path() -> std::path::PathBuf { + let project_base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + project_base.join("../shader-crate-template") + } + + #[test_log::test] + fn parsing_spirv_std_dep_for_shader_template() { + let shader_template_path = shader_crate_template_path(); + let source = SpirvSource::get_rust_gpu_deps_from_shader(&shader_template_path).unwrap(); + assert_eq!( + source, + SpirvSource::Git { + url: "https://github.com/Rust-GPU/rust-gpu".to_owned(), + rev: "86fc48032c4cd4afb74f1d81ae859711d20386a1".to_owned() + } + ); + } + + #[test_log::test] + fn path_sanity() { + let path = std::path::PathBuf::from("./"); + assert!(path.is_relative()); + } + + #[test_log::test] + fn cached_checkout_dir_sanity() { + let shader_template_path = shader_crate_template_path(); + let source = SpirvSource::get_rust_gpu_deps_from_shader(&shader_template_path).unwrap(); + let dir = source.install_dir().unwrap(); + let name = dir + .file_name() + .unwrap() + .to_str() + .map(std::string::ToString::to_string) + .unwrap(); + assert_eq!("https___github_com_Rust-GPU_rust-gpu+86fc4803", &name); + } + + #[test_log::test] + fn parse_git_with_rev() { + let source = parse_git( + "git+https://github.com/Rust-GPU/rust-gpu?rev=86fc48032c4cd4afb74f1d81ae859711d20386a1#86fc4803", + ); + assert_eq!( + source, + SpirvSource::Git { + url: "https://github.com/Rust-GPU/rust-gpu".to_owned(), + rev: "86fc4803".to_owned(), + } + ); + } + + #[test_log::test] + fn parse_git_no_question_mark() { + // taken directly from Graphite + let source = parse_git( + "git+https://github.com/Rust-GPU/rust-gpu.git#6e2c84d4fe64e32df4c060c5a7f3e35a32e45421", + ); + assert_eq!( + source, + SpirvSource::Git { + url: "https://github.com/Rust-GPU/rust-gpu.git".to_owned(), + rev: "6e2c84d4fe64e32df4c060c5a7f3e35a32e45421".to_owned(), + } + ); + } + + fn parse_git(source: &str) -> SpirvSource { + let package = PackageBuilder::new( + PackageName::new("spirv-std".to_owned()).unwrap(), + Version::new(0, 9, 0), + PackageId { + repr: String::new(), + }, + "", + ) + .source(Some(Source { + repr: source.to_owned(), + })) + .build() + .unwrap(); + SpirvSource::parse_spirv_std_source_and_version(&package).unwrap() + } +} diff --git a/crates/rustc_codegen_spirv-cache/src/target_specs.rs b/crates/rustc_codegen_spirv-cache/src/target_specs.rs new file mode 100644 index 0000000..128733c --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/src/target_specs.rs @@ -0,0 +1,208 @@ +//! This module deals with target specs, which are json metadata files that need to be passed to +//! rustc to add foreign targets such as `spirv_unknown_vulkan1.2`. +//! +//! There are 4 version ranges of `rustc_codegen_spirv` and they all need different handling of +//! their target specs: +//! * "ancient" versions such as 0.9.0 or earlier do not need target specs, just passing the target +//! string (`spirv-unknown-vulkan1.2`) directly is sufficient. We still prep target-specs for them +//! like the "legacy" variant below, spirv-builder will just [ignore] it. +//! * "legacy" versions require target specs to compile, which is a requirement introduced by some +//! rustc version. Back then it was decided that cargo gpu would ship them, as they'd probably +//! never change, right? So now we're stuck with having to ship these "legacy" target specs with +//! cargo gpu *forever*. These are [`TARGET_SPECS`] from a **fixed** version +//! of [`rustc_codegen_spirv-target-specs`] which must **never** update. +//! * As of [PR 256], `rustc_codegen_spirv` now has a direct dependency on [`rustc_codegen_spirv-target-specs`], +//! allowing cargo gpu to pull the required target specs directly from that dependency. +//! At this point, the target specs are still the same as the legacy target specs. +//! * The [edition 2024 PR] must update the target specs to comply with newly added validation within rustc. +//! This is why the new system was implemented, so we can support both old and new target specs +//! without having to worry which version of cargo gpu you are using. +//! It'll "just work". +//! +//! [ignore]: https://github.com/Rust-GPU/rust-gpu/blob/369122e1703c0c32d3d46f46fa11ccf12667af03/crates/spirv-builder/src/lib.rs#L987 +//! [`TARGET_SPECS`]: legacy_target_specs::TARGET_SPECS +//! [`rustc_codegen_spirv-target-specs`]: legacy_target_specs +//! [PR 256]: https://github.com/Rust-GPU/rust-gpu/pull/256 +//! [edition 2024 PR]: https://github.com/Rust-GPU/rust-gpu/pull/249 + +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +use cargo_metadata::Metadata; + +use crate::{ + cache::{cache_dir, CacheDirError}, + metadata::MetadataExt as _, + spirv_source::SpirvSource, +}; + +/// Extract legacy target specs from our executable into the directory by given path. +/// +/// # Errors +/// +/// Returns an error if the directory cannot be created +/// or if any target spec json cannot be written into a file. +#[inline] +#[expect(clippy::module_name_repetitions, reason = "this is intentional")] +pub fn write_legacy_target_specs( + target_spec_dir: &Path, +) -> Result<(), WriteLegacyTargetSpecsError> { + if let Err(source) = fs::create_dir_all(target_spec_dir) { + let path = target_spec_dir.to_path_buf(); + return Err(WriteLegacyTargetSpecsError::CreateDir { path, source }); + } + + for (filename, contents) in legacy_target_specs::TARGET_SPECS { + let path = target_spec_dir.join(filename); + if let Err(source) = fs::write(&path, contents.as_bytes()) { + return Err(WriteLegacyTargetSpecsError::WriteFile { path, source }); + } + } + Ok(()) +} + +/// An error indicating a failure to write target specs files. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum WriteLegacyTargetSpecsError { + /// Failed to create the target specs directory. + #[error("failed to create target specs directory at {path}: {source}")] + CreateDir { + /// Path of the target specs directory. + path: PathBuf, + /// Source of the error. + source: io::Error, + }, + /// Failed to write a target spec file. + #[error("failed to write target spec file at {path}: {source}")] + WriteFile { + /// Path of the target spec file. + path: PathBuf, + /// Source of the error. + source: io::Error, + }, +} + +/// Copy spec files from one dir to another, assuming no subdirectories. +fn copy_spec_files(src: &Path, dst: &Path) -> io::Result<()> { + fs::create_dir_all(dst)?; + let dir = fs::read_dir(src)?; + for dir_entry in dir { + let file = dir_entry?; + let file_path = file.path(); + if file_path.is_file() { + fs::copy(file_path, dst.join(file.file_name()))?; + } + } + Ok(()) +} + +/// Computes the `target-specs` directory to use and updates the target spec files, if enabled. +/// +/// # Errors +/// +/// Returns an error if: +/// * cache directory is not available, +/// * legacy target specs dependency is invalid, +/// * target specs files cannot be copied, +/// * or legacy target specs cannot be written. +#[inline] +pub fn update_target_specs_files( + source: &SpirvSource, + metadata: &Metadata, + update_files: bool, +) -> Result { + log::info!( + "target-specs: Resolving target specs `{}`", + if update_files { + "and update them" + } else { + "without updating" + } + ); + + let mut target_specs_dst = source.install_dir()?.join("target-specs"); + if let Ok(target_specs) = metadata.find_package("rustc_codegen_spirv-target-specs") { + log::info!( + "target-specs: found crate `rustc_codegen_spirv-target-specs` with manifest at `{}`", + target_specs.manifest_path + ); + + let target_specs_src = target_specs + .manifest_path + .as_std_path() + .parent() + .and_then(|root| { + let src = root.join("target-specs"); + src.is_dir().then_some(src) + }) + .ok_or(UpdateTargetSpecsFilesError::InvalidLegacy)?; + log::info!( + "target-specs: found `rustc_codegen_spirv-target-specs` with `target-specs` directory `{}`", + target_specs_dst.display() + ); + + if source.is_path() { + // skip copy + log::info!( + "target-specs resolution: source is local path, use target-specs directly from `{}`", + target_specs_dst.display() + ); + target_specs_dst = target_specs_src; + } else { + // copy over the target-specs + log::info!( + "target-specs resolution: copying target-specs from `{}`{}", + target_specs_dst.display(), + if update_files { "" } else { " was skipped" } + ); + if update_files { + copy_spec_files(&target_specs_src, &target_specs_dst) + .map_err(UpdateTargetSpecsFilesError::CopySpecFiles)?; + } + } + } else { + // use legacy target specs bundled with cargo gpu + if source.is_path() { + // This is a stupid situation: + // * We can't be certain that there are `target-specs` in the local checkout (there may be some in `spirv-builder`) + // * We can't dump our legacy ones into the `install_dir`, as that would modify the local rust-gpu checkout + // -> do what the old cargo gpu did, one global dir for all target specs + // and hope parallel runs don't shred each other + target_specs_dst = cache_dir()?.join("legacy-target-specs-for-local-checkout"); + } + log::info!( + "target-specs resolution: legacy target specs in directory `{}`", + target_specs_dst.display() + ); + if update_files { + log::info!( + "target-specs: writing legacy target specs into `{}`", + target_specs_dst.display() + ); + write_legacy_target_specs(&target_specs_dst)?; + } + } + + Ok(target_specs_dst) +} + +/// An error indicating a failure to update target specs files. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum UpdateTargetSpecsFilesError { + /// There is no cache directory available. + #[error(transparent)] + CacheDir(#[from] CacheDirError), + /// Legacy target specs dependency is invalid. + #[error("could not find `target-specs` directory within `rustc_codegen_spirv-target-specs` dependency")] + InvalidLegacy, + /// Could not copy target specs files. + #[error("could not copy target specs files: {0}")] + CopySpecFiles(#[source] io::Error), + /// Could not write legacy target specs. + #[error("could not write legacy target specs ({0})")] + WriteLegacy(#[from] WriteLegacyTargetSpecsError), +} diff --git a/crates/rustc_codegen_spirv-cache/src/toolchain.rs b/crates/rustc_codegen_spirv-cache/src/toolchain.rs new file mode 100644 index 0000000..2f03200 --- /dev/null +++ b/crates/rustc_codegen_spirv-cache/src/toolchain.rs @@ -0,0 +1,278 @@ +//! This module deals with an installation of Rust toolchain required by `rust-gpu` +//! (and all of its [required components](REQUIRED_TOOLCHAIN_COMPONENTS)). + +#![allow( + clippy::multiple_inherent_impl, + reason = "should be separate, see FIXME of type aliases below" +)] + +use std::process::{Command, Stdio}; + +use crate::command::{execute_command, CommandExecError}; + +/// Allows to halt the installation process of toolchain or its [required components](REQUIRED_TOOLCHAIN_COMPONENTS). +#[derive(Debug, Clone, Copy)] +#[expect(clippy::exhaustive_structs, reason = "intended to be exhaustive")] +pub struct HaltToolchainInstallation { + /// Closure which is called to halt the installation process of toolchain. + pub on_toolchain_install: T, + /// Closure which is called to halt the installation process of required toolchain components. + pub on_components_install: C, +} + +/// Type of [`on_toolchain_install`](HaltToolchainInstallation::on_toolchain_install) which does nothing. +// FIXME: replace with `impl FnOnce` once it's stabilized +pub type NoopOnToolchainInstall = fn(&str) -> Result<(), CommandExecError>; + +/// Type of [`on_components_install`](HaltToolchainInstallation::on_components_install) which does nothing. +// FIXME: replace with `impl FnOnce` once it's stabilized +pub type NoopOnComponentsInstall = fn(&str) -> Result<(), CommandExecError>; + +/// Type of [`HaltToolchainInstallation`] which does nothing. +pub type NoopHaltToolchainInstallation = + HaltToolchainInstallation; + +impl NoopHaltToolchainInstallation { + /// Do not halt the installation process of toolchain or its [required components](REQUIRED_TOOLCHAIN_COMPONENTS). + /// + /// Calling either [`on_toolchain_install`] or [`on_components_install`] + /// returns [`Ok`] without any side effects. + /// + /// [`on_toolchain_install`]: HaltToolchainInstallation::on_toolchain_install + /// [`on_components_install`]: HaltToolchainInstallation::on_components_install + #[inline] + #[expect(clippy::must_use_candidate, reason = "contains no state")] + pub fn noop() -> Self { + Self { + on_toolchain_install: |_: &str| Ok(()), + on_components_install: |_: &str| Ok(()), + } + } +} + +/// Configuration for `stdout` and `stderr` of commands executed by this [module](self). +#[derive(Debug, Clone, Copy)] +#[expect(clippy::exhaustive_structs, reason = "intended to be exhaustive")] +pub struct StdioCfg { + /// Configuration for [`Command::stdout()`]. + pub stdout: O, + /// Configuration for [`Command::stderr()`]. + pub stderr: E, +} + +/// Type of [`stdout`](StdioCfg::stdout) which returns [`Stdio::null()`]. +// FIXME: replace with `impl FnOnce` once it's stabilized +pub type NullStdout = fn() -> Stdio; + +/// Type of [`stderr`](StdioCfg::stderr) which returns [`Stdio::null()`]. +// FIXME: replace with `impl FnOnce` once it's stabilized +pub type NullStderr = fn() -> Stdio; + +/// Type of [`StdioCfg`] which uses [`Stdio::null()`] +/// for both [`stdout`](StdioCfg::stdout) and [`stderr`](StdioCfg::stderr). +pub type NullStdioCfg = StdioCfg; + +impl NullStdioCfg { + /// Configures both [`stdout`](StdioCfg::stdout) and [`stderr`](StdioCfg::stderr) + /// to use [`Stdio::null()`]. + #[inline] + #[expect(clippy::must_use_candidate, reason = "contains no state")] + pub fn null() -> Self { + Self { + stdout: Stdio::null, + stderr: Stdio::null, + } + } +} + +/// Type of [`stdout`](StdioCfg::stdout) which returns [`Stdio::inherit()`]. +// FIXME: replace with `impl FnOnce` once it's stabilized +pub type InheritStdout = fn() -> Stdio; + +/// Type of [`stderr`](StdioCfg::stderr) which returns [`Stdio::inherit()`]. +// FIXME: replace with `impl FnOnce` once it's stabilized +pub type InheritStderr = fn() -> Stdio; + +/// Type of [`StdioCfg`] which uses [`Stdio::inherit()`] +/// for both [`stdout`](StdioCfg::stdout) and [`stderr`](StdioCfg::stderr). +pub type InheritStdoutCfg = StdioCfg; + +impl InheritStdoutCfg { + /// Configures both [`stdout`](StdioCfg::stdout) and [`stderr`](StdioCfg::stderr) + /// to use [`Stdio::inherit()`]. + #[inline] + #[expect(clippy::must_use_candidate, reason = "contains no state")] + pub fn inherit() -> Self { + Self { + stdout: Stdio::inherit, + stderr: Stdio::inherit, + } + } +} + +/// Uses `rustup` to install the toolchain and all the [required components](REQUIRED_TOOLCHAIN_COMPONENTS), +/// if not already installed. +/// +/// Pretty much runs: +/// +/// ```text +/// rustup toolchain add nightly-2024-04-24 +/// rustup component add --toolchain nightly-2024-04-24 rust-src rustc-dev llvm-tools +/// ``` +/// +/// where `nightly-2024-04-24` is an example of a toolchain +/// provided as an argument to this function. +/// +/// The second parameter allows you to halt the installation process +/// of toolchain or its required components. +/// +/// # Errors +/// +/// Returns an error if any error occurs while using `rustup` +/// or the installation process was halted. +#[inline] +pub fn ensure_toolchain_installation( + channel: &str, + halt: HaltToolchainInstallation, + cfg: StdioCfg, +) -> Result<(), R> +where + R: From, + T: FnOnce(&str) -> Result<(), R>, + C: FnOnce(&str) -> Result<(), R>, + O: FnMut() -> Stdio, + E: FnMut() -> Stdio, +{ + let HaltToolchainInstallation { + on_toolchain_install, + on_components_install, + } = halt; + let StdioCfg { + mut stdout, + mut stderr, + } = cfg; + + if is_toolchain_installed(channel)? { + log::debug!("toolchain {channel} is already installed"); + } else { + log::debug!("toolchain {channel} is not installed yet"); + on_toolchain_install(channel)?; + install_toolchain(channel, stdout(), stderr())?; + } + + if all_required_toolchain_components_installed(channel)? { + log::debug!("all required components of toolchain {channel} are installed"); + } else { + log::debug!("not all required components of toolchain {channel} are installed yet"); + on_components_install(channel)?; + install_required_toolchain_components(channel, stdout(), stderr())?; + } + + Ok(()) +} + +/// Checks if the given toolchain is installed using `rustup`. +/// +/// # Errors +/// +/// Returns an error if any error occurs while using `rustup`. +#[inline] +pub fn is_toolchain_installed(channel: &str) -> Result { + let mut command = Command::new("rustup"); + command.args(["toolchain", "list"]); + let output = execute_command(command)?; + + let toolchain_list = String::from_utf8_lossy(&output.stdout); + let installed = toolchain_list + .split_whitespace() + .any(|toolchain| toolchain.starts_with(channel)); + Ok(installed) +} + +/// Installs the given toolchain using `rustup`. +/// +/// # Errors +/// +/// Returns an error if any error occurs while using `rustup`. +#[inline] +#[expect(clippy::module_name_repetitions, reason = "this is intended")] +pub fn install_toolchain( + channel: &str, + stdout_cfg: O, + stderr_cfg: E, +) -> Result<(), CommandExecError> +where + O: Into, + E: Into, +{ + let mut command = Command::new("rustup"); + command + .args(["toolchain", "add"]) + .arg(channel) + .stdout(stdout_cfg) + .stderr(stderr_cfg); + let _output = execute_command(command)?; + + Ok(()) +} + +/// Components which are required to be installed for a toolchain to be usable with `rust-gpu`. +pub const REQUIRED_TOOLCHAIN_COMPONENTS: [&str; 3] = ["rust-src", "rustc-dev", "llvm-tools"]; + +/// Checks if all the [required components](REQUIRED_TOOLCHAIN_COMPONENTS) +/// of the given toolchain are installed using `rustup`. +/// +/// # Errors +/// +/// Returns an error if any error occurs while using `rustup`. +#[inline] +pub fn all_required_toolchain_components_installed( + channel: &str, +) -> Result { + let mut command = Command::new("rustup"); + command + .args(["component", "list", "--toolchain"]) + .arg(channel); + let output = execute_command(command)?; + + let component_list = String::from_utf8_lossy(&output.stdout); + let component_list_lines = component_list.lines().collect::>(); + let installed = REQUIRED_TOOLCHAIN_COMPONENTS.iter().all(|component| { + component_list_lines + .iter() + .any(|maybe_installed_component| { + let is_component = maybe_installed_component.starts_with(component); + let is_installed = maybe_installed_component.ends_with("(installed)"); + is_component && is_installed + }) + }); + Ok(installed) +} + +/// Installs all the [required components](REQUIRED_TOOLCHAIN_COMPONENTS) +/// for the given toolchain using `rustup`. +/// +/// # Errors +/// +/// Returns an error if any error occurs while using `rustup`. +#[inline] +pub fn install_required_toolchain_components( + channel: &str, + stdout_cfg: O, + stderr_cfg: E, +) -> Result<(), CommandExecError> +where + O: Into, + E: Into, +{ + let mut command = Command::new("rustup"); + command + .args(["component", "add", "--toolchain"]) + .arg(channel) + .args(REQUIRED_TOOLCHAIN_COMPONENTS) + .stdout(stdout_cfg) + .stderr(stderr_cfg); + let _output = execute_command(command)?; + + Ok(()) +}