Add Skip and add a vulkano test to try to debug windows

Author: LegNeato

Cargo.lock

@@ -99,12 +99,18 @@ The library provides helper types for common test patterns:
 - `WgpuComputeTestMultiBuffer` - Multi-buffer compute shader test with input/output
   separation
 - `WgpuComputeTestPushConstant` - Compute shader test with push constants support
+- `Skip` - Marks a test variant as skipped with a reason
 
 **Shader source types:**
 
 - `RustComputeShader` - Compiles the current crate as a Rust GPU shader
 - `WgslComputeShader` - Loads WGSL shader from file (shader.wgsl or compute.wgsl)
 
+**Backend types:**
+
+- `WgpuBackend` - Default wgpu-based compute backend
+- `VulkanoBackend` - Vulkano-based compute backend (useful for testing different GPU drivers)
+
 For examples, see:
 
 - [`tests/lang/core/ops/math_ops/`](tests/lang/core/ops/math_ops/) - Multi-buffer test
@@ -205,6 +211,40 @@ The harness automatically writes human-readable `.txt` files alongside binary ou
 For floating-point data (F32/F64), these show the array values in decimal format. For
 raw/integer data, these show the values as hex bytes or integers
 
+## Skipping Tests on Specific Platforms
+
+Sometimes a test variant needs to be skipped on certain platforms (e.g., due to driver 
+issues or platform limitations). The difftest framework provides a clean way to handle
+this using the `Skip` scaffolding type:
+
+```rust
+use difftest::scaffold::Skip;
+
+fn main() {
+    let config = Config::from_path(std::env::args().nth(1).unwrap()).unwrap();
+    
+    // Skip on macOS due to platform-specific issues
+    #[cfg(target_os = "macos")]
+    {
+        let skip = Skip::new("This test is not supported on macOS");
+        skip.run_test(&config).unwrap();
+        return;
+    }
+    
+    // Run the actual test on other platforms
+    #[cfg(not(target_os = "macos"))]
+    {
+        // ... normal test implementation ...
+    }
+}
+```
+
+When a test is skipped:
+- The skip reason is recorded in the test metadata
+- The test runner logs the skip reason
+- The test doesn't contribute to the output comparison
+- If all variants are skipped, the test fails with an error
+
 ## Harness logs
 
 If you suspect a bug in the test harness, you can view detailed test harness logs:

tests/difftests/README.md

@@ -336,6 +336,12 @@ impl Runner {
                 if !metadata_content.trim().is_empty() {
                     match serde_json::from_str::<TestMetadata>(&metadata_content) {
                         Ok(metadata) => {
+                            // Check if test was skipped
+                            if let Some(skip_reason) = &metadata.skipped {
+                                info!("Package '{}' was skipped: {}", pkg_name, skip_reason);
+                                continue;
+                            }
+
                             if let Some(meta_epsilon) = metadata.epsilon {
                                 epsilon = match epsilon {
                                     Some(e) => Some(e.max(meta_epsilon)),
@@ -390,6 +396,12 @@ impl Runner {
             });
         }
 
+        // Check if we have any valid outputs
+        if pkg_outputs.is_empty() {
+            error!("All packages were skipped. At least one package must produce output.");
+            return Err(RunnerError::EmptyOutput);
+        }
+
         if pkg_outputs.iter().all(|po| po.output.is_empty()) {
             error!("All packages produced empty output.");
             return Err(RunnerError::EmptyOutput);

tests/difftests/bin/src/runner.rs

@@ -20,6 +20,8 @@ spirv-builder.workspace = true
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 wgpu = { version = "25.0.2", features = ["spirv", "vulkan-portability", "static-dxc"] }
+vulkano = { version = "0.35.1", default-features = false }
+naga = { version = "25.0.1", features = ["glsl-in", "spv-out"] }
 tempfile = "3.5"
 futures = "0.3.31"
 bytemuck = "1.21.0"

tests/difftests/lib/Cargo.toml

@@ -27,6 +27,10 @@ pub struct TestMetadata {
     /// (F32, F64, etc.) to enable epsilon-based comparison for numeric types.
     #[serde(default)]
     pub output_type: OutputType,
+
+    /// If present, indicates this test was skipped with the given reason
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub skipped: Option<String>,
 }
 
 /// Specifies how test output data should be interpreted for comparison

tests/difftests/lib/src/config.rs

@@ -0,0 +1,84 @@
+use crate::config::Config;
+use anyhow::Result;
+
+/// Configuration for a GPU buffer
+#[derive(Clone)]
+pub struct BufferConfig {
+    pub size: u64,
+    pub usage: BufferUsage,
+    pub initial_data: Option<Vec<u8>>,
+}
+
+/// Buffer usage type
+#[derive(Clone, Copy, PartialEq)]
+pub enum BufferUsage {
+    Storage,
+    StorageReadOnly,
+    Uniform,
+}
+
+/// A generic trait for compute backends
+pub trait ComputeBackend: Sized {
+    /// Initialize the backend
+    fn init() -> Result<Self>;
+
+    /// Create and run a compute shader with multiple buffers
+    fn run_compute(
+        &self,
+        spirv_bytes: &[u8],
+        entry_point: &str,
+        dispatch: [u32; 3],
+        buffers: Vec<BufferConfig>,
+    ) -> Result<Vec<Vec<u8>>>;
+}
+
+/// A compute test that can run on any backend
+pub struct ComputeTest<B: ComputeBackend> {
+    backend: B,
+    spirv_bytes: Vec<u8>,
+    entry_point: String,
+    dispatch: [u32; 3],
+    buffers: Vec<BufferConfig>,
+}
+
+impl<B: ComputeBackend> ComputeTest<B> {
+    pub fn new(
+        spirv_bytes: Vec<u8>,
+        entry_point: String,
+        dispatch: [u32; 3],
+        buffers: Vec<BufferConfig>,
+    ) -> Result<Self> {
+        Ok(Self {
+            backend: B::init()?,
+            spirv_bytes,
+            entry_point,
+            dispatch,
+            buffers,
+        })
+    }
+
+    pub fn run(self) -> Result<Vec<Vec<u8>>> {
+        self.backend.run_compute(
+            &self.spirv_bytes,
+            &self.entry_point,
+            self.dispatch,
+            self.buffers,
+        )
+    }
+
+    pub fn run_test(self, config: &Config) -> Result<()> {
+        let buffers = self.buffers.clone();
+        let outputs = self.run()?;
+        // Write the first storage buffer output to the file
+        for (output, buffer_config) in outputs.iter().zip(&buffers) {
+            if matches!(buffer_config.usage, BufferUsage::Storage) && !output.is_empty() {
+                use std::fs::File;
+                use std::io::Write;
+                let mut f = File::create(&config.output_path)?;
+                f.write_all(output)?;
+                return Ok(());
+            }
+        }
+        anyhow::bail!("No storage buffer output found")
+    }
+}

tests/difftests/lib/src/scaffold/compute/backend.rs

@@ -1,5 +1,10 @@
+mod backend;
+mod vulkano;
 mod wgpu;
+
+pub use backend::{BufferConfig, BufferUsage, ComputeBackend, ComputeTest};
+pub use vulkano::VulkanoBackend;
 pub use wgpu::{
-    BufferConfig, BufferUsage, RustComputeShader, WgpuComputeTest, WgpuComputeTestMultiBuffer,
+    RustComputeShader, WgpuBackend, WgpuComputeTest, WgpuComputeTestMultiBuffer,
     WgpuComputeTestPushConstants, WgslComputeShader,
 };