Add an xtask to run the CTS (#7719)

Author: andyleiserson

Cargo.toml

@@ -190,7 +190,7 @@ unicode-ident = "1.0.5"
 walkdir = "2.3"
 winit = { version = "0.29", features = ["android-native-activity"] }
 which = "7"
-xshell = "0.2"
+xshell = "0.2.2"
 
 # Metal dependencies
 metal = "0.32"

README.md

@@ -216,26 +216,30 @@ WebGPU includes a Conformance Test Suite to validate that implementations are wo
 
 To have GitHub run the CTS against a pull request, you can add the `PR: run CTS` label to the PR.
 
-To run the CTS locally, first, you need to check it out:
+To run the CTS locally, run:
 
 ```
-# In the root of your wgpu tree:
-git clone https://github.com/gpuweb/cts.git
-cd cts
-# works in bash and powershell
-git checkout $(cat ../cts_runner/revision.txt)
+cargo xtask cts
 ```
 
-To run a given set of tests:
+This will clone the CTS into the `cts` directory, check out the
+[pinned version](./cts_runner/revision.txt), and run the
+[default list of enabled tests](./cts_runner/test.lst).
+
+You can also specify a test selector on the command line:
 
 ```
-# Must be inside the `cts` folder we just checked out, else this will fail
-cargo run --manifest-path ../Cargo.toml -p cts_runner --bin cts_runner -- ./tools/run_deno --verbose "<test string>"
+cargo xtask cts 'webgpu:api,operation,command_buffer,basic:*'
 ```
 
-To find the full list of tests, go to the [online cts viewer](https://gpuweb.github.io/cts/standalone/?runnow=0&worker=0&debug=0&q=webgpu:*).
+Or supply your own test list in a file:
+
+```
+cargo xtask cts -f your_tests.lst
+```
 
-The list of currently enabled CTS tests can be found [here](./cts_runner/test.lst).
+To find the full list of tests, go to the
+[web version of the CTS](https://gpuweb.github.io/cts/standalone/?runnow=0&worker=0&debug=0&q=webgpu:*).
 
 ## Tracking the WebGPU and WGSL draft specifications
 

xtask/src/cts.rs

@@ -0,0 +1,87 @@
+use anyhow::Context;
+use pico_args::Arguments;
+use std::ffi::OsString;
+use xshell::Shell;
+
+/// Path within the repository where the CTS will be checked out.
+const CTS_CHECKOUT_PATH: &str = "cts";
+
+/// Path within the repository to a file containing the git revision of the CTS to check out.
+const CTS_REVISION_PATH: &str = "cts_runner/revision.txt";
+
+/// URL of the CTS git repository.
+const CTS_GIT_URL: &str = "https://github.com/gpuweb/cts.git";
+
+/// Path to default CTS test list.
+const CTS_DEFAULT_TEST_LIST: &str = "cts_runner/test.lst";
+
+pub fn run_cts(shell: Shell, mut args: Arguments) -> anyhow::Result<()> {
+    let mut list_files = Vec::<OsString>::new();
+
+    while let Some(file) = args.opt_value_from_str("-f")? {
+        list_files.push(file);
+    }
+
+    let mut tests = args.finish();
+
+    if tests.is_empty() && list_files.is_empty() {
+        log::info!("Reading default test list from {CTS_DEFAULT_TEST_LIST}");
+        list_files.push(OsString::from(CTS_DEFAULT_TEST_LIST));
+    }
+
+    for file in list_files {
+        tests.extend(shell.read_file(file)?.lines().filter_map(|line| {
+            let trimmed = line.trim();
+            let is_comment = trimmed.starts_with("//") || trimmed.starts_with("#");
+            (!is_comment).then(|| OsString::from(trimmed))
+        }))
+    }
+
+    let cts_revision = shell
+        .read_file(CTS_REVISION_PATH)
+        .context(format!(
+            "Failed to read CTS git SHA from {CTS_REVISION_PATH}"
+        ))?
+        .trim()
+        .to_string();
+
+    if !shell.path_exists(CTS_CHECKOUT_PATH) {
+        log::info!("Cloning CTS");
+        shell
+            .cmd("git")
+            .args(["clone", CTS_GIT_URL, CTS_CHECKOUT_PATH])
+            .quiet()
+            .run()
+            .context("Failed to clone CTS")?;
+    }
+
+    shell.change_dir(CTS_CHECKOUT_PATH);
+
+    log::info!("Checking out CTS");
+    shell
+        .cmd("git")
+        .args(["checkout", "--quiet", &cts_revision])
+        .quiet()
+        .run()
+        .context("Failed to check out CTS")?;
+
+    log::info!("Running CTS");
+    for test in &tests {
+        shell
+            .cmd("cargo")
+            .args(["run"])
+            .args(["--manifest-path", "../Cargo.toml"])
+            .args(["-p", "cts_runner"])
+            .args(["--bin", "cts_runner"])
+            .args(["--", "./tools/run_deno", "--verbose"])
+            .args([test])
+            .run()
+            .context("CTS failed")?;
+    }
+
+    if tests.len() > 1 {
+        log::info!("Summary reflects only tests from the last selector, not the entire run.");
+    }
+
+    Ok(())
+}

xtask/src/main.rs

@@ -6,6 +6,7 @@ use std::process::ExitCode;
 use anyhow::Context;
 use pico_args::Arguments;
 
+mod cts;
 mod run_wasm;
 mod test;
 mod util;
@@ -15,6 +16,9 @@ const HELP: &str = "\
 Usage: xtask <COMMAND>
 
 Commands:
+  cts [<test selector> | -f <test list file>]...
+    Check out, build, and run CTS tests
+
   run-wasm
     Build and run web examples
 
@@ -76,6 +80,7 @@ fn main() -> anyhow::Result<ExitCode> {
     shell.change_dir(String::from(env!("CARGO_MANIFEST_DIR")) + "/..");
 
     match subcommand.as_deref() {
+        Some("cts") => cts::run_cts(shell, args)?,
         Some("run-wasm") => run_wasm::run_wasm(shell, args)?,
         Some("test") => test::run_tests(shell, args)?,
         Some("vendor-web-sys") => vendor_web_sys::run_vendor_web_sys(shell, args)?,