Fix running all tests with cargo nextest (#420)

Author: jayvdb

.config/nextest.toml

@@ -0,0 +1,30 @@
+[profile.default]
+# Default filter runs all tests on non-Windows platforms
+default-filter = 'all()'
+
+# Retry flaky test on macOS
+[[profile.default.overrides]]
+platform = 'cfg(target_os = "macos")'
+filter = 'test(cleanup_on_drop)'
+retries = 3
+
+# Test priority configuration - run tests in order of importance
+[[profile.default.overrides]]
+filter = 'package(butane_test_helper)'
+priority = 100
+
+[[profile.default.overrides]]
+filter = 'package(butane_core)'
+priority = 80
+
+[[profile.default.overrides]]
+filter = 'package(butane_cli)'
+priority = 60
+
+[[profile.default.overrides]]
+filter = 'package(butane_tests)'
+priority = 40
+
+[[profile.default.overrides]]
+filter = 'package(trybuild_tests)'
+priority = 20

.github/workflows/ci.yml

@@ -71,16 +71,13 @@ jobs:
         echo "C:\Program Files\PostgreSQL\12\lib" >> $GITHUB_PATH
         echo "PQ_LIB_DIR=C:\Program Files\PostgreSQL\12\lib" >> $GITHUB_ENV
         echo BUTANE_PG_CONNSTR="host=localhost user=postgres password=root sslmode=disable port=5432" >> $GITHUB_ENV
-    - name: Install sqlite (Windows)
+    - name: Install sqlite on Windows using vcpkg
       if: runner.os == 'Windows'
-      shell: cmd
       run: |
-        choco install sqlite
-        cd /D C:\ProgramData\chocolatey\lib\SQLite\tools
-        call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
-        lib /machine:x64 /def:sqlite3.def /out:sqlite3.lib
-        echo "C:\ProgramData\chocolatey\lib\SQLite\tools" >> $GITHUB_PATH
-        echo "SQLITE3_LIB_DIR=C:\ProgramData\chocolatey\lib\SQLite\tools" >> $GITHUB_ENV
+        vcpkg install sqlite3:x64-windows
+        echo "VCPKG_ROOT=C:/vcpkg" >> $GITHUB_ENV
+        echo "SQLITE3_LIB_DIR=C:/vcpkg/installed/x64-windows/lib" >> $GITHUB_ENV
+        echo "SQLITE3_INCLUDE_DIR=C:/vcpkg/installed/x64-windows/include" >> $GITHUB_ENV
 
     - name: Add Rust nightly toolchain
       uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -96,7 +93,7 @@ jobs:
     - name: Install tool binaries
       uses: taiki-e/install-action@v2
       with:
-        tool: cargo-deny,editorconfig-checker,mise,typos
+        tool: cargo-deny,cargo-nextest,editorconfig-checker,mise,typos
     - name: Install ephemeral-postgres via mise
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -115,24 +112,18 @@ jobs:
       run: make lint-ci
     - name: Run cargo-deny
       run: cargo deny check all
-    - name: Test Butane test helper with ephemeral-postgres
+    - name: Test with mise tools
       if: runner.os != 'Windows'
-      run: cd butane_test_helper && mise exec ephemeral-postgres -- cargo +stable test --all-features
-    - name: Test Butane test helper without ephemeral-postgres
+      run: mise exec ephemeral-postgres -- cargo +stable nextest run --all-features --no-fail-fast
+    - name: Test without mise
       if: runner.os == 'Windows'
-      run: cd butane_test_helper && cargo +stable test --all-features
-    - name: Test Core
-      run: cd butane_core && cargo +stable test --all-features
-    - name: Test CLI
-      run: cd butane_cli && cargo +stable test --all-features
-    - name: Test
-      run: cargo +stable test -p butane_tests --all-features
-    - name: Test using trybuild
-      run: cargo +stable test -p trybuild_tests
+      run: cargo +stable nextest run --all-features --no-fail-fast
+
     - name: Test doctests
       run: |
-        cargo +stable test -p butane_codegen --all-features
-        cargo +stable test -p butane --all-features
+        cargo +stable test -p butane_codegen --doc
+        cargo +stable test -p butane --doc --features async
+
     - name: Check example migrations have been updated
       run: |
         set -ex
@@ -144,8 +135,8 @@ jobs:
     - name: Run tests in examples
       run: |
         set -ex
-        cargo +stable test -p example --all-features
+        cargo +stable nextest run -p example --all-features
         cd examples
         for dir in *; do
-          cargo +stable test -p $dir --all-features
+          cargo +stable nextest run -p $dir --all-features
         done

.mise.toml

@@ -1,5 +1,6 @@
 [tools]
 "cargo:cargo-expand" = "latest"
+"cargo:cargo-nextest" = "latest"
 "cargo:pep257" = "latest"
 "cargo:timeout-cli" = "latest"
 editorconfig-checker = "latest"

CONTRIBUTING.md

@@ -91,12 +91,26 @@ Run all tests:
 cargo test
 ```
 
+It is also recommended to use cargo-nextest which provides many benefits, especially when running all tests:
+
+```bash
+cargo nextest run
+```
+
+Note cargo-nextest does not support doctests, [yet](https://github.com/nextest-rs/nextest/issues/16).
+
+To check those:
+
+```bash
+cargo test -p butane --doc --features async
+cargo test -p butane_codegen --doc  # all doctest are ignored
+```
+
 Run tests for a specific package:
 
 ```bash
-cargo test -p butane_core
-cargo test -p butane  # doctests only
-cargo test -p butane_tests
+cargo test -p butane_core --all-features
+cargo test -p butane_tests --all-features
 cargo test -p trybuild_tests
 ```