From 6144e5b4afed98d8ad0cb40590e9788eec5867d1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 11 May 2025 05:30:22 -0400 Subject: [PATCH 01/17] Add a case-folding GNU/Linux CI test job This can likely be removed later. But it might be kept if it finds anything interesting, especially if it helps test a bugfix. In that case, it may make sense to have it run only tests that are of special interest in connection with case-sensitivity. --- .github/workflows/ci.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31927f159f6..b101bd4d076 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,6 +185,38 @@ jobs: - name: Check that tracked archives are up to date run: git diff --exit-code # If this fails, the fix is usually to commit a regenerated archive. + test-ext4-casefold: + runs-on: ubuntu-latest + + steps: + - name: Use case-folding workspace directory + run: | + set -ux + dd if=/dev/zero of=~/ext4-casefold.img bs=100M count=150 + mkfs.ext4 -O casefold -- ~/ext4-casefold.img + mkdir -- ~/ext4-casefold + sudo mount -- ~/ext4-casefold.img ~/ext4-casefold + sudo chown -- "$USER" ~/ext4-casefold + mkdir -- ~/ext4-casefold/icase + chattr +F -- ~/ext4-casefold/icase + sudo mount --rbind -- ~/ext4-casefold/icase . + - name: Verify case-folding + run: | + set -ux + shopt -s nullglob + touch a A + files=(?) + test "${#files[@]}" -eq 1 + rm a + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@v2 + with: + tool: nextest + - name: Test (nextest) + run: cargo nextest run --workspace --no-fail-fast + test-fixtures-windows: runs-on: windows-latest @@ -497,6 +529,7 @@ jobs: - test - test-journey - test-fast + - test-ext4-casefold - test-fixtures-windows - test-32bit - test-32bit-windows-size-doc From 135d2880e5f3727faca13652739d7978d1c7328e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 11 May 2025 06:17:20 -0400 Subject: [PATCH 02/17] Case-fold `$TMPDIR` as well in CI casefold test --- .github/workflows/ci.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b101bd4d076..42e2a75c23d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,7 +189,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: Use case-folding workspace directory + - name: Set up casefold directory run: | set -ux dd if=/dev/zero of=~/ext4-casefold.img bs=100M count=150 @@ -199,15 +199,24 @@ jobs: sudo chown -- "$USER" ~/ext4-casefold mkdir -- ~/ext4-casefold/icase chattr +F -- ~/ext4-casefold/icase - sudo mount --rbind -- ~/ext4-casefold/icase . + - name: Map workspace and TMPDIR + run: | + set -ux + mkdir -- ~/ext4-casefold/icase/{workspace,tmp} + sudo mount --rbind -- ~/ext4-casefold/icase/workspace . + printf 'TMPDIR=%s\n' ~/ext4-casefold/icase/tmp >> "$GITHUB_ENV" - name: Verify case-folding run: | set -ux shopt -s nullglob - touch a A - files=(?) - test "${#files[@]}" -eq 1 - rm a + verify() { + touch a A + files=(?) + test "${#files[@]}" -eq 1 + rm a + } + verify + (cd -- "$TMPDIR"; verify) - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 From f183ee20ab62dbff7452ee8657d988cd97b17a6d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 11 May 2025 07:07:47 -0400 Subject: [PATCH 03/17] Run only 2 parallel test processes in test-ext4-casefold This seemed to make the bug *easier* to reproduce (compared to larger values) locally, so maybe it will surface it on CI even in GNU/Linux. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42e2a75c23d..bf94f26b5d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -224,7 +224,7 @@ jobs: with: tool: nextest - name: Test (nextest) - run: cargo nextest run --workspace --no-fail-fast + run: cargo nextest run --workspace --no-fail-fast --test-threads=2 test-fixtures-windows: runs-on: windows-latest From afa16b66b58fb3394bb34d2beee5f343b3c1cb45 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 11 May 2025 06:51:22 -0400 Subject: [PATCH 04/17] Increase symlink experiment reps 10x in test-ext4-casefold From 1000 (0..=999) to 10000 (0..=9999). --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf94f26b5d2..3026b8fbe07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,6 +223,11 @@ jobs: - uses: taiki-e/install-action@v2 with: tool: nextest + - name: More writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed reps + run: | + # Add another "9" digit to the upper bound of the label range. + sed -Ei 's/^(#\[test_matrix\(0\.\.=)([[:digit:]]+\)])$/\19\2/' \ + gix-worktree-state/tests/state/checkout.rs - name: Test (nextest) run: cargo nextest run --workspace --no-fail-fast --test-threads=2 From 1a2615ee2370be52ee3de37827b573e5d1a3391b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 11 May 2025 07:38:30 -0400 Subject: [PATCH 05/17] Try even harder to reproduce the failure on GNU/Linux By repeating `cargo nextext` on the affected test group, up to 20 times. --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3026b8fbe07..f3a99ca706c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,7 +229,15 @@ jobs: sed -Ei 's/^(#\[test_matrix\(0\.\.=)([[:digit:]]+\)])$/\19\2/' \ gix-worktree-state/tests/state/checkout.rs - name: Test (nextest) - run: cargo nextest run --workspace --no-fail-fast --test-threads=2 + run: cargo nextest run --workspace --no-fail-fast + - name: Retest writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed (nextest) + run: | + set -x + for i in {1..20}; do + cargo nextest run -p gix-worktree-state-tests \ + writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed \ + --no-fail-fast --test-threads=2 + done test-fixtures-windows: runs-on: windows-latest From d6c7a2202939a2751ac3880773ac10d424bf38cc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 12 May 2025 05:21:48 -0400 Subject: [PATCH 06/17] Add `ubuntu-24.04-arm` to `test-ext4-casefold` job In case the failure might somehow be easier to reproduce there. --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3a99ca706c..7179d8fe87c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,7 +186,12 @@ jobs: run: git diff --exit-code # If this fails, the fix is usually to commit a regenerated archive. test-ext4-casefold: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + fail-fast: false + + runs-on: ${{ matrix.os }} steps: - name: Set up casefold directory From cb91552ef0f9a68ca576aa0400c783f0635347df Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 12 May 2025 21:13:29 -0400 Subject: [PATCH 07/17] Use tmpfs casefold instead of ext4 casefold --- .github/workflows/ci.yml | 46 +++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7179d8fe87c..aa06d6a9529 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,7 +185,7 @@ jobs: - name: Check that tracked archives are up to date run: git diff --exit-code # If this fails, the fix is usually to commit a regenerated archive. - test-ext4-casefold: + test-linux-casefold: strategy: matrix: os: [ubuntu-latest, ubuntu-24.04-arm] @@ -194,34 +194,32 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Set up casefold directory + - name: Set up casefold directories run: | set -ux - dd if=/dev/zero of=~/ext4-casefold.img bs=100M count=150 - mkfs.ext4 -O casefold -- ~/ext4-casefold.img - mkdir -- ~/ext4-casefold - sudo mount -- ~/ext4-casefold.img ~/ext4-casefold - sudo chown -- "$USER" ~/ext4-casefold - mkdir -- ~/ext4-casefold/icase - chattr +F -- ~/ext4-casefold/icase - - name: Map workspace and TMPDIR - run: | - set -ux - mkdir -- ~/ext4-casefold/icase/{workspace,tmp} - sudo mount --rbind -- ~/ext4-casefold/icase/workspace . - printf 'TMPDIR=%s\n' ~/ext4-casefold/icase/tmp >> "$GITHUB_ENV" - - name: Verify case-folding - run: | - set -ux - shopt -s nullglob - verify() { + + fold() ( + cd -- "$1" + + # Set up a case-folding tmpfs filesystem. + sudo mount -t tmpfs -o casefold tmpfs . + sudo chown -- "$USER" . + chattr +F . + + # Test that case-folding really works. touch a A files=(?) test "${#files[@]}" -eq 1 rm a - } - verify - (cd -- "$TMPDIR"; verify) + ) + + # Set up workspace. + fold . + + # Set up TMPDIR. + mkdir -- ~/tmp + fold ~/tmp + printf 'TMPDIR=%s\n' ~/tmp >> "$GITHUB_ENV" - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -556,7 +554,7 @@ jobs: - test - test-journey - test-fast - - test-ext4-casefold + - test-linux-casefold - test-fixtures-windows - test-32bit - test-32bit-windows-size-doc From 626217989d57cf1827643c2c699e0400d81eba2f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 16 May 2025 22:27:27 -0400 Subject: [PATCH 08/17] Use tmpfs-backed ext4 as a workaround For when tmpfs does not suppport `-o casefold` in mounting. This does require that ext4 support `-O casefold`, but that was already working before. Even though that worked before, this could fail when using an ext4 image on tmpfs, because the ext4 image is smaller than it was when tmpfs was not used, in order to allow it to be created in memory. While this allows it to be created, the tests may fail if build artifacts and other files need more space than is available. --- .github/workflows/ci.yml | 88 +++++++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa06d6a9529..633cd2a83ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,28 +198,76 @@ jobs: run: | set -ux - fold() ( - cd -- "$1" - - # Set up a case-folding tmpfs filesystem. - sudo mount -t tmpfs -o casefold tmpfs . - sudo chown -- "$USER" . - chattr +F . + # FIXME: Either split the cases into two separate steps in the job, or remove the first + # case and have the check gate a warning that tmpfs casefold is available and preferable. + case "$(cat &2 + fold() ( + cd -- "$1" + + # Set up a case-folding tmpfs filesystem. + sudo mount -t tmpfs -o casefold tmpfs . + sudo chown -- "$USER" . + chattr +F . + + # Test that case-folding really works. + touch a A + files=(?) + test "${#files[@]}" -eq 1 + rm a + ) + + # Set up workspace. + fold . + + # Set up TMPDIR. + mkdir -- ~/tmp + fold ~/tmp + printf 'TMPDIR=%s\n' ~/tmp >> "$GITHUB_ENV" + + ;; + *) + echo 'Case-folding tmpfs is unavailable. tmpfs-backed ext4 will be used.' >&2 + + # Create mount points. + mkdir -- ~/mnt ~/mnt/tmpfs-nocasefold ~/mnt/ext4-casefold + + # Set up a tmpfs filesystem. + sudo mount -t tmpfs tmpfs -- ~/mnt/tmpfs-nocasefold + sudo chown -- "$USER" ~/mnt/tmpfs-nocasefold + + # Set up a casefold-enabled ext4 filesystem backed by the tmpfs filesystem. + dd if=/dev/zero of=~/mnt/tmpfs-nocasefold/ext4-casefold.img bs=100M count=60 + mkfs.ext4 -O casefold -- ~/mnt/tmpfs-nocasefold/ext4-casefold.img + sudo mount -- ~/mnt/tmpfs-nocasefold/ext4-casefold.img ~/mnt/ext4-casefold + sudo chown -- "$USER" ~/mnt/ext4-casefold + + # Create a case-folding directory with `workspace` and `tmp` subdirectories. + mkdir -- ~/mnt/ext4-casefold/icase + chattr +F -- ~/mnt/ext4-casefold/icase + mkdir -- ~/mnt/ext4-casefold/icase/{workspace,tmp} + + # Map our workspace to the `workspace` subdirectory. + sudo mount --rbind -- ~/mnt/ext4-casefold/icase/workspace . + cd . # Pick up the bind. + + # Arrange for the `tmp` subdirectory to be used instead of `/tmp`. + export TMPDIR=~/mnt/ext4-casefold/icase/tmp + echo "TMPDIR=$TMPDIR" >> "$GITHUB_ENV" # Test that case-folding really works. - touch a A - files=(?) - test "${#files[@]}" -eq 1 - rm a - ) - - # Set up workspace. - fold . - - # Set up TMPDIR. - mkdir -- ~/tmp - fold ~/tmp - printf 'TMPDIR=%s\n' ~/tmp >> "$GITHUB_ENV" + verify() { + touch a A + files=(?) + test "${#files[@]}" -eq 1 + rm a + } + verify + (cd -- "$TMPDIR"; verify) + + ;; + esac - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 From fae6c8ddcf939525bbd6fe1864301dc705ce2d86 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 16 May 2025 23:27:52 -0400 Subject: [PATCH 09/17] Run only the test(s) of interest In `test-linux-casefold`, this runs only the tests that are duplicates of each other, where the failure has been observed, rather than first running the whole test suite. The reason is to try to build fewer crates and test executables, since this is currenntly failing because it runs out of space on the ext4 image whose size is itself constrained by the amount of available memory for the tmpfs filesystem on which the image is created. --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 633cd2a83ab..c87c6db7e59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -279,9 +279,7 @@ jobs: # Add another "9" digit to the upper bound of the label range. sed -Ei 's/^(#\[test_matrix\(0\.\.=)([[:digit:]]+\)])$/\19\2/' \ gix-worktree-state/tests/state/checkout.rs - - name: Test (nextest) - run: cargo nextest run --workspace --no-fail-fast - - name: Retest writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed (nextest) + - name: Test writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed (nextest) run: | set -x for i in {1..20}; do From 59469945b2d99418299197d4dc9479cae8103187 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 17 May 2025 00:10:27 -0400 Subject: [PATCH 10/17] Don't attempt to use case-folding tmpfs But check if it is reported as available and, if so, issue a GHA warning that it should be used instead of the current way. --- .github/workflows/ci.yml | 105 ++++++++++++++------------------------- 1 file changed, 38 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c87c6db7e59..1e4ff784bfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,80 +194,51 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Set up casefold directories + - name: Set up casefold directories (in tmpfs-backed ext4) run: | set -ux - # FIXME: Either split the cases into two separate steps in the job, or remove the first - # case and have the check gate a warning that tmpfs casefold is available and preferable. case "$(cat &2 - fold() ( - cd -- "$1" - - # Set up a case-folding tmpfs filesystem. - sudo mount -t tmpfs -o casefold tmpfs . - sudo chown -- "$USER" . - chattr +F . - - # Test that case-folding really works. - touch a A - files=(?) - test "${#files[@]}" -eq 1 - rm a - ) - - # Set up workspace. - fold . - - # Set up TMPDIR. - mkdir -- ~/tmp - fold ~/tmp - printf 'TMPDIR=%s\n' ~/tmp >> "$GITHUB_ENV" - - ;; - *) - echo 'Case-folding tmpfs is unavailable. tmpfs-backed ext4 will be used.' >&2 - - # Create mount points. - mkdir -- ~/mnt ~/mnt/tmpfs-nocasefold ~/mnt/ext4-casefold - - # Set up a tmpfs filesystem. - sudo mount -t tmpfs tmpfs -- ~/mnt/tmpfs-nocasefold - sudo chown -- "$USER" ~/mnt/tmpfs-nocasefold - - # Set up a casefold-enabled ext4 filesystem backed by the tmpfs filesystem. - dd if=/dev/zero of=~/mnt/tmpfs-nocasefold/ext4-casefold.img bs=100M count=60 - mkfs.ext4 -O casefold -- ~/mnt/tmpfs-nocasefold/ext4-casefold.img - sudo mount -- ~/mnt/tmpfs-nocasefold/ext4-casefold.img ~/mnt/ext4-casefold - sudo chown -- "$USER" ~/mnt/ext4-casefold - - # Create a case-folding directory with `workspace` and `tmp` subdirectories. - mkdir -- ~/mnt/ext4-casefold/icase - chattr +F -- ~/mnt/ext4-casefold/icase - mkdir -- ~/mnt/ext4-casefold/icase/{workspace,tmp} - - # Map our workspace to the `workspace` subdirectory. - sudo mount --rbind -- ~/mnt/ext4-casefold/icase/workspace . - cd . # Pick up the bind. - - # Arrange for the `tmp` subdirectory to be used instead of `/tmp`. - export TMPDIR=~/mnt/ext4-casefold/icase/tmp - echo "TMPDIR=$TMPDIR" >> "$GITHUB_ENV" - - # Test that case-folding really works. - verify() { - touch a A - files=(?) - test "${#files[@]}" -eq 1 - rm a - } - verify - (cd -- "$TMPDIR"; verify) - + echo '::warning:: The runner system supports case-folding tmpfs, which is preferable.' ;; esac + + # Create mount points. + mkdir -- ~/mnt ~/mnt/tmpfs-nocasefold ~/mnt/ext4-casefold + + # Set up a tmpfs filesystem. + sudo mount -t tmpfs tmpfs -- ~/mnt/tmpfs-nocasefold + sudo chown -- "$USER" ~/mnt/tmpfs-nocasefold + + # Set up a casefold-enabled ext4 filesystem backed by the tmpfs filesystem. + dd if=/dev/zero of=~/mnt/tmpfs-nocasefold/ext4-casefold.img bs=100M count=60 + mkfs.ext4 -O casefold -- ~/mnt/tmpfs-nocasefold/ext4-casefold.img + sudo mount -- ~/mnt/tmpfs-nocasefold/ext4-casefold.img ~/mnt/ext4-casefold + sudo chown -- "$USER" ~/mnt/ext4-casefold + + # Create a case-folding directory with `workspace` and `tmp` subdirectories. + mkdir -- ~/mnt/ext4-casefold/icase + chattr +F -- ~/mnt/ext4-casefold/icase + mkdir -- ~/mnt/ext4-casefold/icase/{workspace,tmp} + + # Map our workspace to the `workspace` subdirectory. + sudo mount --rbind -- ~/mnt/ext4-casefold/icase/workspace . + cd . # Pick up the bind. + + # Arrange for the `tmp` subdirectory to be used instead of `/tmp`. + export TMPDIR=~/mnt/ext4-casefold/icase/tmp + echo "TMPDIR=$TMPDIR" >> "$GITHUB_ENV" + + # Test that case-folding really works. + verify() { + touch a A + files=(?) + test "${#files[@]}" -eq 1 + rm a + } + verify + (cd -- "$TMPDIR"; verify) - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 From 58c539659332ef9d82259d63008e41bd05541bf6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 17 May 2025 01:00:19 -0400 Subject: [PATCH 11/17] Try to fail at least as often, with less overhead This combines a few changes: - Only report the status of tests that actually fail. - Run the `cargo nextest` run command once, not 20 times. - Multiply the number of test cases by another factor of 10. - Stop after the first failing case. --- .github/workflows/ci.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e4ff784bfc..61ad142f482 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -247,17 +247,14 @@ jobs: tool: nextest - name: More writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed reps run: | - # Add another "9" digit to the upper bound of the label range. - sed -Ei 's/^(#\[test_matrix\(0\.\.=)([[:digit:]]+\)])$/\19\2/' \ + # Add two more "9" digits to the upper bound of the label range. + sed -Ei 's/^(#\[test_matrix\(0\.\.=)([[:digit:]]+\)])$/\199\2/' \ gix-worktree-state/tests/state/checkout.rs - name: Test writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed (nextest) run: | - set -x - for i in {1..20}; do - cargo nextest run -p gix-worktree-state-tests \ - writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed \ - --no-fail-fast --test-threads=2 - done + cargo nextest run -p gix-worktree-state-tests \ + writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed \ + --status-level=fail --test-threads=2 test-fixtures-windows: runs-on: windows-latest From 9c3ba7ae1c260f0717391873f7ad77d4fcbc6b12 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 17 May 2025 02:04:30 -0400 Subject: [PATCH 12/17] Move the ext4 image off tmpfs This tries going back to not using tmpfs for I/O speedup, with the idea that the increased complexity and decreased flexibility might be possible to avoid now that other adjustments are made to surface the failure more reliably. --- .github/workflows/ci.yml | 58 +++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61ad142f482..ec9a6fc83da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,7 +185,7 @@ jobs: - name: Check that tracked archives are up to date run: git diff --exit-code # If this fails, the fix is usually to commit a regenerated archive. - test-linux-casefold: + test-ext4-casefold: strategy: matrix: os: [ubuntu-latest, ubuntu-24.04-arm] @@ -194,43 +194,33 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Set up casefold directories (in tmpfs-backed ext4) + - name: Warn if we could use tmpfs mounted with `-o casefold` instead run: | - set -ux - case "$(cat > "$GITHUB_ENV" - - # Test that case-folding really works. + - name: Set up casefold directory + run: | + set -ux + dd if=/dev/zero of=~/ext4-casefold.img bs=100M count=150 + mkfs.ext4 -O casefold -- ~/ext4-casefold.img + mkdir -- ~/ext4-casefold + sudo mount -- ~/ext4-casefold.img ~/ext4-casefold + sudo chown -- "$USER" ~/ext4-casefold + mkdir -- ~/ext4-casefold/icase + chattr +F -- ~/ext4-casefold/icase + - name: Arrange to map workspace and TMPDIR in all subsequent steps + run: | + set -ux + mkdir -- ~/ext4-casefold/icase/{workspace,tmp} + sudo mount --rbind -- ~/ext4-casefold/icase/workspace . + printf 'TMPDIR=%s\n' ~/ext4-casefold/icase/tmp >> "$GITHUB_ENV" + - name: Verify case folding in workspace and TMPDIR + run: | + set -ux + shopt -s nullglob verify() { touch a A files=(?) @@ -568,7 +558,7 @@ jobs: - test - test-journey - test-fast - - test-linux-casefold + - test-ext4-casefold - test-fixtures-windows - test-32bit - test-32bit-windows-size-doc From cbd0f8ffd66674e3259d7d1bad5df18f1bcd0a3b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 17 May 2025 20:29:35 -0400 Subject: [PATCH 13/17] Try to fail more often, with less delay The `test-ext4-casefold` jobs were still not surfacing the error quite reliably enough. The wait was also longer than ideal. This commit makes some changes that hope to improve on both. - Decrease the number of duplicate test cases by a factor of 4. This has the effect that they are increased by 25x compared to the multiplier in the commited code, instead of 100x. - Create 10 times as many CI jobs in the matrix, by adding a `num` variable whose values are those in the range 0..=9. This variable is not actually used, it just serves to produce more CI jobs. - Don't use `Swatinem/rust-cache` in these jobs anymore. It is not obvious that it will make things work better overall, in terms of speedup, cache usage, and cache access, now that we are increasing the number of matrix jobs by a factor of 10. Moving some of the repetition into separate jobs, which may run in parallel, is not only, nor even primarily, to leverage runner parallelism to do more test runs faster. Instead, it is because it looks like some chaotic initial state may contribute to how likely the failure is to occur. No definitive experiment on CI shows this to be the case -- but in local testing, on some systems, I often have runs that fail early over and over again, and then wait a while, come back, and have many runs that don't surface the bug, then come back later and it is easy to produce again, and so on. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec9a6fc83da..b94bbaebec6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,6 +189,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, ubuntu-24.04-arm] + num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] fail-fast: false runs-on: ${{ matrix.os }} @@ -231,14 +232,13 @@ jobs: (cd -- "$TMPDIR"; verify) - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@v2 with: tool: nextest - name: More writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed reps run: | - # Add two more "9" digits to the upper bound of the label range. - sed -Ei 's/^(#\[test_matrix\(0\.\.=)([[:digit:]]+\)])$/\199\2/' \ + # Prepend leading digits "24" to the upper bound of the label range. + sed -Ei 's/^(#\[test_matrix\(0\.\.=)([[:digit:]]+\)])$/\124\2/' \ gix-worktree-state/tests/state/checkout.rs - name: Test writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed (nextest) run: | From 54ae0aa47920b68616e28ca644838dbd763bc531 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 17 May 2025 21:20:00 -0400 Subject: [PATCH 14/17] Further diversify runners --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b94bbaebec6..7b6c77c38a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,11 +188,11 @@ jobs: test-ext4-casefold: strategy: matrix: - os: [ubuntu-latest, ubuntu-24.04-arm] - num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + os-suffix: ['22.04', '24.04', '22.04-arm', '24.04-arm'] + rep: [A, B, C, D, E] fail-fast: false - runs-on: ${{ matrix.os }} + runs-on: ubuntu-${{ matrix.os-suffix }} steps: - name: Warn if we could use tmpfs mounted with `-o casefold` instead From dfe002fd2ce86c3ef935c4b492ff5c6821db32df Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 17 May 2025 22:05:39 -0400 Subject: [PATCH 15/17] Undiversify runners but vary test parallelism Since varying `runs-on` didn't surface more errors, nor help smooth out inconsistencies across runs. The `*-arm` runners seem to have fewer errors with the particular way the tests are running now, so this just uses `ubuntu-latest`. This also adjusts `rep`, having it take on more values, so as to more than make up for what would otherwise be a smaller number of jobs. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b6c77c38a1..ba3953b9080 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,11 +188,11 @@ jobs: test-ext4-casefold: strategy: matrix: - os-suffix: ['22.04', '24.04', '22.04-arm', '24.04-arm'] - rep: [A, B, C, D, E] + parallel-tests: [2, 3, 4] + rep: [A, B, C, D, E, F, G, H] fail-fast: false - runs-on: ubuntu-${{ matrix.os-suffix }} + runs-on: ubuntu-latest steps: - name: Warn if we could use tmpfs mounted with `-o casefold` instead @@ -244,7 +244,7 @@ jobs: run: | cargo nextest run -p gix-worktree-state-tests \ writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed \ - --status-level=fail --test-threads=2 + --status-level=fail --test-threads=${{ matrix.parallel-tests }} test-fixtures-windows: runs-on: windows-latest From c3f156f359f77d8924ebdd4db39f7fd34cf99441 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 18 May 2025 00:09:20 -0400 Subject: [PATCH 16/17] Test more values for test parallelism This creates additional CI `test-ext4-casefold` jobs, for more values of the `--test-threads` operand passed to `cargo nextest`. The motivation is that the value of 3 curiously seems never to surface the failure on CI. Unlike 1, 3 should is expected to surface the failure comparably to 2 and 4, both of which do fail more often than not. Yet the jobs with `--test-threads=3` keep passing. (This is unlikely to be due to case-sensitivity misdetection, since the assertions for when the filesystem is case-sensitive would also fail, if they fire in a job where the filesystem is case-folding.) If these jobs are kept, then `rep` should probably be adjusted to have fewer, perhaps 7, values. However, this adjusts `rep` to have 16 values (i.e. twice as many as before), since the higher operands to `--test-threads` are expected to produce the failure less often. --- .github/workflows/ci.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba3953b9080..e23694e6231 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,8 +188,17 @@ jobs: test-ext4-casefold: strategy: matrix: - parallel-tests: [2, 3, 4] - rep: [A, B, C, D, E, F, G, H] + # `--test-threads` operand + parallel-tests: + - 1 # Never seems to fail, which makes sense. + - 2 + - 3 # Never seems to fail, which does NOT make sense. + - 4 + - 5 + - 6 + - 7 + # Duplicate jobs + rep: ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', O, P] fail-fast: false runs-on: ubuntu-latest From 378f489c2d762a1529f0dea05e1989fa3c69d5ad Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 18 May 2025 01:14:58 -0400 Subject: [PATCH 17/17] Test fewer test parallelism values and reps - Test `--test-threads` operands of 1 to 4, rather than 1 to 7. - Do 7 reps instead of 16. In addition to the 1000x repetition in `checkout.rs` of the writes_through_symlinks_are_prevented_even_if_overwriting_is_allowed test case -- which almost always surfaces #2006 on CI on macOS and Windows without further modifications -- the `test-ext4-casefold` jobs as they currenly stand seem like they would ber sufficient to test whether a possible future solution would fix the bug on GNU/Linux. This looks like about the minmum amount we should reasonably verify still fails (in some jobs) immediately before a #2006 fix, while no longer failing *any* jobs afterwards. But the `test-ext4-casefold` jobs are still much more than would be wanted on the main branch. Assuming the cause of the failures is the same on all systems, and the solution neither explicitly nor implicitly operates differently across systems (when comparing how it works when the filesystem is case-insensitive, that is), it may be enough on CI to regression test only macOS and Windows for #2006. Thus, this also adds a TODO comment atop the `test-ext4-casefold` definition reminding that it should not be kept, or not in full. --- .github/workflows/ci.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e23694e6231..8470ff51722 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,20 +185,18 @@ jobs: - name: Check that tracked archives are up to date run: git diff --exit-code # If this fails, the fix is usually to commit a regenerated archive. + # TODO(integration): Remove all or at least most test-ext4-casefold jobs prior to merging #2008. test-ext4-casefold: strategy: matrix: # `--test-threads` operand parallel-tests: - - 1 # Never seems to fail, which makes sense. - - 2 - - 3 # Never seems to fail, which does NOT make sense. - - 4 - - 5 - - 6 - - 7 + - 1 # Never fails. + - 2 # Usually fails. + - 3 # Almost never fails, somehow! + - 4 # Usually fails. # Duplicate jobs - rep: ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', O, P] + rep: [A, B, C, D, E, F, G] fail-fast: false runs-on: ubuntu-latest