Add back some testing

Author: grahamc

.github/workflows/cache-test.sh

@@ -0,0 +1,93 @@
+#! /usr/bin/env bash
+
+set -e
+set -ux
+
+seed="$(date)-$RANDOM"
+
+log="${MAGIC_NIX_CACHE_DAEMONDIR}/daemon.log"
+
+flakehub_binary_cache=https://cache.flakehub.com
+gha_binary_cache=http://127.0.0.1:37515
+
+is_gh_throttled() {
+  grep 'GitHub Actions Cache throttled Magic Nix Cache' "${log}"
+}
+
+# Check that the action initialized correctly.
+grep 'FlakeHub cache is enabled' "${log}"
+grep 'Using cache' "${log}"
+
+if [ "$EXPECT_GITHUB_CACHE" == "true" ]; then
+  grep 'GitHub Action cache is enabled' "${log}"
+else
+  grep 'Native GitHub Action cache is disabled' "${log}"
+fi
+
+# Build something.
+outpath=$(nix-build .github/workflows/cache-tester.nix --argstr seed "$seed")
+
+# Wait until it has been pushed succesfully.
+found=
+for ((i = 0; i < 60; i++)); do
+    sleep 1
+    if grep "✅ $(basename "${outpath}")" "${log}"; then
+        found=1
+        break
+    fi
+done
+if [[ -z $found ]]; then
+    echo "FlakeHub push did not happen." >&2
+    exit 1
+fi
+
+if [ "$EXPECT_GITHUB_CACHE" == "true" ]; then
+  found=
+  for ((i = 0; i < 60; i++)); do
+      sleep 1
+      if grep "Uploaded '${outpath}' to the GitHub Action Cache" "${log}"; then
+          found=1
+          break
+      fi
+  done
+  if [[ -z $found ]]; then
+      echo "GitHub Actions Cache push did not happen." >&2
+
+      if ! is_gh_throttled; then
+        exit 1
+      fi
+  fi
+fi
+
+
+
+# Check the FlakeHub binary cache to see if the path is really there.
+nix path-info --store "${flakehub_binary_cache}" "${outpath}"
+
+if [ "$EXPECT_GITHUB_CACHE" == "true" ] && ! is_gh_throttled; then
+  # Check the GitHub binary cache to see if the path is really there.
+  nix path-info --store "${gha_binary_cache}" "${outpath}"
+fi
+
+rm ./result
+nix store delete "${outpath}"
+if [ -f "$outpath" ]; then
+    echo "$outpath still exists? can't test"
+    exit 1
+fi
+
+rm -rf ~/.cache/nix
+
+echo "-------"
+echo "Trying to substitute the build again..."
+echo "if it fails, the cache is broken."
+
+# Check the FlakeHub binary cache to see if the path is really there.
+nix path-info --store "${flakehub_binary_cache}" "${outpath}"
+
+if [ "$EXPECT_GITHUB_CACHE" == "true" ] && ! is_gh_throttled; then
+  # Check the FlakeHub binary cache to see if the path is really there.
+  nix path-info --store "${gha_binary_cache}" "${outpath}"
+fi
+
+nix-store --realize -vvvvvvvv "$outpath"

.github/workflows/cache-tester.nix

@@ -0,0 +1,10 @@
+{ seed }:
+derivation {
+  name = "cache-test";
+  system = builtins.currentSystem;
+
+  builder = "/bin/sh";
+  args = [ "-euxc" "echo \"$seed\" > $out" ];
+
+  inherit seed;
+}

.github/workflows/ci.yml

@@ -4,7 +4,45 @@ on:
     branches: [main]
 
 jobs:
-  success:
+  run-systems:
+    if: github.event_name == 'merge_group'
+    name: "Test: ${{ matrix.systems.nix-system }}"
+    runs-on: "${{ matrix.systems.runner }}"
+    permissions:
+      id-token: "write"
+      contents: "read"
+    env:
+      ACTIONS_STEP_DEBUG: true
+    strategy:
+      fail-fast: false
+      matrix:
+        systems:
+          - nix-system: "aarch64-darwin"
+            runner: "macos-latest"
+          - nix-system: "x86_64-darwin"
+            runner: "macos-13"
+          - nix-system: "aarch64-linux"
+            runner: "namespace-profile-default-arm64"
+          - nix-system: "x86_64-linux"
+            runner: "ubuntu-22.04"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Nix on ${{ matrix.systems.nix-system }} system
+        uses: DeterminateSystems/determinate-nix-action@main
+        with:
+          extra-conf: |
+            narinfo-cache-negative-ttl = 0
+      - name: Cache the store
+        uses: ./
+        with:
+          _internal-strict-mode: true
+      - name: Check the cache for liveness
+        env:
+          EXPECT_GITHUB_CACHE: ${{ toJson(false) }}
+        run: |
+          .github/workflows/cache-test.sh
+
+  lint:
     name: Build
     runs-on: ubuntu-latest
     permissions:
@@ -27,3 +65,18 @@ jobs:
       - name: Assert no changes were made
         run: git diff --exit-code
         if: success() || failure()
+
+  success:
+    runs-on: ubuntu-latest
+    needs:
+      - run-systems
+      - lint
+    if: always()
+    steps:
+      - run: "true"
+      - run: |
+          echo "A dependent in the build matrix failed."
+          exit 1
+        if: |
+          contains(needs.*.result, 'failure') ||
+          contains(needs.*.result, 'cancelled')