Run the container / vm tests twice if they fail, with a timeout of 10min each (#1591)

Author: grahamc

.github/workflows/ci.yml

@@ -598,8 +598,8 @@ jobs:
       matrix:
         command:
           - nix flake check -L
-          - nix build -L --tarball-ttl 0 --keep-going .#hydraJobs.container-test.all.x86_64-linux.all
-          - nix build -L --tarball-ttl 0 --keep-going .#hydraJobs.vm-test.all.x86_64-linux.all
+          - nix build -L --tarball-ttl 0 --timeout 600 --keep-going .#hydraJobs.container-test.all.x86_64-linux.all
+          - nix build -L --tarball-ttl 0 --timeout 600 --keep-going .#hydraJobs.vm-test.all.x86_64-linux.all
     permissions:
       id-token: "write"
       contents: "read"
@@ -624,7 +624,12 @@ jobs:
           log-directives: nix_installer=debug
           logger: pretty
       - uses: DeterminateSystems/flakehub-cache-action@main
-      - run: ${{ matrix.command }}
+      - run: |
+          if ! (${{ matrix.command }}); then
+            echo "failed, retrying once ..."
+            printf "\n\n\n\n\n\n\n\n"
+            (${{ matrix.command }})
+          fi
 
   run-x86_64-linux-release-checks:
     name: Run x86_64 Linux release checks