Merge pull request #14184 from lovesegfault/nix-better-ci

ci: integrate vm_tests into main tests job

Author: Mic92

.github/workflows/ci.yml

@@ -126,6 +126,14 @@ jobs:
           --argstr stdenv "${{ matrix.stdenv }}" \
           ${{ format('--arg withAWS {0}', matrix.withAWS) }} \
           ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }}
+    - name: Run VM tests
+      run: |
+        nix build --file ci/gha/tests/wrapper.nix vmTests -L \
+          --arg withInstrumentation ${{ matrix.instrumented }} \
+          --argstr stdenv "${{ matrix.stdenv }}" \
+          ${{ format('--arg withAWS {0}', matrix.withAWS) }} \
+          ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }}
+      if: ${{ matrix.os == 'linux' }}
     - name: Run flake checks and prepare the installer tarball
       run: |
         ci/gha/tests/build-checks
@@ -213,7 +221,7 @@ jobs:
           echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
 
   docker_push_image:
-    needs: [tests, vm_tests, check_secrets]
+    needs: [tests, check_secrets]
     permissions:
       contents: read
       packages: write
@@ -266,43 +274,8 @@ jobs:
         docker tag nix:$NIX_VERSION $IMAGE_ID:master
         docker push $IMAGE_ID:master
 
-  vm_tests:
-    needs: basic-checks
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # TODO: remove once curl-based-s3 fully lands
-          - scenario: legacy s3
-            withAWS: true
-            withCurlS3: false
-          - scenario: curl s3
-            withAWS: false
-            withCurlS3: true
-    name: vm_tests (${{ matrix.scenario }})
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v5
-      - uses: ./.github/actions/install-nix-action
-        with:
-          dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
-          extra_nix_config:
-            experimental-features = nix-command flakes
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: DeterminateSystems/magic-nix-cache-action@main
-      - name: Build VM tests
-        run: |
-          nix build -L \
-            --file ci/gha/vm-tests/wrapper.nix \
-            ${{ format('--arg withAWS {0}', matrix.withAWS) }} \
-            ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }} \
-            functional_user \
-            githubFlakes \
-            nix-docker \
-            tarballFlakes
-
   flake_regressions:
-    needs: vm_tests
+    needs: tests
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout nix

ci/gha/tests/default.nix

@@ -79,6 +79,14 @@ rec {
     }
   );
 
+  # Import NixOS tests using the instrumented components
+  nixosTests = import ../../../tests/nixos {
+    inherit lib pkgs;
+    nixComponents = nixComponentsInstrumented;
+    nixpkgs = nixFlake.inputs.nixpkgs;
+    inherit (nixFlake.inputs) nixpkgs-23-11;
+  };
+
   /**
     Top-level tests for the flake outputs, as they would be built by hydra.
     These tests generally can't be overridden to run with sanitizers.
@@ -229,4 +237,24 @@ rec {
     {
       inherit coverageProfileDrvs mergedProfdata coverageReports;
     };
+
+  vmTests = {
+  }
+  # FIXME: when the curlS3 implementation is complete, it should also enable these tests.
+  // lib.optionalAttrs (withAWS == true) {
+    # S3 binary cache store test only runs when S3 support is enabled
+    inherit (nixosTests) s3-binary-cache-store;
+  }
+  // lib.optionalAttrs (!withSanitizers && !withCoverage) {
+    # evalNixpkgs uses non-instrumented components from hydraJobs, so only run it
+    # when not testing with sanitizers to avoid rebuilding nix
+    inherit (hydraJobs.tests) evalNixpkgs;
+    # FIXME: CI times out when building vm tests instrumented
+    inherit (nixosTests)
+      functional_user
+      githubFlakes
+      nix-docker
+      tarballFlakes
+      ;
+  };
 }