flake.nix: build all packages though a single evaluation

charludo · charludo · commit b9dc09cbd2d5 · 2025-12-18T08:32:29.000+01:00
diff --git a/.github/workflows/bm_maintenance.yml b/.github/workflows/bm_maintenance.yml
@@ -16,9 +16,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   build-image:
     name: "Build cleanup-bare-metal image"
     runs-on: ubuntu-24.04
+    needs: nix-build
     permissions:
       contents: read
       packages: write
diff --git a/.github/workflows/check_patches.yml b/.github/workflows/check_patches.yml
@@ -9,8 +9,15 @@ on:
       - packages/by-name/kata/kata-runtime/0*.patch
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   check-patches:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read
diff --git a/.github/workflows/docs_publish.yml b/.github/workflows/docs_publish.yml
@@ -27,9 +27,16 @@ permissions:
   pull-requests: write
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   deploy:
     name: Publish docs to GitHub Pages
     runs-on: ubuntu-24.04
+    needs: nix-build
     # Prevent the job from running on forked PRs, for security reasons.
     if: |
       (github.event_name == 'push' && github.ref_name == 'main') ||
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -38,9 +38,16 @@ env:
   DO_NOT_TRACK: 1
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   test:
     name: "${{ inputs.test-name }}${{ inputs.debug-shell && ' (with debug shell)' || '' }}"
     runs-on: ${{ inputs.runner }}
+    needs: nix-build
     permissions:
       contents: read
       packages: write
diff --git a/.github/workflows/imagepuller-benchmark.yml b/.github/workflows/imagepuller-benchmark.yml
@@ -12,8 +12,15 @@ on:
       - main
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   run-benchmark:
     runs-on: ubuntu-24.04
+    needs: nix-build
     permissions:
       contents: read
       packages: read
diff --git a/.github/workflows/k3s_compatibility.yml b/.github/workflows/k3s_compatibility.yml
@@ -12,8 +12,15 @@ env:
   container_registry: ghcr.io/edgelesssys
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   test:
     runs-on: ubuntu-24.04
+    needs: nix-build
     permissions:
       contents: read
       packages: write
diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
@@ -9,9 +9,16 @@ on:
       - ".github/workflows/links.yml"
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   check-links:
     name: check external links
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read
diff --git a/.github/workflows/nix_build.yml b/.github/workflows/nix_build.yml
@@ -0,0 +1,31 @@
+name: build and cache all flake outputs
+
+on:
+  workflow_call:
+    secrets:
+      GITHUB_TOKEN_IN:
+        required: true
+      CACHIX_AUTH_TOKEN:
+        required: true
+
+jobs:
+  nix-build:
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        include:
+          - runner: yet-to-be-set-up-nix-build-host
+            system: x86_64-linux
+    steps:
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/setup_nix
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN_IN || secrets.GITHUB_TOKEN }}
+          cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: pre-build all flake outputs
+        run: |
+          nix build .#matrix.${{ matrix.system }}
diff --git a/.github/workflows/pr_release_artifacts.yml b/.github/workflows/pr_release_artifacts.yml
@@ -12,10 +12,17 @@ on:
         default: false
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   create-release-artifacts:
     name: Create release artifacts for PR
     if: github.event_name == 'workflow_dispatch' && github.event.inputs.cleanup == 'false'
     runs-on: ubuntu-24.04
+    needs: nix-build
     permissions:
       pull-requests: write
       issues: write
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -119,10 +119,16 @@ jobs:
             exit 1
           fi
 
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   update-main:
     name: Update main branch
     runs-on: ubuntu-24.04
-    needs: [process-inputs, release]
+    needs: [process-inputs, release, nix-build]
     permissions:
       contents: write
     env:
@@ -220,7 +226,7 @@ jobs:
   release:
     name: Build and push artifacts, create release
     runs-on: ubuntu-24.04
-    needs: process-inputs
+    needs: [process-inputs, nix-build]
     permissions:
       contents: write
       packages: write
@@ -311,7 +317,7 @@ jobs:
       # Job needs content:write to see draft releases.
       contents: write
       packages: read
-    needs: [process-inputs, release]
+    needs: [process-inputs, release, nix-build]
     env:
       VERSION: ${{ inputs.version }}
     steps:
diff --git a/.github/workflows/rim_updates.yml b/.github/workflows/rim_updates.yml
@@ -6,9 +6,16 @@ on:
     - cron: "0 18 * * 0"
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   update-rim-ids:
     name: update NVIDIA RIM IDs
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 20
     permissions:
       contents: read
diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml
@@ -8,8 +8,15 @@ on:
   pull_request:
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   flake-check:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read
@@ -27,6 +34,7 @@ jobs:
 
   generate:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 60
     permissions:
       contents: write
@@ -52,6 +60,7 @@ jobs:
 
   govulncheck:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read
@@ -69,6 +78,7 @@ jobs:
 
   golangci-lint:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read
@@ -86,6 +96,7 @@ jobs:
 
   go-licenses:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read
diff --git a/flake.nix b/flake.nix
@@ -36,12 +36,13 @@
           overlays = [
             (final: _prev: { fenix = self.inputs.fenix.packages.${final.stdenv.hostPlatform.system}; })
             (import ./overlays/nixpkgs.nix)
-            (import ./overlays/contrast.nix)
+            (import ./overlays/contrast.nix { inherit (self) inputs; })
           ];
           config.allowUnfree = true;
           config.nvidia.acceptLicense = true;
         };
         treefmtEval = treefmt-nix.lib.evalModule pkgs ./treefmt.nix;
+        inherit (pkgs) lib;
       in
 
       {
@@ -57,6 +58,8 @@
             "contrastPkgs"
           ];
         };
+
+        matrix = pkgs.writeText "output-matrix.json" (builtins.toJSON (lib.ci.allOutputs self system));
       }
     );
 
diff --git a/lib/ci.nix b/lib/ci.nix
@@ -0,0 +1,22 @@
+{ lib, ... }:
+rec {
+  fromPackageOutputs =
+    flake: system:
+    lib.concatMap (kind: lib.attrValues (lib.attrByPath [ kind system ] { } flake)) [
+      "legacyPackages"
+      "packages"
+      "checks"
+      "devShells"
+      "formatters"
+    ];
+
+  allOutputs =
+    flake: system:
+    lib.filter lib.isDerivation (
+      lib.unique (
+        lib.concatMap (from: from flake system) [
+          fromPackageOutputs
+        ]
+      )
+    );
+}
diff --git a/lib/default.nix b/lib/default.nix
@@ -0,0 +1,12 @@
+{ inputs }:
+
+let
+  mkLib =
+    nixpkgs:
+    nixpkgs.lib.extend (
+      self: _: {
+        ci = import ./ci.nix { lib = self; };
+      }
+    );
+in
+mkLib inputs.nixpkgs
diff --git a/overlays/contrast.nix b/overlays/contrast.nix
@@ -1,8 +1,11 @@
 # Copyright 2024 Edgeless Systems GmbH
 # SPDX-License-Identifier: BUSL-1.1
 
+{ inputs }:
+
 final: _prev:
 
 {
   contrastPkgs = import ../packages { pkgs = final; };
+  lib = import ../lib { inherit inputs; };
 }
diff --git a/packages/contrast-releases.json b/packages/contrast-releases.json
@@ -1,9 +1,5 @@
 {
   "contrast": [
-    {
-      "version": "v0.2.0",
-      "hash": "sha256-cGyWvUmL/rbjX3Bu1OPGTZlk5PgVW/O0qdo9KmZPo+U="
-    },
     {
       "version": "v0.3.0",
       "hash": "sha256-8yVZLhwf2bsWIMxSAqXUKDNCjltCSGA6bwrXTi+cJB0="
diff --git a/packages/static.nix b/packages/static.nix
@@ -0,0 +1,16 @@
+# Copyright 2025 Edgeless Systems GmbH
+# SPDX-License-Identifier: BUSL-1.1
+
+{ pkgs }:
+
+let
+  inherit (pkgs.lib) makeScope;
+in
+
+makeScope pkgs.pkgsStatic.newScope (
+  self:
+  pkgs.lib.packagesFromDirectoryRecursive {
+    inherit (self) callPackage newScope;
+    directory = ./by-name;
+  }
+)
