flake.nix: build all packages though a single evaluation

.github/workflows/bm_maintenance.yml

@@ -16,9 +16,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   build-image:
     name: "Build cleanup-bare-metal image"
     runs-on: ubuntu-24.04
+    needs: nix-build
     permissions:
       contents: read
       packages: write

.github/workflows/check_patches.yml

@@ -9,8 +9,15 @@ on:
       - packages/by-name/kata/kata-runtime/0*.patch
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   check-patches:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read

.github/workflows/docs_publish.yml

@@ -27,9 +27,16 @@ permissions:
   pull-requests: write
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   deploy:
     name: Publish docs to GitHub Pages
     runs-on: ubuntu-24.04
+    needs: nix-build
     # Prevent the job from running on forked PRs, for security reasons.
     if: |
       (github.event_name == 'push' && github.ref_name == 'main') ||

.github/workflows/e2e.yml

@@ -38,9 +38,16 @@ env:
   DO_NOT_TRACK: 1
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   test:
     name: "${{ inputs.test-name }}${{ inputs.debug-shell && ' (with debug shell)' || '' }}"
     runs-on: ${{ inputs.runner }}
+    needs: nix-build
     permissions:
       contents: read
       packages: write

.github/workflows/imagepuller-benchmark.yml

@@ -12,8 +12,15 @@ on:
       - main
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   run-benchmark:
     runs-on: ubuntu-24.04
+    needs: nix-build
     permissions:
       contents: read
       packages: read

.github/workflows/k3s_compatibility.yml

@@ -12,8 +12,15 @@ env:
   container_registry: ghcr.io/edgelesssys
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   test:
     runs-on: ubuntu-24.04
+    needs: nix-build
     permissions:
       contents: read
       packages: write

.github/workflows/links.yml

@@ -9,9 +9,16 @@ on:
       - ".github/workflows/links.yml"
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   check-links:
     name: check external links
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read

.github/workflows/nix_build.yml

@@ -0,0 +1,31 @@
+name: build and cache all flake outputs
+
+on:
+  workflow_call:
+    secrets:
+      GITHUB_TOKEN_IN:
+        required: true
+      CACHIX_AUTH_TOKEN:
+        required: true
+
+jobs:
+  nix-build:
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        include:
+          - runner: yet-to-be-set-up-nix-build-host
+            system: x86_64-linux
+    steps:
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/setup_nix
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN_IN || secrets.GITHUB_TOKEN }}
+          cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: pre-build all flake outputs
+        run: |
+          nix build .#matrix.${{ matrix.system }}

.github/workflows/pr_release_artifacts.yml

@@ -12,10 +12,17 @@ on:
         default: false
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   create-release-artifacts:
     name: Create release artifacts for PR
     if: github.event_name == 'workflow_dispatch' && github.event.inputs.cleanup == 'false'
     runs-on: ubuntu-24.04
+    needs: nix-build
     permissions:
       pull-requests: write
       issues: write

.github/workflows/release.yml

@@ -119,10 +119,16 @@ jobs:
             exit 1
           fi
 
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   update-main:
     name: Update main branch
     runs-on: ubuntu-24.04
-    needs: [process-inputs, release]
+    needs: [process-inputs, release, nix-build]
     permissions:
       contents: write
     env:
@@ -220,7 +226,7 @@ jobs:
   release:
     name: Build and push artifacts, create release
     runs-on: ubuntu-24.04
-    needs: process-inputs
+    needs: [process-inputs, nix-build]
     permissions:
       contents: write
       packages: write
@@ -311,7 +317,7 @@ jobs:
       # Job needs content:write to see draft releases.
       contents: write
       packages: read
-    needs: [process-inputs, release]
+    needs: [process-inputs, release, nix-build]
     env:
       VERSION: ${{ inputs.version }}
     steps:

.github/workflows/rim_updates.yml

@@ -6,9 +6,16 @@ on:
     - cron: "0 18 * * 0"
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   update-rim-ids:
     name: update NVIDIA RIM IDs
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 20
     permissions:
       contents: read

.github/workflows/static.yml

@@ -8,8 +8,15 @@ on:
   pull_request:
 
 jobs:
+  nix-build:
+    uses: ./.github/workflows/nix_build.yml
+    secrets:
+      GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }}
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
   flake-check:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read
@@ -27,6 +34,7 @@ jobs:
 
   generate:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 60
     permissions:
       contents: write
@@ -52,6 +60,7 @@ jobs:
 
   govulncheck:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read
@@ -69,6 +78,7 @@ jobs:
 
   golangci-lint:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read
@@ -86,6 +96,7 @@ jobs:
 
   go-licenses:
     runs-on: ubuntu-24.04
+    needs: nix-build
     timeout-minutes: 15
     permissions:
       contents: read

flake.nix

@@ -36,12 +36,13 @@
           overlays = [
             (final: _prev: { fenix = self.inputs.fenix.packages.${final.stdenv.hostPlatform.system}; })
             (import ./overlays/nixpkgs.nix)
-            (import ./overlays/contrast.nix)
+            (import ./overlays/contrast.nix { inherit (self) inputs; })
           ];
           config.allowUnfree = true;
           config.nvidia.acceptLicense = true;
         };
         treefmtEval = treefmt-nix.lib.evalModule pkgs ./treefmt.nix;
+        inherit (pkgs) lib;
       in
 
       {
@@ -57,6 +58,8 @@
             "contrastPkgs"
           ];
         };
+
+        matrix = pkgs.writeText "output-matrix.json" (builtins.toJSON (lib.ci.allOutputs self system));
       }
     );
 

lib/ci.nix

@@ -0,0 +1,22 @@
+{ lib, ... }:
+rec {
+  fromPackageOutputs =
+    flake: system:
+    lib.concatMap (kind: lib.attrValues (lib.attrByPath [ kind system ] { } flake)) [
+      "legacyPackages"
+      "packages"
+      "checks"
+      "devShells"
+      "formatters"
+    ];
+
+  allOutputs =
+    flake: system:
+    lib.filter lib.isDerivation (
+      lib.unique (
+        lib.concatMap (from: from flake system) [
+          fromPackageOutputs
+        ]
+      )
+    );
+}

lib/default.nix

@@ -0,0 +1,12 @@
+{ inputs }:
+
+let
+  mkLib =
+    nixpkgs:
+    nixpkgs.lib.extend (
+      self: _: {
+        ci = import ./ci.nix { lib = self; };
+      }
+    );
+in
+mkLib inputs.nixpkgs

overlays/contrast.nix

@@ -1,8 +1,11 @@
 # Copyright 2024 Edgeless Systems GmbH
 # SPDX-License-Identifier: BUSL-1.1
 
+{ inputs }:
+
 final: _prev:
 
 {
   contrastPkgs = import ../packages { pkgs = final; };
+  lib = import ../lib { inherit inputs; };
 }

packages/contrast-releases.json

@@ -1,9 +1,5 @@
 {
   "contrast": [
-    {
-      "version": "v0.2.0",
-      "hash": "sha256-cGyWvUmL/rbjX3Bu1OPGTZlk5PgVW/O0qdo9KmZPo+U="
-    },
     {
       "version": "v0.3.0",
       "hash": "sha256-8yVZLhwf2bsWIMxSAqXUKDNCjltCSGA6bwrXTi+cJB0="

packages/scripts.nix

@@ -809,4 +809,46 @@ lib.makeScope pkgs.newScope (scripts: {
       echo "$timestamp $digest"
     '';
   };
+
+  # Shows which derivations' dependency changes cause rebuilds
+  show-input-diff = writeShellApplication {
+    name = "show-input-diff";
+    runtimeInputs = with pkgs; [
+      nix-diff
+      jq
+    ];
+    text = ''
+      maxDepth=999
+      new_args=()
+      while [[ $# -gt 0 ]]; do
+          case $1 in
+              --max-depth)
+                  maxDepth="$2"
+                  shift 2
+                  ;;
+              --max-depth=*)
+                  maxDepth="''${1#*=}"
+                  shift
+                  ;;
+              *)
+                  new_args+=("$1")
+                  shift
+                  ;;
+          esac
+      done
+      set -- "''${new_args[@]}"
+
+      left=$(nix eval --raw "''${1:-"github:edgelesssys/contrast#matrix.x86_64-linux"}.drvPath" | tr -d '\n')
+      right=$(nix eval --raw "''${2:-".#matrix.x86_64-linux"}.drvPath" | tr -d '\n')
+      nix-diff "$left" "$right" --json | jq -r --argjson maxDepth "$maxDepth" '
+        def printTree(level):
+          (
+            select(.drvName != null and .drvName != "") | ("  " * level) + .drvName,
+            (.drvNames // [] | .[] | ("  " * (level + 1)) + .),
+            (if level + 1 < $maxDepth then (.drvDiff.inputsDiff.inputDerivationDiffs // [] | .[] | printTree(level + 1)) else empty end)
+          );
+        .inputsDiff.inputDerivationDiffs[] | printTree(0)
+      '
+    '';
+  };
 })