Merge master into staging-next

Author: nixpkgs-ci[bot]

.github/actions/checkout/action.yml

@@ -5,8 +5,6 @@ description: 'Checkout into trusted / untrusted / pinned folders consistently.'
 inputs:
   merged-as-untrusted-at:
     description: "Whether and which SHA to checkout for the merge commit in the ./untrusted folder."
-  pinned-from:
-    description: "Whether to checkout the pinned nixpkgs for CI and from where (trusted, untrusted)."
   target-as-trusted-at:
     description: "Whether and which SHA to checkout for the target commit in the ./trusted folder."
 
@@ -16,7 +14,6 @@ runs:
     - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
       env:
         MERGED_SHA: ${{ inputs.merged-as-untrusted-at }}
-        PINNED_FROM: ${{ inputs.pinned-from }}
         TARGET_SHA: ${{ inputs.target-as-trusted-at }}
       with:
         script: |
@@ -45,6 +42,7 @@ runs:
           // A single fetch call comes with a lot less overhead. The fetch takes essentially the
           // same time no matter whether its 1, 2 or 3 commits at once.
           async function getPinnedSha(ref) {
+            if (!ref) return undefined
             const { content, encoding } = (await github.rest.repos.getContent({
               ...context.repo,
               path: 'ci/pinned.json',
@@ -60,16 +58,16 @@ runs:
               path: 'untrusted',
             },
             {
-              sha: process.env.PINNED_FROM === 'untrusted' && (await getPinnedSha(process.env.MERGED_SHA)),
-              path: 'pinned'
+              sha: await getPinnedSha(process.env.MERGED_SHA),
+              path: 'untrusted-pinned'
             },
             {
               sha: process.env.TARGET_SHA,
               path: 'trusted',
             },
             {
-              sha: process.env.PINNED_FROM === 'trusted' && (await getPinnedSha(process.env.TARGET_SHA)),
-              path: 'pinned'
+              sha: await getPinnedSha(process.env.TARGET_SHA),
+              path: 'trusted-pinned'
             }
           ].filter(({ sha }) => Boolean(sha))
 
@@ -79,22 +77,20 @@ runs:
           // This would fail without --refetch, because the we had a partial clone before, but changed it above.
           await run('git', 'fetch', '--depth=1', '--refetch', 'origin', ...(commits.map(({ sha }) => sha)))
 
+          // Checking out onto tmpfs takes 1s and is faster by at least factor 10x.
+          await run('mkdir', 'nixpkgs')
+          switch (process.env.RUNNER_OS) {
+            case 'macOS':
+              await run('sudo', 'mount_tmpfs', 'nixpkgs')
+              break
+            case 'Linux':
+              await run('sudo', 'mount', '-t', 'tmpfs', 'tmpfs', 'nixpkgs')
+              break
+          }
+
           // Create all worktrees in parallel.
           await Promise.all(commits.map(async ({ sha, path }) => {
-            // Checking out onto tmpfs takes 1s and is faster by at least factor 10x.
-            await run('mkdir', path)
-            switch (process.env.RUNNER_OS) {
-              case 'macOS':
-                await run('sudo', 'mount_tmpfs', path)
-                // macOS creates this hidden folder to log file system activity.
-                // This trips up git when adding a worktree below, because the target folder is not empty.
-                await run('sudo', 'rm', '-rf', join(path, '.fseventsd'))
-                break
-              case 'Linux':
-                await run('sudo', 'mount', '-t', 'tmpfs', 'tmpfs', path)
-                break
-            }
-            await run('git', 'worktree', 'add', path, sha, '--no-checkout', '--force')
-            await run('git', '-C', path, 'sparse-checkout', 'disable')
-            await run('git', '-C', path, 'checkout', '--progress')
+            await run('git', 'worktree', 'add', join('nixpkgs', path), sha, '--no-checkout')
+            await run('git', '-C', join('nixpkgs', path), 'sparse-checkout', 'disable')
+            await run('git', '-C', join('nixpkgs', path), 'checkout', '--progress')
           }))

.github/workflows/build.yml

@@ -51,7 +51,6 @@ jobs:
         uses: ./.github/actions/checkout
         with:
           merged-as-untrusted-at: ${{ inputs.mergedSha }}
-          pinned-from: untrusted
 
       - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
         with:
@@ -66,33 +65,33 @@ jobs:
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
           pushFilter: '(-source$|-nixpkgs-tarball-)'
 
-      - run: nix-env --install -f pinned -A nix-build-uncached
+      - run: nix-env --install -f nixpkgs/untrusted-pinned -A nix-build-uncached
 
       - name: Build shell
         if: contains(matrix.builds, 'shell')
-        run: echo "${{ matrix.systems }}" | xargs -n1 nix-build-uncached untrusted/ci --arg nixpkgs ./pinned -A shell --argstr system
+        run: echo "${{ matrix.systems }}" | xargs -n1 nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A shell --argstr system
 
       - name: Build NixOS manual
         if: |
           contains(matrix.builds, 'manual-nixos') && !cancelled() &&
           contains(fromJSON(inputs.baseBranch).type, 'primary')
-        run: nix-build-uncached untrusted/ci --arg nixpkgs ./pinned -A manual-nixos --out-link nixos-manual
+        run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A manual-nixos --out-link nixos-manual
 
       - name: Build Nixpkgs manual
         if: contains(matrix.builds, 'manual-nixpkgs') && !cancelled()
-        run: nix-build-uncached untrusted/ci --arg nixpkgs ./pinned -A manual-nixpkgs -A manual-nixpkgs-tests
+        run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A manual-nixpkgs -A manual-nixpkgs-tests
 
       - name: Build Nixpkgs manual tests
         if: contains(matrix.builds, 'manual-nixpkgs-tests') && !cancelled()
-        run: nix-build-uncached untrusted/ci --arg nixpkgs ./pinned -A manual-nixpkgs-tests
+        run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A manual-nixpkgs-tests
 
       - name: Build lib tests
         if: contains(matrix.builds, 'lib-tests') && !cancelled()
-        run: nix-build-uncached untrusted/ci --arg nixpkgs ./pinned -A lib-tests
+        run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A lib-tests
 
       - name: Build tarball
         if: contains(matrix.builds, 'tarball') && !cancelled()
-        run: nix-build-uncached untrusted/ci --arg nixpkgs ./pinned -A tarball
+        run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A tarball
 
       - name: Upload NixOS manual
         if: |

.github/workflows/check.yml

@@ -103,7 +103,6 @@ jobs:
         uses: ./.github/actions/checkout
         with:
           merged-as-untrusted-at: ${{ inputs.mergedSha }}
-          pinned-from: trusted
           target-as-trusted-at: ${{ inputs.targetSha }}
 
       - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
@@ -117,7 +116,7 @@ jobs:
           pushFilter: -source$
 
       - name: Build codeowners validator
-        run: nix-build trusted/ci --arg nixpkgs ./pinned -A codeownersValidator
+        run: nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A codeownersValidator
 
       - uses: actions/create-github-app-token@0f859bf9e69e887678d5bbfbee594437cb440ffe # v2.1.0
         if: github.event_name == 'pull_request_target' && vars.OWNER_RO_APP_ID
@@ -137,9 +136,9 @@ jobs:
       - name: Validate codeowners
         if: steps.app-token.outputs.token
         env:
-          OWNERS_FILE: untrusted/ci/OWNERS
+          OWNERS_FILE: nixpkgs/untrusted/ci/OWNERS
           GITHUB_ACCESS_TOKEN: ${{ steps.app-token.outputs.token }}
-          REPOSITORY_PATH: untrusted
+          REPOSITORY_PATH: nixpkgs/untrusted
           OWNER_CHECKER_REPOSITORY: ${{ github.repository }}
           # Set this to "notowned,avoid-shadowing" to check that all files are owned by somebody
           EXPERIMENTAL_CHECKS: "avoid-shadowing"

.github/workflows/eval.yml

@@ -71,8 +71,6 @@ jobs:
     # to not interrupt main Eval's compare step.
     continue-on-error: ${{ matrix.version != '' }}
     name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
-    outputs:
-      targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
     timeout-minutes: 15
     steps:
       # This is not supposed to be used and just acts as a fallback.
@@ -89,11 +87,11 @@ jobs:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: .github/actions
-      - name: Check out the PR at the test merge commit
+      - name: Check out the PR at merged and target commits
         uses: ./.github/actions/checkout
         with:
           merged-as-untrusted-at: ${{ inputs.mergedSha }}
-          pinned-from: untrusted
+          target-as-trusted-at: ${{ inputs.targetSha }}
 
       - name: Install Nix
         uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
@@ -106,98 +104,64 @@ jobs:
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
           pushFilter: '(-source|-single-chunk)$'
 
-      - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
+      - name: Evaluate the ${{ matrix.system }} output paths at the merge commit
         env:
           MATRIX_SYSTEM: ${{ matrix.system }}
           MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
         run: |
-          nix-build untrusted/ci --arg nixpkgs ./pinned  -A eval.singleSystem \
+          nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned  -A eval.singleSystem \
             --argstr evalSystem "$MATRIX_SYSTEM" \
             --arg chunkSize 8000 \
             --argstr nixPath "$MATRIX_VERSION" \
             --out-link merged
-          # If it uses too much memory, slightly decrease chunkSize
-
-      - name: Upload the output paths and eval stats
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }}
-          path: merged/*
-
-      - name: Log current API rate limits
+          # If it uses too much memory, slightly decrease chunkSize.
+          # Note: Keep the same further down in sync!
+
+      # Running the attrpath generation step separately from the outpath step afterwards.
+      # The idea is that, *if* Eval on the target branch has not finished, yet, we will
+      # generate the attrpaths in the meantime - and the separate command command afterwards
+      # will check cachix again for whether Eval has finished. If no Eval result from the
+      # target branch can be found the second time, we proceed to run it in here. Attrpaths
+      # generation takes roughly 30 seconds, so for every normal use-case this should be more
+      # than enough of a head start for Eval on the target branch to finish.
+      # This edge-case, that Eval on the target branch is delayed is unlikely to happen anyway:
+      # For a commit to become the target commit of a PR, it must *already* be on the branch.
+      # Normally, CI should always start running on that push event *before* it starts running
+      # on the PR.
+      - name: Evaluate the ${{ matrix.system }} attribute paths at the target commit
+        if: inputs.targetSha
         env:
-          GH_TOKEN: ${{ github.token }}
-        run: gh api /rate_limit | jq
+          MATRIX_SYSTEM: ${{ matrix.system }}
+        run: |
+          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.attrpathsSuperset \
+            --argstr evalSystem "$MATRIX_SYSTEM" \
+            --argstr nixPath "nixVersions.latest"
 
-      - name: Get target run id
+      - name: Evaluate the ${{ matrix.system }} output paths at the target commit
         if: inputs.targetSha
-        id: targetRunId
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         env:
           MATRIX_SYSTEM: ${{ matrix.system }}
-          TARGET_SHA: ${{ inputs.targetSha }}
-        with:
-          script: |
-            const system = process.env.MATRIX_SYSTEM
-            const targetSha = process.env.TARGET_SHA
-
-            let run_id
-            try {
-              run_id = (await github.rest.actions.listWorkflowRuns({
-                ...context.repo,
-                workflow_id: 'push.yml',
-                event: 'push',
-                head_sha: targetSha
-              })).data.workflow_runs[0].id
-            } catch {
-              throw new Error(`Could not find a push.yml workflow run for ${targetSha}.`)
-            }
-
-            // Waiting 120 * 5 sec = 10 min. max.
-            // Eval takes max 5-6 minutes, normally.
-            for (let i = 0; i < 120; i++) {
-              const result = await github.rest.actions.listWorkflowRunArtifacts({
-                ...context.repo,
-                run_id,
-                name: `merged-${system}`
-              })
-              if (result.data.total_count > 0) {
-                core.setOutput('targetRunId', run_id)
-                return
-              }
-              await new Promise(resolve => setTimeout(resolve, 5000))
-            }
-            // No artifact found at this stage. This usually means that Eval failed on the target branch.
-            // This should only happen when Eval is broken on the target branch and this PR fixes it.
-            // Continue without targetRunId to skip the remaining steps, but pass the job.
-
-      - name: Log current API rate limits
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: gh api /rate_limit | jq
-
-      - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
-        if: steps.targetRunId.outputs.targetRunId
-        with:
-          run-id: ${{ steps.targetRunId.outputs.targetRunId }}
-          name: merged-${{ matrix.system }}
-          path: target
-          github-token: ${{ github.token }}
-          merge-multiple: true
+        # This should be very quick, because it pulls the eval results from Cachix.
+        run: |
+          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
+            --argstr evalSystem "$MATRIX_SYSTEM" \
+            --arg chunkSize 8000 \
+            --argstr nixPath "nixVersions.latest" \
+            --out-link target
 
       - name: Compare outpaths against the target branch
-        if: steps.targetRunId.outputs.targetRunId
+        if: inputs.targetSha
         env:
           MATRIX_SYSTEM: ${{ matrix.system }}
         run: |
-          nix-build untrusted/ci --arg nixpkgs ./pinned -A eval.diff \
-            --arg beforeDir ./target \
+          nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
+            --arg beforeDir "$(readlink ./target)" \
             --arg afterDir "$(readlink ./merged)" \
             --argstr evalSystem "$MATRIX_SYSTEM" \
             --out-link diff
 
       - name: Upload outpaths diff and stats
-        if: steps.targetRunId.outputs.targetRunId
+        if: inputs.targetSha
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
@@ -206,7 +170,7 @@ jobs:
   compare:
     runs-on: ubuntu-24.04-arm
     needs: [eval]
-    if: needs.eval.outputs.targetRunId && !cancelled() && !failure()
+    if: inputs.targetSha && !cancelled() && !failure()
     permissions:
       statuses: write
     timeout-minutes: 5
@@ -219,7 +183,6 @@ jobs:
         with:
           merged-as-untrusted-at: ${{ inputs.mergedSha }}
           target-as-trusted-at: ${{ inputs.targetSha }}
-          pinned-from: trusted
 
       - name: Download output paths and eval stats for all systems
         uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
@@ -233,19 +196,19 @@ jobs:
 
       - name: Combine all output paths and eval stats
         run: |
-          nix-build trusted/ci --arg nixpkgs ./pinned -A eval.combine \
+          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
             --arg diffDir ./diff \
             --out-link combined
 
       - name: Compare against the target branch
         env:
           AUTHOR_ID: ${{ github.event.pull_request.user.id }}
         run: |
-          git -C trusted diff --name-only ${{ inputs.mergedSha }} \
+          git -C nixpkgs/trusted diff --name-only ${{ inputs.mergedSha }} \
             | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
 
           # Use the target branch to get accurate maintainer info
-          nix-build trusted/ci --arg nixpkgs ./pinned -A eval.compare \
+          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
             --arg combinedDir "$(realpath ./combined)" \
             --arg touchedFilesJson ./touched-files.json \
             --argstr githubAuthorId "$AUTHOR_ID" \
@@ -392,8 +355,8 @@ jobs:
         uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
 
       - name: Ensure flake outputs on all systems still evaluate
-        run: nix flake check --all-systems --no-build './untrusted?shallow=1'
+        run: nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1'
 
       - name: Query nixpkgs with aliases enabled to check for basic syntax errors
         run: |
-          time nix-env -I ./untrusted -f ./untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null
+          time nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null