Skip to content

Commit 6d2d99e

Browse files
Mic92Mic92
authored
Parallel GH actions workflow for Nixpkgs eval (#356023)
2 parents cbc70ce + fbbe972 commit 6d2d99e

File tree

12 files changed

+510
-87
lines changed

12 files changed

+510
-87
lines changed

‎.github/workflows/eval.yml‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
name: Eval
2+
3+
on: pull_request_target
4+
5+
permissions:
6+
contents: read
7+
8+
jobs:
9+
attrs:
10+
name: Attributes
11+
runs-on: ubuntu-latest
12+
outputs:
13+
mergedSha: ${{ steps.merged.outputs.mergedSha }}
14+
systems: ${{ steps.systems.outputs.systems }}
15+
steps:
16+
# Important: Because of `pull_request_target`, this doesn't check out the PR,
17+
# but rather the base branch of the PR, which is needed so we don't run untrusted code
18+
- name: Check out the ci directory of the base branch
19+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
20+
with:
21+
path: base
22+
sparse-checkout: ci
23+
- name: Check if the PR can be merged and get the test merge commit
24+
id: merged
25+
env:
26+
GH_TOKEN: ${{ github.token }}
27+
run: |
28+
if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
29+
echo "Checking the merge commit $mergedSha"
30+
echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
31+
else
32+
# Skipping so that no notifications are sent
33+
echo "Skipping the rest..."
34+
fi
35+
rm -rf base
36+
- name: Check out the PR at the test merge commit
37+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
38+
# Add this to _all_ subsequent steps to skip them
39+
if: steps.merged.outputs.mergedSha
40+
with:
41+
ref: ${{ env.mergedSha }}
42+
path: nixpkgs
43+
44+
- name: Install Nix
45+
uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
46+
if: steps.merged.outputs.mergedSha
47+
48+
- name: Evaluate the list of all attributes and get the systems matrix
49+
id: systems
50+
if: steps.merged.outputs.mergedSha
51+
run: |
52+
nix-build nixpkgs/ci -A eval.attrpathsSuperset
53+
echo "systems=$(<result/systems.json)" >> "$GITHUB_OUTPUT"
54+
55+
- name: Upload the list of all attributes
56+
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
57+
if: steps.merged.outputs.mergedSha
58+
with:
59+
name: paths
60+
path: result/*
61+
62+
outpaths:
63+
name: Outpaths
64+
runs-on: ubuntu-latest
65+
needs: attrs
66+
# Skip this and future steps if the PR can't be merged
67+
if: needs.attrs.outputs.mergedSha
68+
strategy:
69+
matrix:
70+
system: ${{ fromJSON(needs.attrs.outputs.systems) }}
71+
steps:
72+
- name: Download the list of all attributes
73+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
74+
with:
75+
name: paths
76+
path: paths
77+
78+
- name: Check out the PR at the test merge commit
79+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
80+
with:
81+
ref: ${{ needs.attrs.outputs.mergedSha }}
82+
path: nixpkgs
83+
84+
- name: Install Nix
85+
uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
86+
87+
- name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
88+
run: |
89+
nix-build nixpkgs/ci -A eval.singleSystem \
90+
--argstr evalSystem ${{ matrix.system }} \
91+
--arg attrpathFile ./paths/paths.json \
92+
--arg chunkSize 10000
93+
# If it uses too much memory, slightly decrease chunkSize
94+
95+
- name: Upload the output paths and eval stats
96+
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
97+
if: needs.attrs.outputs.mergedSha
98+
with:
99+
name: intermediate-${{ matrix.system }}
100+
path: result/*
101+
102+
process:
103+
name: Process
104+
runs-on: ubuntu-latest
105+
needs: outpaths
106+
steps:
107+
- name: Download output paths and eval stats for all systems
108+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
109+
with:
110+
pattern: intermediate-*
111+
path: intermediate
112+
113+
- name: Check out the PR at the test merge commit
114+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
115+
with:
116+
ref: ${{ needs.attrs.outputs.mergedSha }}
117+
path: nixpkgs
118+
119+
- name: Install Nix
120+
uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
121+
122+
- name: Combine all output paths and eval stats
123+
run: |
124+
nix-build nixpkgs/ci -A eval.combine \
125+
--arg resultsDir ./intermediate
126+
127+
- name: Upload the combined results
128+
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
129+
with:
130+
name: result
131+
path: result/*
132+
133+
134+
# TODO: Run this workflow also on `push` (on at least the main development branches)
135+
# Then add an extra step here that waits for the base branch (not the merge base, because that could be very different)
136+
# to have completed the eval, then use
137+
# gh api --method GET /repos/NixOS/nixpkgs/actions/workflows/eval.yml/runs -f head_sha=<BASE>
138+
# and follow it to the artifact results, where you can then download the outpaths.json from the base branch
139+
# That can then be used to compare the number of changed paths, get evaluation stats and ping appropriate reviewers

‎ci/default.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,5 @@ in
2626
inherit pkgs;
2727
requestReviews = pkgs.callPackage ./request-reviews { };
2828
codeownersValidator = pkgs.callPackage ./codeowners-validator { };
29+
eval = pkgs.callPackage ./eval { };
2930
}

‎ci/eval/README.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Nixpkgs CI evaluation
2+
3+
The code in this directory is used by the [eval.yml](../../.github/workflows/eval.yml) GitHub Actions workflow to evaluate the majority of Nixpkgs for all PRs, effectively making sure that when the development branches are processed by Hydra, no evaluation failures are encountered.
4+
5+
Furthermore it also allows local evaluation using
6+
```
7+
nix-build ci -A eval.full \
8+
--max-jobs 4
9+
--cores 2
10+
--arg chunkSize 10000
11+
```
12+
13+
- `--max-jobs`: The maximum number of derivations to run at the same time. Only each [supported system](../supportedSystems.nix) gets a separate derivation, so it doesn't make sense to set this higher than that number.
14+
- `--cores`: The number of cores to use for each job. Recommended to set this to the amount of cores on your system divided by `--max-jobs`.
15+
- `chunkSize`: The number of attributes that are evaluated simultaneously on a single core. Lowering this decreases memory usage at the cost of increased evaluation time. If this is too high, there won't be enough chunks to process them in parallel, and will also increase evaluation time.
16+
17+
A good default is to set `chunkSize` to 10000, which leads to about 3.6GB max memory usage per core, so suitable for fully utilising machines with 4 cores and 16GB memory, 8 cores and 32GB memory or 16 cores and 64GB memory.
18+
19+
Note that 16GB memory is the recommended minimum, while with less than 8GB memory evaluation time suffers greatly.

0 commit comments

Comments
 (0)