Skip to content

Commit af1aa40

Browse files
infinisilinfinisil
committed
workflows/eval.yml: Run on dev branch pushes and apply rebuild labels
1 parent 0892479 commit af1aa40

File tree

4 files changed

+292
-16
lines changed

4 files changed

+292
-16
lines changed

‎.github/workflows/eval.yml‎

Lines changed: 120 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
name: Eval
22

3-
on: pull_request_target
3+
on:
4+
pull_request_target:
5+
push:
6+
# Keep this synced with ci/request-reviews/dev-branches.txt
7+
branches:
8+
- master
9+
- staging
10+
- release-*
11+
- staging-*
12+
- haskell-updates
13+
- python-updates
414

515
permissions:
616
contents: read
@@ -11,6 +21,7 @@ jobs:
1121
runs-on: ubuntu-latest
1222
outputs:
1323
mergedSha: ${{ steps.merged.outputs.mergedSha }}
24+
baseSha: ${{ steps.baseSha.outputs.baseSha }}
1425
systems: ${{ steps.systems.outputs.systems }}
1526
steps:
1627
# Important: Because of `pull_request_target`, this doesn't check out the PR,
@@ -24,23 +35,39 @@ jobs:
2435
id: merged
2536
env:
2637
GH_TOKEN: ${{ github.token }}
38+
GH_EVENT: ${{ github.event_name }}
2739
run: |
28-
if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
29-
echo "Checking the merge commit $mergedSha"
30-
echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
31-
else
32-
# Skipping so that no notifications are sent
33-
echo "Skipping the rest..."
34-
fi
40+
case "$GH_EVENT" in
41+
push)
42+
echo "mergedSha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
43+
;;
44+
pull_request_target)
45+
if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
46+
echo "Checking the merge commit $mergedSha"
47+
echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
48+
else
49+
# Skipping so that no notifications are sent
50+
echo "Skipping the rest..."
51+
fi
52+
;;
53+
esac
3554
rm -rf base
3655
- name: Check out the PR at the test merge commit
3756
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
3857
# Add this to _all_ subsequent steps to skip them
3958
if: steps.merged.outputs.mergedSha
4059
with:
4160
ref: ${{ steps.merged.outputs.mergedSha }}
61+
fetch-depth: 2
4262
path: nixpkgs
4363

64+
- name: Determine base commit
65+
if: github.event_name == 'pull_request_target' && steps.merged.outputs.mergedSha
66+
id: baseSha
67+
run: |
68+
baseSha=$(git -C nixpkgs rev-parse HEAD^1)
69+
echo "baseSha=$baseSha" >> "$GITHUB_OUTPUT"
70+
4471
- name: Install Nix
4572
uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
4673
if: steps.merged.outputs.mergedSha
@@ -105,6 +132,8 @@ jobs:
105132
name: Process
106133
runs-on: ubuntu-latest
107134
needs: [ outpaths, attrs ]
135+
outputs:
136+
baseRunId: ${{ steps.baseRunId.outputs.baseRunId }}
108137
steps:
109138
- name: Download output paths and eval stats for all systems
110139
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
@@ -124,18 +153,93 @@ jobs:
124153
- name: Combine all output paths and eval stats
125154
run: |
126155
nix-build nixpkgs/ci -A eval.combine \
127-
--arg resultsDir ./intermediate
156+
--arg resultsDir ./intermediate \
157+
-o prResult
128158
129159
- name: Upload the combined results
130160
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
131161
with:
132162
name: result
133-
path: result/*
163+
path: prResult/*
134164

165+
- name: Get base run id
166+
if: needs.attrs.outputs.baseSha
167+
id: baseRunId
168+
run: |
169+
# Get the latest eval.yml workflow run for the PR's base commit
170+
if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
171+
-f head_sha="$BASE_SHA" \
172+
--jq '.workflow_runs | sort_by(.run_started_at) | .[-1]') \
173+
|| [[ -z "$run" ]]; then
174+
echo "Could not find an eval.yml workflow run for $BASE_SHA, cannot make comparison"
175+
exit 0
176+
fi
177+
echo "Comparing against $(jq .html_url <<< "$run")"
178+
runId=$(jq .id <<< "$run")
179+
conclusion=$(jq -r .conclusion <<< "$run")
180+
181+
while [[ "$conclusion" == null ]]; do
182+
echo "Workflow not done, waiting 10 seconds before checking again"
183+
sleep 10
184+
conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion')
185+
done
186+
187+
if [[ "$conclusion" != "success" ]]; then
188+
echo "Workflow was not successful, cannot make comparison"
189+
exit 0
190+
fi
135191
136-
# TODO: Run this workflow also on `push` (on at least the main development branches)
137-
# Then add an extra step here that waits for the base branch (not the merge base, because that could be very different)
138-
# to have completed the eval, then use
139-
# gh api --method GET /repos/NixOS/nixpkgs/actions/workflows/eval.yml/runs -f head_sha=<BASE>
140-
# and follow it to the artifact results, where you can then download the outpaths.json from the base branch
141-
# That can then be used to compare the number of changed paths, get evaluation stats and ping appropriate reviewers
192+
echo "baseRunId=$runId" >> "$GITHUB_OUTPUT"
193+
env:
194+
REPOSITORY: ${{ github.repository }}
195+
BASE_SHA: ${{ needs.attrs.outputs.baseSha }}
196+
GH_TOKEN: ${{ github.token }}
197+
198+
- uses: actions/download-artifact@v4
199+
if: steps.baseRunId.outputs.baseRunId
200+
with:
201+
name: result
202+
path: baseResult
203+
github-token: ${{ github.token }}
204+
run-id: ${{ steps.baseRunId.outputs.baseRunId }}
205+
206+
- name: Compare against the base branch
207+
if: steps.baseRunId.outputs.baseRunId
208+
run: |
209+
nix-build nixpkgs/ci -A eval.compare \
210+
--arg beforeResultDir ./baseResult \
211+
--arg afterResultDir ./prResult \
212+
-o comparison
213+
214+
# TODO: Request reviews from maintainers for packages whose files are modified in the PR
215+
216+
- name: Upload the combined results
217+
if: steps.baseRunId.outputs.baseRunId
218+
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
219+
with:
220+
name: comparison
221+
path: comparison/*
222+
223+
# Separate job to have a very tightly scoped PR write token
224+
tag:
225+
name: Tag
226+
runs-on: ubuntu-latest
227+
needs: process
228+
if: needs.process.outputs.baseRunId
229+
permissions:
230+
pull-requests: write
231+
steps:
232+
- name: Download process result
233+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
234+
with:
235+
name: comparison
236+
path: comparison
237+
238+
- name: Tagging pull request
239+
run: |
240+
gh api \
241+
--method POST \
242+
/repos/${{ github.repository }}/issues/${{ github.event.number }}/labels \
243+
--input <(jq -c '{ labels: .labels }' comparison/changed-paths.json)
244+
env:
245+
GH_TOKEN: ${{ github.token }}

‎ci/eval/compare.jq‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
# Turns
2+
#
3+
# {
4+
# "hello.aarch64-linux": "a",
5+
# "hello.x86_64-linux": "b",
6+
# "hello.aarch64-darwin": "c",
7+
# "hello.x86_64-darwin": "d"
8+
# }
9+
#
10+
# into
11+
#
12+
# {
13+
# "hello": {
14+
# "linux": {
15+
# "aarch64": "a",
16+
# "x86_64": "b"
17+
# },
18+
# "darwin": {
19+
# "aarch64": "c",
20+
# "x86_64": "d"
21+
# }
22+
# }
23+
# }
24+
#
25+
# while filtering out any attribute paths that don't match this pattern
26+
def expand_system:
27+
to_entries
28+
| map(
29+
.key |= split(".")
30+
| select(.key | length > 1)
31+
| .double = (.key[-1] | split("-"))
32+
| select(.double | length == 2)
33+
)
34+
| group_by(.key[0:-1])
35+
| map(
36+
{
37+
key: .[0].key[0:-1] | join("."),
38+
value:
39+
group_by(.double[1])
40+
| map(
41+
{
42+
key: .[0].double[1],
43+
value: map(.key = .double[0]) | from_entries
44+
}
45+
)
46+
| from_entries
47+
})
48+
| from_entries
49+
;
50+
51+
# Transposes
52+
#
53+
# {
54+
# "a": [ "x", "y" ],
55+
# "b": [ "x" ],
56+
# }
57+
#
58+
# into
59+
#
60+
# {
61+
# "x": [ "a", "b" ],
62+
# "y": [ "a" ]
63+
# }
64+
def transpose:
65+
[
66+
to_entries[]
67+
| {
68+
key: .key,
69+
value: .value[]
70+
}
71+
]
72+
| group_by(.value)
73+
| map({
74+
key: .[0].value,
75+
value: map(.key)
76+
})
77+
| from_entries
78+
;
79+
80+
# Computes the key difference for two objects:
81+
# {
82+
# added: [ <keys only in the second object> ],
83+
# removed: [ <keys only in the first object> ],
84+
# changed: [ <keys with different values between the two objects> ],
85+
# }
86+
#
87+
def diff($before; $after):
88+
{
89+
added: $after | delpaths($before | keys | map([.])) | keys,
90+
removed: $before | delpaths($after | keys | map([.])) | keys,
91+
changed:
92+
$before
93+
| to_entries
94+
| map(
95+
$after."\(.key)" as $after2
96+
| select(
97+
# Filter out attributes that don't exist anymore
98+
($after2 != null)
99+
and
100+
# Filter out attributes that are the same as the new value
101+
(.value != $after2)
102+
)
103+
| .key
104+
)
105+
}
106+
;
107+
108+
($before[0] | expand_system) as $before
109+
| ($after[0] | expand_system) as $after
110+
| .attrdiff = diff($before; $after)
111+
| .rebuildsByKernel = (
112+
.attrdiff.changed
113+
| map({
114+
key: .,
115+
value: diff($before."\(.)"; $after."\(.)").changed
116+
})
117+
| from_entries
118+
| transpose
119+
)
120+
| .rebuildCountByKernel = (
121+
.rebuildsByKernel
122+
| with_entries(.value |= length)
123+
| pick(.linux, .darwin)
124+
| {
125+
linux: (.linux // 0),
126+
darwin: (.darwin // 0),
127+
}
128+
)
129+
| .labels = (
130+
.rebuildCountByKernel
131+
| to_entries
132+
| map(
133+
"10.rebuild-\(.key): " +
134+
if .value == 0 then
135+
"0"
136+
elif .value <= 10 then
137+
"1-10"
138+
elif .value <= 100 then
139+
"11-100"
140+
elif .value <= 500 then
141+
"101-500"
142+
elif .value <= 1000 then
143+
"501-1000"
144+
elif .value <= 2500 then
145+
"1001-2500"
146+
elif .value <= 5000 then
147+
"2501-5000"
148+
else
149+
"5000+"
150+
end
151+
)
152+
)

‎ci/eval/default.nix‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,24 @@ let
238238
jq -s from_entries > $out/stats.json
239239
'';
240240

241+
compare =
242+
{ beforeResultDir, afterResultDir }:
243+
runCommand "compare"
244+
{
245+
nativeBuildInputs = [
246+
jq
247+
];
248+
}
249+
''
250+
mkdir $out
251+
jq -n -f ${./compare.jq} \
252+
--slurpfile before ${beforeResultDir}/outpaths.json \
253+
--slurpfile after ${afterResultDir}/outpaths.json \
254+
> $out/changed-paths.json
255+
256+
# TODO: Compare eval stats
257+
'';
258+
241259
full =
242260
{
243261
# Whether to evaluate just a single system, by default all are evaluated
@@ -268,6 +286,7 @@ in
268286
attrpathsSuperset
269287
singleSystem
270288
combine
289+
compare
271290
# The above three are used by separate VMs in a GitHub workflow,
272291
# while the below is intended for testing on a single local machine
273292
full

‎ci/request-reviews/dev-branches.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Trusted development branches:
22
# These generally require PRs to update and are built by Hydra.
3+
# Keep this synced with the branches in .github/workflows/eval.yml
34
master
45
staging
56
release-*

0 commit comments

Comments
 (0)