Skip to content

Commit 728223a

Browse files
jfrochejfroche
committed
fix(ci): simplify GitHub Actions workflow for Nix builds
Create a single nix-eval job to determine packages to build, removing redundant extension and check matrices.
1 parent dd4f250 commit 728223a

File tree

3 files changed

+40
-142
lines changed

3 files changed

+40
-142
lines changed

‎.github/actionlint.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
self-hosted-runner:
2+
labels:
3+
- aarch64-darwin
4+
- aarch64-linux

‎.github/workflows/nix-build.yml‎

Lines changed: 18 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ permissions:
1515
packages: write
1616

1717
jobs:
18-
extensions-matrix:
18+
nix-eval:
1919
runs-on:
2020
group: self-hosted-runners-nix
2121
labels:
@@ -29,95 +29,17 @@ jobs:
2929
name: Generate Nix Matrix
3030
run: |
3131
set -Eeu
32-
echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT"
32+
echo matrix="$(python scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT"
3333
34-
build-extensions-aarch64-linux:
35-
name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux)
36-
needs: extensions-matrix
37-
runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
38-
if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux != null }}
39-
strategy:
40-
fail-fast: false
41-
max-parallel: 5
42-
matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux }}
43-
steps:
44-
- name: Checkout Repo
45-
uses: actions/checkout@v4
46-
- name: Build Nix Package
47-
uses: ./.github/actions/nix-build-setup
48-
with:
49-
attr: ${{ matrix.attr }}
50-
51-
build-extensions-aarch64-darwin:
52-
name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin)
53-
needs: extensions-matrix
54-
runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
55-
if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin != null }}
56-
strategy:
57-
fail-fast: false
58-
max-parallel: 5
59-
matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin }}
60-
steps:
61-
- name: Checkout Repo
62-
uses: actions/checkout@v4
63-
- name: Build Nix Package
64-
uses: ./.github/actions/nix-build-setup
65-
with:
66-
attr: ${{ matrix.attr }}
67-
68-
# TODO
69-
# build-extensions-x86_64-linux:
70-
# name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux)
71-
# needs: extensions-matrix
72-
# runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
73-
# if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux != null }}
74-
# strategy:
75-
# fail-fast: false
76-
# max-parallel: 3
77-
# matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux }}
78-
# steps:
79-
# - name: Checkout Repo
80-
# uses: actions/checkout@v4
81-
# - name: Build Nix Package
82-
# uses: ./.github/actions/nix-build-setup
83-
# with:
84-
# attr: ${{ matrix.attr }}
85-
86-
87-
checks-matrix:
88-
needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux]
89-
if: |
90-
!cancelled() &&
91-
(needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') &&
92-
(needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success')
93-
runs-on:
94-
group: self-hosted-runners-nix
95-
labels:
96-
- aarch64-darwin
97-
outputs:
98-
matrix: ${{ steps.set-matrix.outputs.matrix }}
99-
steps:
100-
- name: Checkout Repo
101-
uses: actions/checkout@v4
102-
- id: set-matrix
103-
name: Generate Nix Matrix
104-
run: |
105-
set -Eeu
106-
echo matrix="$(python scripts/github-matrix.py checks)" >> "$GITHUB_OUTPUT"
107-
108-
build-checks-aarch64-linux:
34+
nix-build-aarch64-linux:
10935
name: ${{ matrix.name }} (aarch64-linux)
110-
needs: checks-matrix
36+
needs: nix-eval
11137
runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
112-
if: |
113-
!cancelled() &&
114-
${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux != null }} &&
115-
(needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') &&
116-
(needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success')
38+
if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux != null }}
11739
strategy:
11840
fail-fast: false
11941
max-parallel: 5
120-
matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux }}
42+
matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux }}
12143
steps:
12244
- name: Checkout Repo
12345
uses: actions/checkout@v4
@@ -126,19 +48,15 @@ jobs:
12648
with:
12749
attr: ${{ matrix.attr }}
12850

129-
build-checks-aarch64-darwin:
51+
nix-build-aarch64-darwin:
13052
name: ${{ matrix.name }} (aarch64-darwin)
131-
needs: checks-matrix
53+
needs: nix-eval
13254
runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
133-
if: |
134-
!cancelled() &&
135-
${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin != null }} &&
136-
(needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') &&
137-
(needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success')
55+
if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin != null }}
13856
strategy:
13957
fail-fast: false
14058
max-parallel: 5
141-
matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin }}
59+
matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin }}
14260
steps:
14361
- name: Checkout Repo
14462
uses: actions/checkout@v4
@@ -148,15 +66,15 @@ jobs:
14866
attr: ${{ matrix.attr }}
14967

15068
# TODO
151-
# build-checks-x86_64-linux:
69+
# nix-build-x86_64-linux:
15270
# name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux)
153-
# needs: checks-matrix
71+
# needs: nix-eval
15472
# runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
155-
# if: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux != null }}
73+
# if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }}
15674
# strategy:
15775
# fail-fast: false
158-
# max-parallel: 5
159-
# matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux }}
76+
# max-parallel: 3
77+
# matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }}
16078
# steps:
16179
# - name: Checkout Repo
16280
# uses: actions/checkout@v4
@@ -166,9 +84,9 @@ jobs:
16684
# attr: ${{ matrix.attr }}
16785

16886
run-tests:
169-
needs: [build-checks-aarch64-linux, build-checks-aarch64-darwin] #, build-checks-x86_64-linux]
87+
needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux]
17088
if: |
17189
!cancelled() &&
172-
(needs.build-checks-aarch64-linux.result == 'skipped' || needs.build-checks-aarch64-linux.result == 'success') &&
173-
(needs.build-checks-aarch64-darwin.result == 'skipped' || needs.build-checks-aarch64-darwin.result == 'success')
90+
(needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') &&
91+
(needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success')
17492
uses: ./.github/workflows/test.yml

‎scripts/github-matrix.py‎

Lines changed: 18 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -71,24 +71,12 @@ class GitHubActionPackage(TypedDict):
7171
}
7272

7373

74-
def get_worker_count() -> int:
75-
"""Get optimal worker count based on CPU cores."""
76-
try:
77-
return max(1, int(os.cpu_count()))
78-
except (OSError, AttributeError):
79-
print(
80-
"Warning: Unable to get CPU count, using default max_workers=1",
81-
file=sys.stderr,
82-
)
83-
return 1
84-
85-
86-
def build_nix_eval_command(max_workers: int, target: str) -> List[str]:
74+
def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[str]:
8775
"""Build the nix-eval-jobs command with appropriate flags."""
8876
nix_eval_cmd = [
8977
"nix-eval-jobs",
9078
"--flake",
91-
f".#{target}",
79+
".",
9280
"--check-cache-status",
9381
"--force-recurse",
9482
"--quiet",
@@ -100,12 +88,14 @@ def build_nix_eval_command(max_workers: int, target: str) -> List[str]:
10088
"true",
10189
"--workers",
10290
str(max_workers),
91+
"--select",
92+
f"outputs: {{ inherit (outputs) {' '.join(flake_outputs)}; }}",
10393
]
10494
return nix_eval_cmd
10595

10696

10797
def parse_nix_eval_line(
108-
line: str, drv_paths: Set[str], target: str
98+
line: str, drv_paths: Set[str]
10999
) -> Optional[GitHubActionPackage]:
110100
"""Parse a single line of nix-eval-jobs output"""
111101
if not line.strip():
@@ -120,7 +110,7 @@ def parse_nix_eval_line(
120110
runs_on_config = BUILD_RUNNER_MAP[data["system"]]
121111

122112
return {
123-
"attr": f"{target}.{data['attr']}",
113+
"attr": f"{data['attr']}",
124114
"name": data["name"],
125115
"system": data["system"],
126116
"already_cached": data.get("cacheStatus") != "notBuilt",
@@ -134,22 +124,23 @@ def parse_nix_eval_line(
134124
return None
135125

136126

137-
def run_nix_eval_jobs(
138-
cmd: List[str], target: str
139-
) -> Generator[GitHubActionPackage, None, None]:
127+
def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, None]:
140128
"""Run nix-eval-jobs and yield parsed package data."""
141129
print(f"Running command: {' '.join(cmd)}", file=sys.stderr)
142130

143131
with subprocess.Popen(
144132
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
145133
) as process:
146-
drv_paths = set()
134+
drv_paths: Set[str] = set()
135+
assert process.stdout is not None # for mypy
136+
assert process.stderr is not None # for mypy
147137
for line in process.stdout:
148-
package = parse_nix_eval_line(line, drv_paths, target)
138+
package = parse_nix_eval_line(line, drv_paths)
149139
if package:
150140
yield package
151141

152-
if process.returncode and process.returncode != 0:
142+
process.wait()
143+
if process.returncode != 0:
153144
print("Error: Evaluation failed", file=sys.stderr)
154145
sys.stderr.write(process.stderr.read())
155146
sys.exit(process.returncode)
@@ -162,7 +153,7 @@ def is_extension_pkg(pkg: GitHubActionPackage) -> bool:
162153

163154

164155
# thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297
165-
def sort_pkgs_by_closures(jobs: list[GitHubActionPackage]) -> list[GitHubActionPackage]:
156+
def sort_pkgs_by_closures(jobs: List[GitHubActionPackage]) -> List[GitHubActionPackage]:
166157
sorted_jobs = []
167158

168159
# Prepare job dependencies
@@ -194,23 +185,16 @@ def main() -> None:
194185
description="Generate GitHub Actions matrix for Nix builds"
195186
)
196187
parser.add_argument(
197-
"target", choices=["checks", "extensions"], help="Type of matrix to generate"
188+
"flake_outputs", nargs="+", help="Nix flake outputs to evaluate"
198189
)
199190

200191
args = parser.parse_args()
201192

202-
max_workers = get_worker_count()
193+
max_workers: int = os.cpu_count() or 1
203194

204-
if args.target == "checks":
205-
flake_output = "checks"
206-
else:
207-
flake_output = "legacyPackages"
195+
cmd = build_nix_eval_command(max_workers, args.flake_outputs)
208196

209-
cmd = build_nix_eval_command(max_workers, flake_output)
210-
211-
gh_action_packages = sort_pkgs_by_closures(
212-
list(run_nix_eval_jobs(cmd, flake_output))
213-
)
197+
gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd)))
214198

215199
def clean_package_for_output(pkg: GitHubActionPackage) -> dict:
216200
"""Remove debug fields from package for final output"""
@@ -226,14 +210,6 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict:
226210
),
227211
}
228212

229-
if args.target == "extensions":
230-
# filter to only include extension packages and add postgresql_version field
231-
gh_action_packages = [
232-
{**pkg, "postgresql_version": pkg["attr"].split(".")[-3]}
233-
for pkg in gh_action_packages
234-
if is_extension_pkg(pkg)
235-
]
236-
237213
# Group packages by system
238214
grouped_by_system = defaultdict(list)
239215
for pkg in gh_action_packages:

0 commit comments

Comments
 (0)