Skip to content

Commit 5d63907

Browse files
jacobweinstockjacobweinstock
committed
restructure mkosi.output to separate kernel artifacts from tools:
Reorganise the mkosi.output directory so kernel artifacts (vmlinuz and modules) and userspace tools (containerd, runc, nerdctl, CNI plugins) live in cleanly separated, versioned paths. Before: mkosi.output/ extra-tree/{arch}/ # mixed: kernel modules + tools vmlinuz/{version}/{arch}/ # kernel image only After: mkosi.output/ tools/{arch}/ # tools only kernel/{version}/{arch}/ # kernel image + modules vmlinuz-{kver} modules/usr/lib/modules/{kver}/ Key changes: - Rename vmlinuz/ → kernel/ and store both vmlinuz and modules under kernel/{version}/{arch}/. The modules/ subtree mirrors a root filesystem layout (usr/lib/modules/{kver}/) so mkosi can consume it directly as --extra-tree. - Rename extra-tree/ → tools/. Now that kernel modules are stored separately, the directory only contains userspace tools, so the name should describe its content rather than leak mkosi implementation jargon. - Pass two --extra-tree= flags to mkosi (tools + modules) instead of one combined tree. - kernel.build() no longer wipes the tools directory on rebuild — it only cleans its own kernel/{version}/{arch}/ output. - Remove the .kernel-{arch}-version marker file — the kernel version is now encoded in the directory path itself. - Update CI workflow cache/artifact paths to match the new layout. Config property renames: vmlinuz_output → kernel_output extra_tree_output → tools_output (new) modules_output Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
1 parent a290a8a commit 5d63907

File tree

11 files changed

+149
-132
lines changed

11 files changed

+149
-132
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 24 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ permissions:
99
contents: read
1010
packages: write
1111

12+
env:
13+
KERNEL_VERSION: 6.18.16
14+
1215
jobs:
1316
# -------------------------------------------------------------------
1417
# Lint — ruff (lint + format) and pyright (type checking)
@@ -44,29 +47,22 @@ jobs:
4447
- name: Checkout code
4548
uses: actions/checkout@v6
4649

47-
- name: Detect kernel version
48-
run: |
49-
KERNEL_VERSION=$(sed -n 's/^DEFAULT_KERNEL_VERSION = "\(.*\)"/\1/p' captain/config.py)
50-
echo "KERNEL_VERSION=$KERNEL_VERSION" >> "$GITHUB_ENV"
51-
5250
- name: Check amd64 kernel cache
5351
id: amd64-cache
5452
uses: actions/cache/restore@v4
5553
with:
5654
path: |
57-
mkosi.output/extra-tree/amd64/usr/lib/modules
58-
mkosi.output/vmlinuz/${{ env.KERNEL_VERSION }}/amd64
59-
key: kernel-amd64-${{ hashFiles('kernel.configs/*', 'Dockerfile') }}
55+
mkosi.output/kernel/${{ env.KERNEL_VERSION }}/amd64
56+
key: kernel-amd64-${{ env.KERNEL_VERSION }}-${{ hashFiles('kernel.configs/*', 'Dockerfile') }}
6057
lookup-only: true
6158

6259
- name: Check arm64 kernel cache
6360
id: arm64-cache
6461
uses: actions/cache/restore@v4
6562
with:
6663
path: |
67-
mkosi.output/extra-tree/arm64/usr/lib/modules
68-
mkosi.output/vmlinuz/${{ env.KERNEL_VERSION }}/arm64
69-
key: kernel-arm64-${{ hashFiles('kernel.configs/*', 'Dockerfile') }}
64+
mkosi.output/kernel/${{ env.KERNEL_VERSION }}/arm64
65+
key: kernel-arm64-${{ env.KERNEL_VERSION }}-${{ hashFiles('kernel.configs/*', 'Dockerfile') }}
7066
lookup-only: true
7167

7268
# -------------------------------------------------------------------
@@ -101,11 +97,6 @@ jobs:
10197
- name: Load shared config
10298
run: cat .github/config.env >> "$GITHUB_ENV"
10399

104-
- name: Detect kernel version
105-
run: |
106-
KERNEL_VERSION=$(sed -n 's/^DEFAULT_KERNEL_VERSION = "\(.*\)"/\1/p' captain/config.py)
107-
echo "KERNEL_VERSION=$KERNEL_VERSION" >> "$GITHUB_ENV"
108-
109100
- name: Log in to GHCR
110101
uses: docker/login-action@v3
111102
with:
@@ -144,9 +135,8 @@ jobs:
144135
uses: actions/cache/restore@v4
145136
with:
146137
path: |
147-
mkosi.output/extra-tree/${{ matrix.arch }}/usr/lib/modules
148-
mkosi.output/vmlinuz/${{ env.KERNEL_VERSION }}/${{ matrix.arch }}
149-
key: kernel-${{ matrix.arch }}-${{ hashFiles('kernel.configs/*', 'Dockerfile') }}
138+
mkosi.output/kernel/${{ env.KERNEL_VERSION }}/${{ matrix.arch }}
139+
key: kernel-${{ matrix.arch }}-${{ env.KERNEL_VERSION }}-${{ hashFiles('kernel.configs/*', 'Dockerfile') }}
150140

151141
- name: Install Python dependencies
152142
run: pip install -r requirements.txt
@@ -162,17 +152,15 @@ jobs:
162152
uses: actions/cache/save@v4
163153
with:
164154
path: |
165-
mkosi.output/extra-tree/${{ matrix.arch }}/usr/lib/modules
166-
mkosi.output/vmlinuz/${{ env.KERNEL_VERSION }}/${{ matrix.arch }}
167-
key: kernel-${{ matrix.arch }}-${{ hashFiles('kernel.configs/*', 'Dockerfile') }}
155+
mkosi.output/kernel/${{ env.KERNEL_VERSION }}/${{ matrix.arch }}
156+
key: kernel-${{ matrix.arch }}-${{ env.KERNEL_VERSION }}-${{ hashFiles('kernel.configs/*', 'Dockerfile') }}
168157

169158
- name: Upload kernel artifacts
170159
uses: actions/upload-artifact@v4
171160
with:
172161
name: kernel-${{ matrix.arch }}
173162
path: |
174-
mkosi.output/extra-tree/${{ matrix.arch }}/usr/lib/modules
175-
mkosi.output/vmlinuz/${{ env.KERNEL_VERSION }}/${{ matrix.arch }}
163+
mkosi.output/kernel/${{ env.KERNEL_VERSION }}/${{ matrix.arch }}
176164
retention-days: 1
177165

178166
# -------------------------------------------------------------------
@@ -199,8 +187,8 @@ jobs:
199187
uses: actions/cache/restore@v4
200188
with:
201189
path: |
202-
mkosi.output/extra-tree/${{ matrix.arch }}/usr/local/bin
203-
mkosi.output/extra-tree/${{ matrix.arch }}/opt/cni
190+
mkosi.output/tools/${{ matrix.arch }}/usr/local/bin
191+
mkosi.output/tools/${{ matrix.arch }}/opt/cni
204192
key: tools-${{ matrix.arch }}-${{ hashFiles('captain/tools.py') }}
205193

206194
- name: Install Python dependencies
@@ -214,17 +202,17 @@ jobs:
214202
uses: actions/cache/save@v4
215203
with:
216204
path: |
217-
mkosi.output/extra-tree/${{ matrix.arch }}/usr/local/bin
218-
mkosi.output/extra-tree/${{ matrix.arch }}/opt/cni
205+
mkosi.output/tools/${{ matrix.arch }}/usr/local/bin
206+
mkosi.output/tools/${{ matrix.arch }}/opt/cni
219207
key: tools-${{ matrix.arch }}-${{ hashFiles('captain/tools.py') }}
220208

221209
- name: Upload tools artifacts
222210
uses: actions/upload-artifact@v4
223211
with:
224212
name: tools-${{ matrix.arch }}
225213
path: |
226-
mkosi.output/extra-tree/${{ matrix.arch }}/usr/local/bin
227-
mkosi.output/extra-tree/${{ matrix.arch }}/opt/cni
214+
mkosi.output/tools/${{ matrix.arch }}/usr/local/bin
215+
mkosi.output/tools/${{ matrix.arch }}/opt/cni
228216
retention-days: 1
229217

230218
# -------------------------------------------------------------------
@@ -249,13 +237,13 @@ jobs:
249237
uses: actions/download-artifact@v4
250238
with:
251239
name: kernel-${{ matrix.arch }}
252-
path: mkosi.output
240+
path: mkosi.output/kernel/${{ env.KERNEL_VERSION }}/${{ matrix.arch }}
253241

254242
- name: Download tools artifacts
255243
uses: actions/download-artifact@v4
256244
with:
257245
name: tools-${{ matrix.arch }}
258-
path: mkosi.output/extra-tree/${{ matrix.arch }}
246+
path: mkosi.output/tools/${{ matrix.arch }}
259247

260248
- name: Refresh apt cache
261249
run: sudo apt-get update
@@ -303,11 +291,6 @@ jobs:
303291
- name: Load shared config
304292
run: cat .github/config.env >> "$GITHUB_ENV"
305293

306-
- name: Detect kernel version
307-
run: |
308-
KERNEL_VERSION=$(sed -n 's/^DEFAULT_KERNEL_VERSION = "\(.*\)"/\1/p' captain/config.py)
309-
echo "KERNEL_VERSION=$KERNEL_VERSION" >> "$GITHUB_ENV"
310-
311294
- name: Log in to GHCR
312295
uses: docker/login-action@v3
313296
with:
@@ -345,7 +328,7 @@ jobs:
345328
uses: actions/download-artifact@v4
346329
with:
347330
name: kernel-${{ matrix.arch }}
348-
path: mkosi.output
331+
path: mkosi.output/kernel/${{ env.KERNEL_VERSION }}/${{ matrix.arch }}
349332

350333
- name: Download initramfs artifacts
351334
uses: actions/download-artifact@v4
@@ -399,7 +382,7 @@ jobs:
399382
uses: actions/download-artifact@v4
400383
with:
401384
name: kernel-${{ matrix.target }}
402-
path: mkosi.output
385+
path: mkosi.output/kernel/${{ env.KERNEL_VERSION }}/${{ matrix.target }}
403386

404387
- name: Download initramfs artifacts
405388
uses: actions/download-artifact@v4
@@ -449,7 +432,7 @@ jobs:
449432
uses: actions/download-artifact@v4
450433
with:
451434
name: kernel-amd64
452-
path: mkosi.output
435+
path: mkosi.output/kernel/${{ env.KERNEL_VERSION }}/amd64
453436

454437
- name: Download initramfs artifacts (amd64)
455438
uses: actions/download-artifact@v4
@@ -467,7 +450,7 @@ jobs:
467450
uses: actions/download-artifact@v4
468451
with:
469452
name: kernel-arm64
470-
path: mkosi.output
453+
path: mkosi.output/kernel/${{ env.KERNEL_VERSION }}/arm64
471454

472455
- name: Download initramfs artifacts (arm64)
473456
uses: actions/download-artifact@v4

‎captain/artifacts.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,18 +32,18 @@ def _human_size(size: int) -> str:
3232

3333

3434
def collect_kernel(cfg: Config, logger: StageLogger | None = None) -> None:
35-
"""Copy the kernel image from mkosi.output/vmlinuz/{arch}/ to out/."""
35+
"""Copy the kernel image from mkosi.output/kernel/{version}/{arch}/ to out/."""
3636
_log = logger or _default_log
3737
out = ensure_dir(cfg.output_dir)
38-
vmlinuz_dir = cfg.vmlinuz_output
38+
vmlinuz_dir = cfg.kernel_output
3939
vmlinuz_files = sorted(vmlinuz_dir.glob("vmlinuz-*")) if vmlinuz_dir.is_dir() else []
4040
if vmlinuz_files:
4141
vmlinuz_src = vmlinuz_files[0]
4242
vmlinuz_dst = out / f"vmlinuz-{cfg.kernel_version}-{cfg.arch}"
4343
shutil.copy2(vmlinuz_src, vmlinuz_dst)
4444
_log.log(f"kernel: {vmlinuz_dst} ({_human_size(vmlinuz_dst.stat().st_size)})")
4545
else:
46-
_log.warn(f"No kernel image found in {cfg.vmlinuz_output}")
46+
_log.warn(f"No kernel image found in {cfg.kernel_output}")
4747

4848

4949
def collect_initramfs(cfg: Config, logger: StageLogger | None = None) -> None:

‎captain/cli.py‎

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ def _format_usage(
9090
# Used by _extract_command to avoid treating a flag value as a subcommand.
9191
_BOOLEAN_FLAGS = frozenset(
9292
{
93+
"--all",
9394
"--force-kernel",
9495
"--force-tools",
9596
"--force-iso",
@@ -621,8 +622,8 @@ def _build_kernel_stage(cfg: Config) -> None:
621622
return
622623

623624
# --- idempotency --------------------------------------------------
624-
modules_dir = cfg.extra_tree_output / "usr" / "lib" / "modules"
625-
vmlinuz_dir = cfg.vmlinuz_output
625+
modules_dir = cfg.modules_output / "usr" / "lib" / "modules"
626+
vmlinuz_dir = cfg.kernel_output
626627
has_vmlinuz = vmlinuz_dir.is_dir() and any(vmlinuz_dir.glob("vmlinuz-*"))
627628

628629
if modules_dir.is_dir() and has_vmlinuz and not cfg.force_kernel:
@@ -658,8 +659,7 @@ def _build_kernel_stage(cfg: Config) -> None:
658659
cfg,
659660
klog,
660661
[
661-
f"/work/mkosi.output/extra-tree/{cfg.arch}",
662-
f"/work/mkosi.output/vmlinuz/{cfg.kernel_version}/{cfg.arch}",
662+
f"/work/mkosi.output/kernel/{cfg.kernel_version}/{cfg.arch}",
663663
"/work/out",
664664
],
665665
)
@@ -713,13 +713,15 @@ def _build_mkosi_stage(cfg: Config, extra_args: list[str]) -> None:
713713
ilog.err("Install them or set --mkosi-mode=docker.")
714714
raise SystemExit(1)
715715
ilog.log("Building initrd with mkosi (native)...")
716-
extra_tree = str(cfg.extra_tree_output)
716+
tools_tree = str(cfg.tools_output)
717+
modules_tree = str(cfg.modules_output)
717718
output_dir = str(cfg.initramfs_output)
718719
run(
719720
[
720721
"mkosi",
721722
f"--architecture={cfg.arch_info.mkosi_arch}",
722-
f"--extra-tree={extra_tree}",
723+
f"--extra-tree={tools_tree}",
724+
f"--extra-tree={modules_tree}",
723725
f"--output-dir={output_dir}",
724726
"build",
725727
*mkosi_args,
@@ -731,11 +733,13 @@ def _build_mkosi_stage(cfg: Config, extra_args: list[str]) -> None:
731733
# --- docker -------------------------------------------------------
732734
docker.build_builder(cfg, logger=ilog)
733735
ilog.log("Building initrd with mkosi (docker)...")
734-
extra_tree = f"/work/mkosi.output/extra-tree/{cfg.arch}"
736+
tools_tree = f"/work/mkosi.output/tools/{cfg.arch}"
737+
modules_tree = f"/work/mkosi.output/kernel/{cfg.kernel_version}/{cfg.arch}/modules"
735738
output_dir = f"/work/mkosi.output/initramfs/{cfg.kernel_version}/{cfg.arch}"
736739
docker.run_mkosi(
737740
cfg,
738-
f"--extra-tree={extra_tree}",
741+
f"--extra-tree={tools_tree}",
742+
f"--extra-tree={modules_tree}",
739743
f"--output-dir={output_dir}",
740744
"build",
741745
*mkosi_args,
@@ -824,7 +828,7 @@ def _check_kernel_modules(cfg: Config) -> None:
824828
producing an initramfs without modules.
825829
"""
826830
ilog = for_stage("initramfs")
827-
modules_dir = cfg.extra_tree_output / "usr" / "lib" / "modules"
831+
modules_dir = cfg.modules_output / "usr" / "lib" / "modules"
828832
if not modules_dir.is_dir():
829833
ilog.err(f"Kernel modules directory not found: {modules_dir}")
830834
ilog.err("Ensure the kernel build artifacts are downloaded correctly.")
@@ -901,10 +905,10 @@ def _clean_version(cfg: Config, clog: StageLogger) -> None:
901905

902906
# Version-specific directories under mkosi.output/{stage}/{version}/{arch}
903907
version_dirs = [
904-
mkosi_output / "vmlinuz" / kver,
905-
mkosi_output / "initramfs" / kver,
906-
mkosi_output / "iso" / kver,
907-
mkosi_output / "iso-staging" / kver,
908+
mkosi_output / "kernel" / kver / cfg.arch,
909+
mkosi_output / "initramfs" / kver / cfg.arch,
910+
mkosi_output / "iso" / kver / cfg.arch,
911+
mkosi_output / "iso-staging" / kver / cfg.arch,
908912
]
909913

910914
has_docker = shutil.which("docker") is not None
@@ -971,16 +975,16 @@ def _clean_all(cfg: Config, clog: StageLogger) -> None:
971975
"-c",
972976
"rm -rf /work/mkosi.output/image*"
973977
" /work/mkosi.output/initramfs"
974-
" /work/mkosi.output/vmlinuz"
975-
" /work/mkosi.output/extra-tree"
978+
" /work/mkosi.output/kernel"
979+
" /work/mkosi.output/tools"
976980
" /work/mkosi.output/iso"
977981
" /work/mkosi.output/iso-staging"
978982
" /work/mkosi.cache",
979983
],
980984
)
981985
else:
982986
# No Docker available — remove directly (may need sudo for root-owned mkosi files)
983-
for pattern in ("image*", "initramfs", "vmlinuz", "extra-tree", "iso", "iso-staging"):
987+
for pattern in ("image*", "initramfs", "kernel", "tools", "iso", "iso-staging"):
984988
for p in mkosi_output.glob(pattern):
985989
if p.is_dir():
986990
shutil.rmtree(p, ignore_errors=True)
@@ -997,16 +1001,19 @@ def _clean_all(cfg: Config, clog: StageLogger) -> None:
9971001
def _cmd_summary(cfg: Config, _extra_args: list[str]) -> None:
9981002
"""Print mkosi configuration summary."""
9991003
slog = for_stage("summary")
1000-
extra_tree = str(cfg.extra_tree_output)
1004+
tools_tree = str(cfg.tools_output)
1005+
modules_tree = str(cfg.modules_output)
10011006
output_dir = str(cfg.initramfs_output)
10021007
match cfg.mkosi_mode:
10031008
case "docker":
10041009
docker.build_builder(cfg, logger=slog)
1005-
container_tree = f"/work/mkosi.output/extra-tree/{cfg.arch}"
1010+
container_tree = f"/work/mkosi.output/tools/{cfg.arch}"
1011+
container_modules = f"/work/mkosi.output/kernel/{cfg.kernel_version}/{cfg.arch}/modules"
10061012
container_outdir = f"/work/mkosi.output/initramfs/{cfg.kernel_version}/{cfg.arch}"
10071013
docker.run_mkosi(
10081014
cfg,
10091015
f"--extra-tree={container_tree}",
1016+
f"--extra-tree={container_modules}",
10101017
f"--output-dir={container_outdir}",
10111018
"summary",
10121019
logger=slog,
@@ -1016,7 +1023,8 @@ def _cmd_summary(cfg: Config, _extra_args: list[str]) -> None:
10161023
[
10171024
"mkosi",
10181025
f"--architecture={cfg.arch_info.mkosi_arch}",
1019-
f"--extra-tree={extra_tree}",
1026+
f"--extra-tree={tools_tree}",
1027+
f"--extra-tree={modules_tree}",
10201028
f"--output-dir={output_dir}",
10211029
"summary",
10221030
],
@@ -1345,12 +1353,12 @@ def main(project_dir: Path | None = None) -> None:
13451353
# Pass through to mkosi (shouldn't happen with _extract_command
13461354
# but kept as a safety net).
13471355
mlog = for_stage("mkosi")
1348-
extra_tree = str(cfg.extra_tree_output)
1356+
tools_tree = str(cfg.tools_output)
13491357
output_dir = str(cfg.initramfs_output)
13501358
match cfg.mkosi_mode:
13511359
case "docker":
13521360
docker.build_builder(cfg, logger=mlog)
1353-
container_tree = f"/work/mkosi.output/extra-tree/{cfg.arch}"
1361+
container_tree = f"/work/mkosi.output/tools/{cfg.arch}"
13541362
container_outdir = f"/work/mkosi.output/initramfs/{cfg.kernel_version}/{cfg.arch}"
13551363
docker.run_mkosi(
13561364
cfg,
@@ -1365,7 +1373,7 @@ def main(project_dir: Path | None = None) -> None:
13651373
[
13661374
"mkosi",
13671375
f"--architecture={cfg.arch_info.mkosi_arch}",
1368-
f"--extra-tree={extra_tree}",
1376+
f"--extra-tree={tools_tree}",
13691377
f"--output-dir={output_dir}",
13701378
command,
13711379
*extra,

0 commit comments

Comments
 (0)