From 170081926106398a55fe33b9f4f6c5ce164f745c Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 8 Aug 2025 15:40:06 -0400 Subject: [PATCH 1/9] cmd-build-with-buildah: simplify build existence check This drops a cmdlib.sh Python-baked function. --- src/cmd-build-with-buildah | 2 +- src/cmdlib.sh | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/cmd-build-with-buildah b/src/cmd-build-with-buildah index 2f6ed700a8..3e6cd00039 100755 --- a/src/cmd-build-with-buildah +++ b/src/cmd-build-with-buildah @@ -75,7 +75,7 @@ build_with_buildah() { argsfile=build-args-${variant}.conf fi - if [ "$(check_build_exists "${VERSION}")" == "True" ]; then + if [ -e "builds/$VERSION" ]; then echo "Build ${VERSION} already exists" exit 0 fi diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 8fd12018a9..34e331b685 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -1105,17 +1105,6 @@ cmdlib.import_ostree_commit(workdir, builddir, buildmeta, extract_json=('${extra ") } -check_build_exists() { - local buildid=$1; shift - (python3 -c " -import sys -sys.path.insert(0, '${DIR}') -from cosalib.builds import Builds -builds = Builds('${workdir:-$(pwd)}') -print(builds.has('${buildid}')) -") -} - # Extract the value of NAME from os-release extract_osrelease_name() { local buildid=$1; shift From 3654be13472445bb3382a3955989b32f4bc209a5 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 8 Aug 2025 15:42:30 -0400 Subject: [PATCH 2/9] cmd-build-with-buildah: fix variant handling The `variant` variable is set by `prepare_build`, which we're purposely not calling here. --- src/cmd-build-with-buildah | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cmd-build-with-buildah b/src/cmd-build-with-buildah index 3e6cd00039..5377be3398 100755 --- a/src/cmd-build-with-buildah +++ b/src/cmd-build-with-buildah @@ -70,9 +70,12 @@ build_with_buildah() { tmp_oci_archive_path=$(realpath "${tempdir}/out.ociarchive") - argsfile=build-args.conf - if [ -n "${variant:-}" ]; then + initconfig="src/config.json" + if [ -f "${initconfig}" ]; then + variant="$(jq --raw-output '."coreos-assembler.config-variant"' "${initconfig}")" argsfile=build-args-${variant}.conf + else + argsfile=build-args.conf fi if [ -e "builds/$VERSION" ]; then From 42349074f0b6a0239806c0bc9e9afdf648e2d0f3 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 8 Aug 2025 15:43:16 -0400 Subject: [PATCH 3/9] cmd-build-with-buildah: add FCOS stream label manually This is for https://github.com/coreos/fedora-coreos-tracker/issues/1996. I initially wanted to just add another `LABEL` to our Containerfile and hook that up to another build arg, but the problem is that our Containerfile is shared with RHCOS/SCOS and there's no way to make `LABEL` directives conditional. I opened https://github.com/coreos/rpm-ostree/pull/5454 which will fix this but for now to not block on this, let's just slap it on from the cosa side. Once it's folded back into the build process proper, then we can rever this. But the label not existing shouldn't really affect the majority of developers anyway. --- src/cmd-build-with-buildah | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/cmd-build-with-buildah b/src/cmd-build-with-buildah index 5377be3398..1abe1196bf 100755 --- a/src/cmd-build-with-buildah +++ b/src/cmd-build-with-buildah @@ -73,8 +73,10 @@ build_with_buildah() { initconfig="src/config.json" if [ -f "${initconfig}" ]; then variant="$(jq --raw-output '."coreos-assembler.config-variant"' "${initconfig}")" + manifest="src/config/manifest-${variant}.yaml" argsfile=build-args-${variant}.conf else + manifest="src/config/manifest.yaml" argsfile=build-args.conf fi @@ -88,6 +90,14 @@ build_with_buildah() { --build-arg VERSION="${VERSION}" \ -t oci-archive:"${tmp_oci_archive_path}" + # XXX: Temporary hack until we have https://github.com/coreos/rpm-ostree/pull/5454 + # which would allow us to fold this back into the build process. + # shellcheck source=/dev/null + stream=$(yaml2json "$manifest" /dev/stdout | jq -r '.variables.stream') + if [ "${stream}" != null ]; then + set -- "$@" --label fedora-coreos.stream="$stream" + fi + if [ -n "$DIRECT" ]; then # turn on layer caching in the direct case; it wouldn't hurt in the # supermin path, but it'd be a waste of space on the rootfs From f1c014e4f692d1198c8951655cfbb747a44cfde4 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 8 Aug 2025 15:48:39 -0400 Subject: [PATCH 4/9] cmd-build-with-buildah: fix `--version` help string --- src/cmd-build-with-buildah | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd-build-with-buildah b/src/cmd-build-with-buildah index 1abe1196bf..324a143e1a 100755 --- a/src/cmd-build-with-buildah +++ b/src/cmd-build-with-buildah @@ -14,7 +14,7 @@ Usage: coreos-assembler build-with-buildah `cosa build` will pivot to this script when the environment variable `COREOS_ASSEMBLER_BUILD_WITH_BUILDAH` is set. The following options are supported: - --version Use the versionary script from the source config to drive version. + --version=VERSION Use the given version instead of using versionary. --direct Run buildah directly rather than within supermin. EOF } From d208a64b8108767fd2677b93d1351777b6eebb9c Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 8 Aug 2025 15:49:47 -0400 Subject: [PATCH 5/9] cmd-build-with-buildah: rework version handling I think it's still useful to have a `dev` marker in the version string for development builds. The versionary script recently learned a new `--dev` switch which will spit out a development version string. Let's tweak the logic so that: - by default, if `--versionary` is not passed, we call `versionary --dev` to get a development version string - if `--versionary` is passed, we _don't_ pass `--dev` to get a production version string - if `--version` is passed, it always wins This is a bit confusing of course. E.g. ideally the `--versionary` switch would be called `--prod-versionary` or something. But the intent is to replicate the UX that exists today with `cosa build` so that this becomes a drop-in replacement for it in the pipeline. Once we cut over and don't need to support both, we could tweak things. --- src/cmd-build-with-buildah | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/cmd-build-with-buildah b/src/cmd-build-with-buildah index 324a143e1a..85791101c1 100755 --- a/src/cmd-build-with-buildah +++ b/src/cmd-build-with-buildah @@ -15,14 +15,16 @@ Usage: coreos-assembler build-with-buildah The following options are supported: --version=VERSION Use the given version instead of using versionary. + --versionary Generate non-development version using versionary. --direct Run buildah directly rather than within supermin. EOF } VERSION= +VERSIONARY= DIRECT= rc=0 -options=$(getopt --options h,v,d --longoptions help,version:,direct -- "$@") || rc=$? +options=$(getopt --options h,d --longoptions help,version:,versionary,direct -- "$@") || rc=$? [ $rc -eq 0 ] || { print_help exit 1 @@ -34,10 +36,13 @@ while true; do print_help exit 0 ;; - -v | --version) + --version) shift VERSION=$1 ;; + --versionary) + VERSIONARY=1 + ;; -d | --direct) DIRECT=1 ;; @@ -57,7 +62,11 @@ done if [ -z "${VERSION}" ]; then # let error out if file does not exist - VERSION=$(src/config/versionary) + if [ -z "${VERSIONARY}" ]; then + VERSION=$(src/config/versionary --dev) + else + VERSION=$(src/config/versionary) + fi fi build_with_buildah() { @@ -112,4 +121,4 @@ build_with_buildah() { rm -rf "${tempdir}" } -build_with_buildah \ No newline at end of file +build_with_buildah From 222434f36e4c6e44156887c69c5630c7aeddc325 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 8 Aug 2025 16:37:15 -0400 Subject: [PATCH 6/9] cmd-build-with-buildah: add yumrepos support If there's a yumrepos source, hook that up to the build so that it has access to the repo contents and the contentset. Also just mount in `/etc/pki/ca-trust` because the repos may require a cert. --- src/cmd-build-with-buildah | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cmd-build-with-buildah b/src/cmd-build-with-buildah index 85791101c1..62b1a6b052 100755 --- a/src/cmd-build-with-buildah +++ b/src/cmd-build-with-buildah @@ -107,6 +107,12 @@ build_with_buildah() { set -- "$@" --label fedora-coreos.stream="$stream" fi + if [ -d "src/yumrepos" ] && [ -e "src/yumrepos/${variant:-}.repo" ]; then + set -- "$@" --secret id=yumrepos,src="$(realpath "src/yumrepos/$variant.repo")" \ + --secret id=contentsets,src="$(realpath src/yumrepos/content_sets.yaml)" \ + -v /etc/pki/ca-trust:/etc/pki/ca-trust:ro + fi + if [ -n "$DIRECT" ]; then # turn on layer caching in the direct case; it wouldn't hurt in the # supermin path, but it'd be a waste of space on the rootfs From e427cf4d58f2dc140c4b53e108f4eee1a577aa45 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Sun, 10 Aug 2025 13:24:00 -0400 Subject: [PATCH 7/9] cmd-import: run `cosa prune` at the end This is equivalent to `cosa build` which also runs `cosa prune` at the end. --- src/cmd-import | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cmd-import b/src/cmd-import index 3726d092e1..aaca2604e2 100755 --- a/src/cmd-import +++ b/src/cmd-import @@ -55,6 +55,8 @@ def main(): # move into official location finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest) + subprocess.check_call(['/usr/lib/coreos-assembler/cmd-prune']) + def parse_args(): parser = argparse.ArgumentParser(prog='cosa import') From 195cc81a91debb0ce328c501b69ffef3d5f69c41 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Sun, 10 Aug 2025 13:26:30 -0400 Subject: [PATCH 8/9] cmd-import: generate `manifest-lock.generated.$arch.json` file This matches `cosa build`, but also this is required for autolocking, which makes use of this. --- src/cmd-import | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/cmd-import b/src/cmd-import index aaca2604e2..a1b355f557 100755 --- a/src/cmd-import +++ b/src/cmd-import @@ -49,11 +49,14 @@ def main(): # import into the tmp/repo to get the ostree-commit but also so it's cached ostree_commit = import_oci_archive(tmpd, tmp_oci_archive, buildid) + # artificially recreate generated lockfile + tmp_lockfile = generate_lockfile(tmpd, ostree_commit) + # create meta.json build_meta = generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_commit) # move into official location - finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest) + finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest, tmp_lockfile) subprocess.check_call(['/usr/lib/coreos-assembler/cmd-prune']) @@ -85,6 +88,25 @@ def generate_oci_manifest(args, tmpd): return tmpf +def generate_lockfile(tmpd, ostree_commit): + tmpf = os.path.join(tmpd, 'lockfile.json') + + out = subprocess.check_output(['rpm-ostree', 'db', 'list', '--repo', 'tmp/repo', ostree_commit], encoding='utf-8') + rpmdb = {} + for line in out.splitlines(): + if not line.startswith(' '): + continue + n, ev, ra = line.strip().rsplit('-', 2) + rpmdb[n] = {'evra': f'{ev}-{ra}'} + + with open(tmpf, 'w') as f: + json.dump(fp=f, obj={ + 'packages': rpmdb + }) + + return tmpf + + def generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_commit): name = metadata['Labels']['com.coreos.osname'] buildid = metadata['Labels']['org.opencontainers.image.version'] @@ -117,7 +139,7 @@ def generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_comm } -def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest): +def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest, tmp_lockfile): buildid = build_meta['buildid'] arch = build_meta['coreos-assembler.basearch'] @@ -126,6 +148,7 @@ def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest): shutil.move(tmp_oci_archive, f'{destdir}/{build_meta['images']['ostree']['path']}') shutil.move(tmp_oci_manifest, f'{destdir}/{build_meta['images']['oci-manifest']['path']}') + shutil.move(tmp_lockfile, f'{destdir}/manifest-lock.generated.{arch}.json') with open(f'{destdir}/meta.json', 'w') as f: json.dump(build_meta, f, indent=4) From d2b49da6250e66ef3a5028b255474bddf90c322a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Sun, 10 Aug 2025 13:27:45 -0400 Subject: [PATCH 9/9] cmd-build-with-buildah: add `--autolock` This matches `cosa build` support for it and is actively used by both RHCOS and FCOS (rawhide). Long-term I think this will become obsolete by hermetic builds in Konflux which will require us to always have in-tree lockfiles. --- src/cmd-build-with-buildah | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/cmd-build-with-buildah b/src/cmd-build-with-buildah index 62b1a6b052..3db99b7e5b 100755 --- a/src/cmd-build-with-buildah +++ b/src/cmd-build-with-buildah @@ -17,14 +17,18 @@ Usage: coreos-assembler build-with-buildah --version=VERSION Use the given version instead of using versionary. --versionary Generate non-development version using versionary. --direct Run buildah directly rather than within supermin. + --autolock=VERSION If no base lockfile used, create one from any arch build of `VERSION`. + Note this is automatically enabled when adding to an existing multi-arch + non-strict build. EOF } VERSION= VERSIONARY= DIRECT= +AUTOLOCK_VERSION= rc=0 -options=$(getopt --options h,d --longoptions help,version:,versionary,direct -- "$@") || rc=$? +options=$(getopt --options h,d --longoptions help,version:,versionary,direct,autolock: -- "$@") || rc=$? [ $rc -eq 0 ] || { print_help exit 1 @@ -46,6 +50,10 @@ while true; do -d | --direct) DIRECT=1 ;; + --autolock) + shift; + AUTOLOCK_VERSION=$1 + ;; --) shift break @@ -94,6 +102,18 @@ build_with_buildah() { exit 0 fi + # Apply autolock from another build for this version (or for another version if + # explicitly provided via --autolock) if no base lockfile exists. + lockfile="manifest-lock.${arch}.json" + if [ ! -f "src/config/${lockfile}" ] && { [ -n "${VERSION}" ] || [ -n "${AUTOLOCK_VERSION}" ]; }; then + autolockfile=$(tmprepo=tmp/repo; workdir=.; + generate_autolock "${AUTOLOCK_VERSION:-${VERSION}}") + if [ -n "${autolockfile}" ]; then + echo "Injecting autolock-generated ${lockfile}..." + cp "${autolockfile}" "${tempdir}/src/${lockfile}" + fi + fi + set -- build --security-opt=label=disable --cap-add=all --device /dev/fuse \ --build-arg-file "$argsfile" -v "$(realpath "${tempdir}/src")":/run/src \ --build-arg VERSION="${VERSION}" \