diff --git a/src/cmd-build-with-buildah b/src/cmd-build-with-buildah index 2f6ed700a8..3db99b7e5b 100755 --- a/src/cmd-build-with-buildah +++ b/src/cmd-build-with-buildah @@ -14,15 +14,21 @@ Usage: coreos-assembler build-with-buildah `cosa build` will pivot to this script when the environment variable `COREOS_ASSEMBLER_BUILD_WITH_BUILDAH` is set. The following options are supported: - --version Use the versionary script from the source config to drive version. + --version=VERSION Use the given version instead of using versionary. + --versionary Generate non-development version using versionary. --direct Run buildah directly rather than within supermin. + --autolock=VERSION If no base lockfile used, create one from any arch build of `VERSION`. + Note this is automatically enabled when adding to an existing multi-arch + non-strict build. EOF } VERSION= +VERSIONARY= DIRECT= +AUTOLOCK_VERSION= rc=0 -options=$(getopt --options h,v,d --longoptions help,version:,direct -- "$@") || rc=$? +options=$(getopt --options h,d --longoptions help,version:,versionary,direct,autolock: -- "$@") || rc=$? [ $rc -eq 0 ] || { print_help exit 1 @@ -34,13 +40,20 @@ while true; do print_help exit 0 ;; - -v | --version) + --version) shift VERSION=$1 ;; + --versionary) + VERSIONARY=1 + ;; -d | --direct) DIRECT=1 ;; + --autolock) + shift; + AUTOLOCK_VERSION=$1 + ;; --) shift break @@ -57,7 +70,11 @@ done if [ -z "${VERSION}" ]; then # let error out if file does not exist - VERSION=$(src/config/versionary) + if [ -z "${VERSIONARY}" ]; then + VERSION=$(src/config/versionary --dev) + else + VERSION=$(src/config/versionary) + fi fi build_with_buildah() { @@ -70,21 +87,52 @@ build_with_buildah() { tmp_oci_archive_path=$(realpath "${tempdir}/out.ociarchive") - argsfile=build-args.conf - if [ -n "${variant:-}" ]; then + initconfig="src/config.json" + if [ -f "${initconfig}" ]; then + variant="$(jq --raw-output '."coreos-assembler.config-variant"' "${initconfig}")" + manifest="src/config/manifest-${variant}.yaml" argsfile=build-args-${variant}.conf + else + manifest="src/config/manifest.yaml" + argsfile=build-args.conf fi - if [ "$(check_build_exists "${VERSION}")" == "True" ]; then + if [ -e "builds/$VERSION" ]; then echo "Build ${VERSION} already exists" exit 0 fi + # Apply autolock from another build for this version (or for another version if + # explicitly provided via --autolock) if no base lockfile exists. + lockfile="manifest-lock.${arch}.json" + if [ ! -f "src/config/${lockfile}" ] && { [ -n "${VERSION}" ] || [ -n "${AUTOLOCK_VERSION}" ]; }; then + autolockfile=$(tmprepo=tmp/repo; workdir=.; + generate_autolock "${AUTOLOCK_VERSION:-${VERSION}}") + if [ -n "${autolockfile}" ]; then + echo "Injecting autolock-generated ${lockfile}..." + cp "${autolockfile}" "${tempdir}/src/${lockfile}" + fi + fi + set -- build --security-opt=label=disable --cap-add=all --device /dev/fuse \ --build-arg-file "$argsfile" -v "$(realpath "${tempdir}/src")":/run/src \ --build-arg VERSION="${VERSION}" \ -t oci-archive:"${tmp_oci_archive_path}" + # XXX: Temporary hack until we have https://github.com/coreos/rpm-ostree/pull/5454 + # which would allow us to fold this back into the build process. + # shellcheck source=/dev/null + stream=$(yaml2json "$manifest" /dev/stdout | jq -r '.variables.stream') + if [ "${stream}" != null ]; then + set -- "$@" --label fedora-coreos.stream="$stream" + fi + + if [ -d "src/yumrepos" ] && [ -e "src/yumrepos/${variant:-}.repo" ]; then + set -- "$@" --secret id=yumrepos,src="$(realpath "src/yumrepos/$variant.repo")" \ + --secret id=contentsets,src="$(realpath src/yumrepos/content_sets.yaml)" \ + -v /etc/pki/ca-trust:/etc/pki/ca-trust:ro + fi + if [ -n "$DIRECT" ]; then # turn on layer caching in the direct case; it wouldn't hurt in the # supermin path, but it'd be a waste of space on the rootfs @@ -99,4 +147,4 @@ build_with_buildah() { rm -rf "${tempdir}" } -build_with_buildah \ No newline at end of file +build_with_buildah diff --git a/src/cmd-import b/src/cmd-import index 3726d092e1..a1b355f557 100755 --- a/src/cmd-import +++ b/src/cmd-import @@ -49,11 +49,16 @@ def main(): # import into the tmp/repo to get the ostree-commit but also so it's cached ostree_commit = import_oci_archive(tmpd, tmp_oci_archive, buildid) + # artificially recreate generated lockfile + tmp_lockfile = generate_lockfile(tmpd, ostree_commit) + # create meta.json build_meta = generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_commit) # move into official location - finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest) + finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest, tmp_lockfile) + + subprocess.check_call(['/usr/lib/coreos-assembler/cmd-prune']) def parse_args(): @@ -83,6 +88,25 @@ def generate_oci_manifest(args, tmpd): return tmpf +def generate_lockfile(tmpd, ostree_commit): + tmpf = os.path.join(tmpd, 'lockfile.json') + + out = subprocess.check_output(['rpm-ostree', 'db', 'list', '--repo', 'tmp/repo', ostree_commit], encoding='utf-8') + rpmdb = {} + for line in out.splitlines(): + if not line.startswith(' '): + continue + n, ev, ra = line.strip().rsplit('-', 2) + rpmdb[n] = {'evra': f'{ev}-{ra}'} + + with open(tmpf, 'w') as f: + json.dump(fp=f, obj={ + 'packages': rpmdb + }) + + return tmpf + + def generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_commit): name = metadata['Labels']['com.coreos.osname'] buildid = metadata['Labels']['org.opencontainers.image.version'] @@ -115,7 +139,7 @@ def generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_comm } -def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest): +def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest, tmp_lockfile): buildid = build_meta['buildid'] arch = build_meta['coreos-assembler.basearch'] @@ -124,6 +148,7 @@ def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest): shutil.move(tmp_oci_archive, f'{destdir}/{build_meta['images']['ostree']['path']}') shutil.move(tmp_oci_manifest, f'{destdir}/{build_meta['images']['oci-manifest']['path']}') + shutil.move(tmp_lockfile, f'{destdir}/manifest-lock.generated.{arch}.json') with open(f'{destdir}/meta.json', 'w') as f: json.dump(build_meta, f, indent=4) diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 8fd12018a9..34e331b685 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -1105,17 +1105,6 @@ cmdlib.import_ostree_commit(workdir, builddir, buildmeta, extract_json=('${extra ") } -check_build_exists() { - local buildid=$1; shift - (python3 -c " -import sys -sys.path.insert(0, '${DIR}') -from cosalib.builds import Builds -builds = Builds('${workdir:-$(pwd)}') -print(builds.has('${buildid}')) -") -} - # Extract the value of NAME from os-release extract_osrelease_name() { local buildid=$1; shift