diff --git a/cmd/coreos-assembler.go b/cmd/coreos-assembler.go index 29f7b99ea2..4cf4ba8fc9 100644 --- a/cmd/coreos-assembler.go +++ b/cmd/coreos-assembler.go @@ -12,9 +12,9 @@ import ( ) // commands we'd expect to use in the local dev path -var buildCommands = []string{"init", "fetch", "build", "run", "prune", "clean", "list"} +var buildCommands = []string{"init", "fetch", "build", "osbuild", "run", "prune", "clean", "list"} var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container"} -var buildextendCommands = []string{"aliyun", "applehv", "aws", "azure", "digitalocean", "exoscale", "extensions-container", "gcp", "hashlist-experimental", "hyperv", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "qemu", "secex", "virtualbox", "vmware", "vultr"} +var buildextendCommands = []string{"aliyun", "applehv", "aws", "azure", "digitalocean", "exoscale", "extensions-container", "gcp", "hyperv", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "qemu", "secex", "virtualbox", "vmware", "vultr"} var utilityCommands = []string{"aws-replicate", "cloud-prune", "compress", "container-prune", "copy-container", "koji-upload", "kola", "push-container-manifest", "remote-build-container", "remote-session", "sign", "tag", "update-variant"} var otherCommands = []string{"shell", "meta"} diff --git a/src/cmd-buildextend-hashlist-experimental b/src/cmd-buildextend-hashlist-experimental deleted file mode 100755 index 5c3fb43ae9..0000000000 --- a/src/cmd-buildextend-hashlist-experimental +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python3 -""" -Creates a KeyLime Hashlist for an image. - -See: https://keylime.dev/ -""" - -import argparse -import datetime -import json -import os -import shutil -import subprocess -import sys -import tempfile - -cosa_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, cosa_dir) - -from cosalib import meta -from cosalib.builds import Builds -from cosalib.cmdlib import ( - ensure_glob, - get_basearch, - sha256sum_file, - import_ostree_commit) - - -class HashListV1(dict): - """ - Abstraction of a HashList in the version 1 series - - See: https://github.com/keylime/enhancements/blob/master/16_remote_allowlist_retrieval.md - """ - - def __init__(self, metadata, release=None, generator=None, timestamp=None): - """ - Initialize HashListV1 instance. - - :param metadata: Loaded metadata instance - :type metadata: GenericBuildMeta - :param release: Optional release override - :type release: str - :param generator: Optional generator override - :type generator: str - :param timestamp: Optional timestamp override - :type timestamp: str - :raises: KeyError - """ - super() - self._metadata = metadata - if release is None: - release = self._metadata['buildid'] - self['release'] = release - - self['meta'] = {} - if generator is None: - generator = 'coreos-assembler-' + self._metadata['coreos-assembler.container-image-git']['commit'] # noqa - self['meta']['generator'] = generator - if timestamp is None: - timestamp = datetime.datetime.utcnow().isoformat() - self['meta']['timestamp'] = timestamp - self['meta']['ostree'] = self._metadata['images']['ostree']['path'] - self['meta']['arch'] = self._metadata['coreos-assembler.basearch'] - - self['hashes'] = {} - - def populate_hash_list(self): - """ - Populates the hashlist instance with ostree and initramfs hashes. - - :param checksum: The ostree checksum from the metadata - :type checksum: str - :param commit: The ostree commit - :type commit: str - :param hashlist: Initialized hash list instance - :type hashlist: HashListV1 - :returns: Nothing - :rtype: None - :raises: IndexError - """ - checkout = 'tmp/repo/tmp/keylime-checkout' - if os.path.isdir(checkout): - shutil.rmtree(checkout) - - import_ostree_commit( - os.getcwd(), - self._metadata.build_dir, - self._metadata) - subprocess.check_call([ - 'ostree', 'checkout', - '--repo=tmp/repo', '-U', - self._metadata['ostree-commit'], checkout]) - # Make all dirs in 'tmp' checkout readable. - # 'find' can't recurse into tmp dir files, we add 'x' so that it can recurse into it. - # With '+', 'find' accumulates paths and runs 'chmod' once (or batched by the arg limit). - # To 'chmod' the dir before it tries to recurse into it, we use ';'. - subprocess.check_call(['find', checkout, '-type', 'd', '-exec', - 'chmod', 'u+rwx', '{}', ';']) - self.hash_from_path(checkout) - - # Extract initramfs contents - initramfs_path = ensure_glob( - os.path.join( - checkout, 'usr/lib/modules/*/initramfs.img'))[0] - initramfs_path = os.path.realpath(initramfs_path) - - with tempfile.TemporaryDirectory() as tmpdir: - subprocess.check_call(['lsinitrd', '--unpack', initramfs_path], - cwd=tmpdir) - self.hash_from_path(tmpdir) - - shutil.rmtree(checkout) - - def hash_from_path(self, toppath): - """ - Create hashes from files starting at a specific path. - - .. note:: If a file can not be read it is skipped - - :param toppath: The starting path to go through - :type toppath: str - :returns: Nothing - :rtype: None - """ - for dirpath, _, filenames in os.walk(toppath): - for fname in filenames: - filepath = os.path.join(dirpath, fname) - # Skip symlinks - if os.path.islink(filepath): - continue - try: - filehash = sha256sum_file(filepath) - relpath = filepath.replace(toppath, '') - self['hashes'][relpath] = [filehash] - except (PermissionError, FileNotFoundError) as err: - print(f'Unable to hash {filepath}: {err}') - - def write(self, output): - """ - Renders the HashList structure to disk. - - :param output: Where to write the file - :type output: str - :raises: FileNotFoundError, IsADirectoryError, PermissionError - """ - with open(output, 'w') as out: - out.write(json.dumps(self, indent=4)) - with open(f'{output}-CHECKSUM', 'w') as out: - subprocess.check_call(['sha256sum', output], stdout=out) - - -def main(): - """ - Main entry point. - """ - parser = argparse.ArgumentParser(description=__doc__) - # parser.add_argument( - # '-s', '--sign', - # help='If set to a key a signature will be created as well') - parser.add_argument( - '-g', '--generator', help='Override the generator name') - parser.add_argument( - '-t', '--timestamp', help='Override the timestamp') - parser.add_argument( - '-r', '--release', help='Override the release') - parser.add_argument( - '-a', '--arch', default=get_basearch(), - help='Target Architecture') - parser.add_argument('-b', '--build', default='latest', help='Target build') - - # Parse arguments and ignore anything we didn't explicitly request - args = parser.parse_args() - - builds = Builds() - builddir = builds.get_build_dir(build_id=args.build, basearch=args.arch) - - # it's not in the schema nor in meta.json yet - output = os.path.join(builddir, "exp-hash.json") - - # Load metadata - metadata = meta.GenericBuildMeta(schema=None, basearch=args.arch) - - # Step 0: Initialize the HashList instance - hashlist = HashListV1( - metadata, args.release, args.generator, args.timestamp) - - # Step 1: Populate the hashlist - hashlist.populate_hash_list() - - # Step 2: Write the hashlist to disk - hashlist.write(output) - print(f'Hash list created at {output}') - - -if __name__ == '__main__': - main() diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal deleted file mode 100755 index d7c373c393..0000000000 --- a/src/cmd-buildextend-metal +++ /dev/null @@ -1,308 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -dn=$(dirname "$0") -# shellcheck source=src/cmdlib.sh -. "${dn}"/cmdlib.sh - -# This script is used for creating both the bare metal and the canonical VM -# image (qemu). `buildextend-qemu` is a symlink to `buildextend-metal`. -case "$(basename "$0")" in - "cmd-buildextend-metal") image_type=metal;; - "cmd-buildextend-metal4k") image_type=metal4k;; - "cmd-buildextend-qemu") image_type=qemu;; - "cmd-buildextend-qemu-secex") image_type=qemu-secex;; - "cmd-buildextend-secex") image_type=qemu-secex;; - *) fatal "called as unexpected name $0";; -esac - -print_help() { - cat 1>&2 < "$PWD/tmp/ostree-size.json" -rootfs_size_mb="$(jq '."estimate-mb".final' "$PWD/tmp/ostree-size.json")" -# The minimum size of a disk image we'll need will be the rootfs_size -# estimate plus the size of the non-root partitions. We'll use this -# size for the metal images, but for the IaaS/virt image we'll use -# the size set in the configs since some of them have minimum sizes that -# the platforms require and we want a "default" disk size that has some -# free space. -nonroot_partition_sizes=513 -# On s390x there is one more build - Secure Execution case, which has -# different image layout. We add the sizes of the se and verity -# partitions so that they don't "eat into" the 35% buffer (though note -# this is all blown away on first boot anyway). For 's390x.mpp.yaml' -# simplicity all s390x images have same size (of secex image). -if [[ $basearch == "s390x" ]]; then - nonroot_partition_sizes=$((nonroot_partition_sizes + 200 + 128 + 256 + 1)) -fi -metal_image_size_mb="$(( rootfs_size_mb + nonroot_partition_sizes ))" -cloud_image_size_mb="$(jq -r ".size*1024" < "${image_json}")" -echo "Disk sizes: metal: ${metal_image_size_mb}M (estimated), cloud: ${cloud_image_size_mb}M" - -set -x -extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")" - -# Generate the JSON describing the disk we want to build -image_dynamic_yaml="${tmp_builddir}/image-dynamic.yaml" -image_dynamic_json="${tmp_builddir}/image-dynamic.json" -image_for_disk_json="${tmp_builddir}/image-for-disk.json" -cat >"${image_dynamic_yaml}" << EOF -container-imgref: "${container_imgref}" -deploy-via-container: "${deploy_via_container}" -osname: "${name}" -ostree-container: "${ostree_container}" -ostree-ref: "${ref}" -extra-kargs-string: "${extra_kargs}" -image-type: "${image_type}" -ostree-repo: "${ostree_repo}" -metal-image-size: "${metal_image_size_mb}" -cloud-image-size: "${cloud_image_size_mb}" -# Note: this is only used in the secex case; there, the rootfs is -# not the last partition on the disk so we need to explicitly size it -rootfs-size: "${rootfs_size_mb}" -EOF -yaml2json "${image_dynamic_yaml}" "${image_dynamic_json}" -cat "${image_json}" "${image_dynamic_json}" | jq -s add > "${image_for_disk_json}" -platforms_json="${tmp_builddir}/platforms.json" -yaml2json "${configdir}/platforms.yaml" "${platforms_json}" - -# In the jenkins pipelines we build the qemu image first and that operation -# will do a lot of the same work required for later artifacts (metal, metal4k, etc) -# so we want the cached output from that run to persist. The later artifacts get -# built in parallel, so we need to be able to access the cache by multiple processes, -# so for those we'll set `snapshot=on` so that each will get their own disk image. -# This is OK because we don't checkpoint (cache) any of those stages. -[ "${image_type}" == "qemu" ] && snapshot="off" || snapshot="on" -runvm_with_cache_snapshot "$snapshot" -- /usr/lib/coreos-assembler/runvm-osbuild \ - --config "${image_for_disk_json}" \ - --mpp "/usr/lib/coreos-assembler/osbuild-manifests/coreos.osbuild.${basearch}.mpp.yaml" \ - --filepath "${imgpath}" - -if [[ "${image_type}" == "qemu-secex" ]]; then - if [ ! -f "${genprotimgvm}" ]; then - fatal "No genprotimgvm provided at ${genprotimgvm}" - fi - - # Basic qemu args: - qemu_args=(); blk_size="512" - [[ $image_type == metal4k ]] && blk_size="4096" - qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${imgpath},cache=unsafe" \ - "-device" "virtio-blk,serial=target,drive=target,physical_block_size=${blk_size},logical_block_size=${blk_size}") - - # SecureVM (holding Universal Key for all IBM Z Mainframes) requires scripts to execute genprotimg - se_script_dir="/usr/lib/coreos-assembler/secex-genprotimgvm-scripts" - genprotimg_img="${PWD}/secex-genprotimg.img" - genprotimg_dir=$(mktemp -p "${tmp_builddir}" -d) - cp "${se_script_dir}/genprotimg-script.sh" "${se_script_dir}/post-script.sh" "${genprotimg_dir}" - # Extra kargs with dm-verity hashes - secex_kargs="ignition.firstboot rootfs.roothash=$(<"${PWD}"/rootfs_hash) bootfs.roothash=$(<"${PWD}"/bootfs_hash)" - echo "${secex_kargs}" > "${genprotimg_dir}/parmfile" - virt-make-fs --format=raw --type=ext4 "${genprotimg_dir}" "${genprotimg_img}" - rm -rf "${genprotimg_dir}" - qemu_args+=("-drive" "if=none,id=genprotimg,format=raw,file=${genprotimg_img}" \ - "-device" "virtio-blk,serial=genprotimg,drive=genprotimg") - - # GPG keys used for protecting Ignition config - tmp_gpg_home=$(mktemp -p "${tmp_builddir}" -d) - ignition_pubkey=$(mktemp -p "${tmp_builddir}") - ignition_prikey=$(mktemp -p "${tmp_builddir}") - gpg --homedir "${tmp_gpg_home}" --batch --passphrase '' --yes --quick-gen-key "Secure Execution (secex) ${build}" rsa4096 encr none - gpg --homedir "${tmp_gpg_home}" --armor --export secex > "${ignition_pubkey}" - gpg --homedir "${tmp_gpg_home}" --armor --export-secret-key secex > "${ignition_prikey}" - exec 9<"${ignition_prikey}" - rm -rf "${tmp_gpg_home}" "${ignition_prikey}" - qemu_args+=("-add-fd" "fd=9,set=3" "-drive" "if=none,id=gpgkey,format=raw,file=/dev/fdset/3,readonly=on" \ - "-device" "virtio-blk,serial=gpgkey,drive=gpgkey") - - /usr/lib/coreos-assembler/secex-genprotimgvm-scripts/runvm.sh \ - --genprotimgvm "${genprotimgvm}" -- "${qemu_args[@]}" - rm -f "${genprotimg_img}" - exec 9>&- -fi - -sha256=$(sha256sum_str < "${imgpath}") -cosa meta --workdir "${workdir}" --build "${build}" --dump | python3 -c " -import sys, json -j = json.load(sys.stdin) -j['images']['${image_type}'] = { - 'path': '${imgname}', - 'sha256': '${sha256}', - 'size': $(stat -c '%s' "${imgpath}") -} -json.dump(j, sys.stdout, indent=4) -" | jq -s add > "meta.json.new" - -# one more artifact for Secure Execution -if [[ -n "${ignition_pubkey}" ]]; then - gpg_key=${name}-${build}-ignition-secex-key.gpg.pub - python3 -c " -import sys, json -j = json.load(sys.stdin) -j['images']['ignition-gpg-key'] = { - 'path': '${gpg_key}', - 'sha256': '$(sha256sum_str < "${ignition_pubkey}")', - 'size': $(stat -c '%s' "${ignition_pubkey}"), - 'skip-compression': True -} -json.dump(j, sys.stdout, indent=4) -" < "meta.json.new" | jq -s add > "key.json" - mv key.json meta.json.new - /usr/lib/coreos-assembler/finalize-artifact "${ignition_pubkey}" "${builddir}/${gpg_key}" -fi - -# and now the crucial bits -cosa meta --workdir "${workdir}" --build "${build}" --artifact "${image_type}" --artifact-json "$(readlink -f meta.json.new)" -/usr/lib/coreos-assembler/finalize-artifact "${imgpath}" "${builddir}/${imgname}" - -# Quiet for the rest of this so the last thing we see is a success message -set +x -# clean up the tmpbuild -rm -rf "${tmp_builddir}" -echo "Successfully generated: ${imgname}" diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal new file mode 120000 index 0000000000..dcbb012994 --- /dev/null +++ b/src/cmd-buildextend-metal @@ -0,0 +1 @@ +cmd-osbuild \ No newline at end of file diff --git a/src/cmd-buildextend-metal4k b/src/cmd-buildextend-metal4k index ad07b13c46..dcbb012994 120000 --- a/src/cmd-buildextend-metal4k +++ b/src/cmd-buildextend-metal4k @@ -1 +1 @@ -cmd-buildextend-metal \ No newline at end of file +cmd-osbuild \ No newline at end of file diff --git a/src/cmd-buildextend-qemu b/src/cmd-buildextend-qemu index ad07b13c46..dcbb012994 120000 --- a/src/cmd-buildextend-qemu +++ b/src/cmd-buildextend-qemu @@ -1 +1 @@ -cmd-buildextend-metal \ No newline at end of file +cmd-osbuild \ No newline at end of file diff --git a/src/cmd-buildextend-qemu-secex b/src/cmd-buildextend-qemu-secex index c2fa2326d8..dcbb012994 120000 --- a/src/cmd-buildextend-qemu-secex +++ b/src/cmd-buildextend-qemu-secex @@ -1 +1 @@ -./cmd-buildextend-metal \ No newline at end of file +cmd-osbuild \ No newline at end of file diff --git a/src/cmd-buildextend-secex b/src/cmd-buildextend-secex index ad07b13c46..dcbb012994 120000 --- a/src/cmd-buildextend-secex +++ b/src/cmd-buildextend-secex @@ -1 +1 @@ -cmd-buildextend-metal \ No newline at end of file +cmd-osbuild \ No newline at end of file diff --git a/src/cmd-osbuild b/src/cmd-osbuild new file mode 100755 index 0000000000..0e2377522f --- /dev/null +++ b/src/cmd-osbuild @@ -0,0 +1,391 @@ +#!/usr/bin/env bash +set -euo pipefail + +dn=$(dirname "$0") +# shellcheck source=src/cmdlib.sh +. "${dn}"/cmdlib.sh + +# A list of supported platforms and the filename suffix of the main +# artifact that platform produces. +declare -A SUPPORTED_PLATFORMS=( + ['applehv']='raw.gz' + ['gcp']='tar.gz' + ['hyperv']='vhdx.zip' + ['metal4k']='raw' + ['metal']='raw' + ['qemu']='qcow2' + ['qemu-secex']='qcow2' +) + +print_help() { + cat 1>&2 < meta.json.new + cosa meta --workdir "${workdir}" --build "${build}" --artifact-json meta.json.new + /usr/lib/coreos-assembler/finalize-artifact "${local_filepath}" "${builddir}/${target_filename}" + echo "Successfully generated: ${target_filename}" +} + +# For qemu-secex we need to do a few extra things like spin up a +# VM to run genprotimg and save off the pubkey for Ignition. +postprocess_qemu_secex() { + if [ ! -f "${genprotimgvm}" ]; then + fatal "No genprotimgvm provided at ${genprotimgvm}" + fi + + # Basic qemu args: + qemu_args=(); blk_size="512" + [[ $platform == metal4k ]] && blk_size="4096" + qemu_args+=("-drive" "if=none,id=target,format=qcow,file=${imgpath},cache=unsafe" \ + "-device" "virtio-blk,serial=target,drive=target,physical_block_size=${blk_size},logical_block_size=${blk_size}") + + # SecureVM (holding Universal Key for all IBM Z Mainframes) requires scripts to execute genprotimg + se_script_dir="/usr/lib/coreos-assembler/secex-genprotimgvm-scripts" + genprotimg_img="${PWD}/secex-genprotimg.img" + genprotimg_dir=$(mktemp -p "${tmp_builddir}" -d) + cp "${se_script_dir}/genprotimg-script.sh" "${se_script_dir}/post-script.sh" "${genprotimg_dir}" + # Extra kargs with dm-verity hashes + secex_kargs="ignition.firstboot" + secex_kargs+=" rootfs.roothash=$(<"${outdir}/${platform}/rootfs_hash")" + secex_kargs+=" bootfs.roothash=$(<"${outdir}/${platform}/bootfs_hash")" + echo "${secex_kargs}" > "${genprotimg_dir}/parmfile" + virt-make-fs --format=raw --type=ext4 "${genprotimg_dir}" "${genprotimg_img}" + rm -rf "${genprotimg_dir}" + qemu_args+=("-drive" "if=none,id=genprotimg,format=raw,file=${genprotimg_img}" \ + "-device" "virtio-blk,serial=genprotimg,drive=genprotimg") + + # GPG keys used for protecting Ignition config + tmp_gpg_home=$(mktemp -p "${tmp_builddir}" -d) + ignition_pubkey=$(mktemp -p "${tmp_builddir}") + ignition_prikey=$(mktemp -p "${tmp_builddir}") + gpg --homedir "${tmp_gpg_home}" --batch --passphrase '' --yes --quick-gen-key "Secure Execution (secex) ${build}" rsa4096 encr none + gpg --homedir "${tmp_gpg_home}" --armor --export secex > "${ignition_pubkey}" + gpg --homedir "${tmp_gpg_home}" --armor --export-secret-key secex > "${ignition_prikey}" + exec 9<"${ignition_prikey}" + rm -rf "${tmp_gpg_home}" "${ignition_prikey}" + qemu_args+=("-add-fd" "fd=9,set=3" "-drive" "if=none,id=gpgkey,format=raw,file=/dev/fdset/3,readonly=on" \ + "-device" "virtio-blk,serial=gpgkey,drive=gpgkey") + + /usr/lib/coreos-assembler/secex-genprotimgvm-scripts/runvm.sh \ + --genprotimgvm "${genprotimgvm}" -- "${qemu_args[@]}" + rm -f "${genprotimg_img}" + exec 9>&- + + # Now store the generated ${ignition_pubkey} in the builddir and meta.json + gpg_key_filename="${name}-${build}-ignition-secex-key.gpg.pub" + postprocess_artifact "ignition-gpg-key" "${ignition_pubkey}" "${gpg_key_filename}" 'True' +} + +# Here we generate the input JSON we pass to runvm_osbuild for all of our image builds +generate_runvm_osbuild_config() { + runvm_osbuild_config_json="${workdir}/tmp/runvm-osbuild-config-${build}.json" + echo "${runvm_osbuild_config_json}" # Let the caller know where the config will be + if [ -f "${runvm_osbuild_config_json}" ]; then + return # only need to generate this once per build + fi + rm -f "${workdir}"/tmp/runvm-osbuild-config-*.json # clean up any previous configs + + # reread these values from the build itself rather than rely on the ones loaded + # by prepare_build since the config might've changed since then + ostree_commit=$(meta_key ostree-commit) + ostree_ref=$(meta_key ref) + if [ "${ostree_ref}" = "None" ]; then + ostree_ref="" + fi + + ostree_repo=${tmprepo} + # Ensure that we have the cached unpacked commit + import_ostree_commit_for_build "${build}" + # Note this overwrote the bits generated in prepare_build + # for image_json. In the future we expect to split prepare_build + # into prepare_ostree_build and prepare_diskimage_build; the + # latter path would only run this. + image_json=${workdir}/tmp/image.json + + # Grab a few values from $image_json + deploy_via_container=$(getconfig_def "deploy-via-container" "" "${image_json}") + extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")" + + # OStree container ociarchive file path + ostree_container="${builddir}/$(meta_key images.ostree.path)" + # If no container_imgref was set let's just set it to some professional + # looking default. The name of the ociarchive file should suffice. + container_imgref_default="ostree-image-signed:oci-archive:/$(basename "${ostree_container}")" + container_imgref=$(getconfig_def "container_imgref" "${container_imgref_default}" "${image_json}") + + echo "Estimating disk size..." >&2 + # The additional 35% here is obviously a hack, but we can't easily completely fill the filesystem, + # and doing so has apparently negative performance implications. + ostree_size_json="$(/usr/lib/coreos-assembler/estimate-commit-disk-size --repo "$ostree_repo" "$ostree_commit" --add-percent 35)" + rootfs_size_mb="$(jq '."estimate-mb".final' <<< "${ostree_size_json}")" + # The minimum size of a disk image we'll need will be the rootfs_size + # estimate plus the size of the non-root partitions. We'll use this + # size for the metal images, but for the IaaS/virt image we'll use + # the size set in the configs since some of them have minimum sizes that + # the platforms require and we want a "default" disk size that has some + # free space. + nonroot_partition_sizes=513 + # On s390x there is one more build - Secure Execution case, which has + # different image layout. We add the sizes of the se and verity + # partitions so that they don't "eat into" the 35% buffer (though note + # this is all blown away on first boot anyway). For 's390x.mpp.yaml' + # simplicity all s390x images have same size (of secex image). + if [[ $basearch == "s390x" ]]; then + nonroot_partition_sizes=$((nonroot_partition_sizes + 200 + 128 + 256 + 1)) + fi + metal_image_size_mb="$(( rootfs_size_mb + nonroot_partition_sizes ))" + cloud_image_size_mb="$(jq -r ".size*1024" < "${image_json}")" + echo "Disk sizes: metal: ${metal_image_size_mb}M (estimated), cloud: ${cloud_image_size_mb}M" >&2 + + # Generate the JSON describing the disk we want to build + yaml2json /dev/stdin "${runvm_osbuild_config_json}" < /dev/null # Run through the preprocessor # Note: don't quote the size arguements since they are numbers, not strings -osbuild-mpp \ - -D arch=\""$(arch)"\" \ - -D ostree_ref=\""${ostree_ref}"\" \ - -D ostree_repo=\""${ostree_repo}"\" \ - -D filename=\""${filename}"\" \ - -D ociarchive=\""${ostree_container}"\" \ - -D osname=\""${osname}"\" \ - -D container_imgref=\""${container_imgref}"\" \ - -D container_repo=\""${container_repo}"\" \ - -D container_tag=\""${container_tag}"\" \ - -D extra_kargs=\""${extra_kargs}"\" \ - -D metal_image_size_mb="${metal_image_size_mb}" \ - -D cloud_image_size_mb="${cloud_image_size_mb}" \ - -D rootfs_size_mb="${rootfs_size_mb}" \ +osbuild-mpp \ + -D arch=\""$(arch)"\" \ + -D artifact_name_prefix=\""${artifact_name_prefix}"\" \ + -D ostree_ref=\""${ostree_ref}"\" \ + -D ostree_repo=\""${ostree_repo}"\" \ + -D ociarchive=\""${ostree_container}"\" \ + -D osname=\""${osname}"\" \ + -D container_imgref=\""${container_imgref}"\" \ + -D container_repo=\""${container_repo}"\" \ + -D container_tag=\""${container_tag}"\" \ + -D extra_kargs=\""${extra_kargs}"\" \ + -D metal_image_size_mb="${metal_image_size_mb}" \ + -D cloud_image_size_mb="${cloud_image_size_mb}" \ + -D rootfs_size_mb="${rootfs_size_mb}" \ "${mppyaml}" "${processed_json}" # Build the image +# shellcheck disable=SC2068 osbuild \ --out "$outdir" \ --store "$storedir" \ @@ -109,18 +120,8 @@ osbuild \ --checkpoint build \ --checkpoint tree \ --checkpoint raw-image \ - --export "$platform" "${processed_json}" - - -# Copy it out to the specified location. Use mv here so we remove it -# from the cache qcow2 so we don't cache it. -mv "${outdir}/${platform}/${filename}" "${filepath}" - -# In case of IBM Secure Execution there are more artifacts -if [ "${platform}" == 'qemu-secex' ]; then - dir=$(dirname "${filepath}") - mv "${outdir}/${platform}/bootfs_hash" "${dir}" - mv "${outdir}/${platform}/rootfs_hash" "${dir}" -fi + ${platforms[@]/#/--export=} \ + "${processed_json}" +mv "${outdir}"/* "${orig_outdir}/" rm -f "${processed_json}" diff --git a/src/vmdeps.txt b/src/vmdeps.txt index ad1a3ff44e..97dd719fd5 100644 --- a/src/vmdeps.txt +++ b/src/vmdeps.txt @@ -37,10 +37,7 @@ tar podman # For running osbuild -osbuild osbuild-ostree osbuild-selinux osbuild-tools python3-pyrsistent -# remove this when osbuild-tools requires python3-dnf -# https://github.com/osbuild/osbuild/pull/1908 -python3-dnf +osbuild osbuild-ostree osbuild-selinux osbuild-tools python3-pyrsistent zip # For resetting the terminal inside supermin shell /usr/bin/reset