osbuild: more refactor to prep for calling osbuild to generate multiple artifacts

src/cmd-build

@@ -181,6 +181,84 @@ build_followup_targets() {
     done
 }
 
+# Parse the passed config JSON and extract a mandatory value
+getconfig() {
+    k=$1
+    config=$2
+    jq -re .\""$k"\" < "${config}"
+}
+
+# Return a configuration value, or default if not set
+getconfig_def() {
+    k=$1
+    shift
+    default=$1
+    config=$2
+    jq -re .\""$k"\"//\""${default}"\" < "${config}"
+}
+
+# Here we generate the input JSON we pass to runvm_osbuild for all of our image builds
+generate_runvm_osbuild_config() {
+    # Grab a few values from $image_json
+    deploy_via_container=$(getconfig_def "deploy-via-container" "" "${image_json}")
+    extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")"
+
+    # The OSTree repo is at $tmprepo and the commit is $commit
+    ostree_repo="${tmprepo}"
+    ostree_commit="${commit}"
+
+    # OStree container ociarchive file path and container_imgref
+    builddir=$(get_build_dir "${buildid}")
+    ostree_container="${builddir}/${ostree_tarfile_path}"
+    # If no container_imgref was set let's just set it to some professional
+    # looking default. The name of the ociarchive file should suffice.
+    container_imgref_default="ostree-image-signed:oci-archive:/$(basename "${ostree_container}")"
+    container_imgref=$(getconfig_def "container_imgref" "${container_imgref_default}" "${image_json}")
+
+    echo "Estimating disk size..."
+    # The additional 35% here is obviously a hack, but we can't easily completely fill the filesystem,
+    # and doing so has apparently negative performance implications.
+    ostree_size_json="$(/usr/lib/coreos-assembler/estimate-commit-disk-size --repo "$ostree_repo" "$commit" --add-percent 35)"
+    rootfs_size_mb="$(jq '."estimate-mb".final' <<< "${ostree_size_json}")"
+    # The minimum size of a disk image we'll need will be the rootfs_size
+    # estimate plus the size of the non-root partitions. We'll use this
+    # size for the metal images, but for the IaaS/virt image we'll use
+    # the size set in the configs since some of them have minimum sizes that
+    # the platforms require and we want a "default" disk size that has some
+    # free space.
+    nonroot_partition_sizes=513
+    # On s390x there is one more build - Secure Execution case, which has
+    # different image layout. We add the sizes of the se and verity
+    # partitions so that they don't "eat into" the 35% buffer (though note
+    # this is all blown away on first boot anyway). For 's390x.mpp.yaml'
+    # simplicity all s390x images have same size (of secex image).
+    if [[ $basearch == "s390x" ]]; then
+        nonroot_partition_sizes=$((nonroot_partition_sizes + 200 + 128 + 256 + 1))
+    fi
+    metal_image_size_mb="$(( rootfs_size_mb + nonroot_partition_sizes ))"
+    cloud_image_size_mb="$(jq -r ".size*1024" < "${image_json}")"
+    echo "Disk sizes: metal: ${metal_image_size_mb}M (estimated), cloud: ${cloud_image_size_mb}M"
+
+    # Generate the JSON describing the disk we want to build
+    runvm_osbuild_config_json="runvm-osbuild-config.json"
+    yaml2json /dev/stdin "tmp/${runvm_osbuild_config_json}" <<EOF
+container-imgref: "${container_imgref}"
+deploy-via-container: "${deploy_via_container}"
+osname: "${name}"
+ostree-container: "${ostree_container}"
+ostree-ref: "${ref}"
+extra-kargs-string: "${extra_kargs}"
+ostree-repo: "${ostree_repo}"
+metal-image-size: "${metal_image_size_mb}"
+cloud-image-size: "${cloud_image_size_mb}"
+# Note: this is only used in the secex case; there, the rootfs is
+# not the last partition on the disk so we need to explicitly size it
+rootfs-size: "${rootfs_size_mb}"
+EOF
+    /usr/lib/coreos-assembler/finalize-artifact "tmp/${runvm_osbuild_config_json}" "${runvm_osbuild_config_json}"
+}
+
+
 prepare_build
 
 ostree --version
@@ -616,6 +694,10 @@ mv "${lockfile_out}" .
 "${dn}"/commitmeta_to_json "${tmprepo}" "${commit}" > commitmeta.json.tmp
 /usr/lib/coreos-assembler/finalize-artifact commitmeta.json{.tmp,}
 
+# Generate the runvm-osbuild config file that will be used for all
+# later image builds.
+generate_runvm_osbuild_config
+
 # Clean up our temporary data
 saved_build_tmpdir="${workdir}/tmp/last-build-tmp"
 rm -rf "${saved_build_tmpdir}"

src/cmd-buildextend-metal

@@ -8,18 +8,18 @@ dn=$(dirname "$0")
 # This script is used for creating both the bare metal and the canonical VM
 # image (qemu). `buildextend-qemu` is a symlink to `buildextend-metal`.
 case "$(basename "$0")" in
-    "cmd-buildextend-metal") image_type=metal;;
-    "cmd-buildextend-metal4k") image_type=metal4k;;
-    "cmd-buildextend-qemu") image_type=qemu;;
-    "cmd-buildextend-qemu-secex") image_type=qemu-secex;;
-    "cmd-buildextend-secex") image_type=qemu-secex;;
+    "cmd-buildextend-metal") platform=metal;;
+    "cmd-buildextend-metal4k") platform=metal4k;;
+    "cmd-buildextend-qemu") platform=qemu;;
+    "cmd-buildextend-qemu-secex") platform=qemu-secex;;
+    "cmd-buildextend-secex") platform=qemu-secex;;
     *) fatal "called as unexpected name $0";;
 esac
 
 print_help() {
     cat 1>&2 <<EOF
-Usage: coreos-assembler buildextend-${image_type} --help
-       coreos-assembler buildextend-${image_type} [--build ID]
+Usage: coreos-assembler buildextend-${platform} --help
+       coreos-assembler buildextend-${platform} [--build ID]
 
   Build a bare metal image.
 EOF
@@ -80,7 +80,7 @@ esac
 
 # shellcheck disable=SC2031
 export LIBGUESTFS_BACKEND=direct
-export IMAGE_TYPE="${image_type}"
+export IMAGE_TYPE="${platform}"
 prepare_build
 
 if [ -z "${build}" ]; then
@@ -96,18 +96,18 @@ if [ ! -d "${builddir}" ]; then
 fi
 
 # add building sempahore
-build_semaphore="${builddir}/.${image_type}.building"
+build_semaphore="${builddir}/.${platform}.building"
 if [ -e "${build_semaphore}" ]; then
-    fatal "${build_semaphore} found: another process is building ${image_type}"
+    fatal "${build_semaphore} found: another process is building ${platform}"
 fi
 touch "${build_semaphore}"
 trap 'rm -f ${build_semaphore}' EXIT
 
 # check if the image already exists in the meta.json
 if [ -z "${force}" ]; then
-    meta_img=$(meta_key "images.${image_type}.path")
+    meta_img=$(meta_key "images.${platform}.path")
     if [ "${meta_img}" != "None" ]; then
-        echo "${image_type} image already exists:"
+        echo "${platform} image already exists:"
         echo "$meta_img"
         exit 0
     fi
@@ -116,123 +116,39 @@ fi
 # reread these values from the build itself rather than rely on the ones loaded
 # by prepare_build since the config might've changed since then
 name=$(meta_key name)
-ref=$(meta_key ref)
-if [ "${ref}" = "None" ]; then
-    ref=""
-fi
-commit=$(meta_key ostree-commit)
-
-ostree_repo=${tmprepo}
-# Ensure that we have the cached unpacked commit
-import_ostree_commit_for_build "${build}"
-# Note this overwrote the bits generated in prepare_build
-# for image_json.  In the future we expect to split prepare_build
-# into prepare_ostree_build and prepare_diskimage_build; the
-# latter path would only run this.
-image_json=${workdir}/tmp/image.json
 
 image_format=raw
-if [[ "${image_type}" == "qemu" || "${image_type}" == "qemu-secex" ]]; then
+if [[ "${platform}" == "qemu" || "${platform}" == "qemu-secex" ]]; then
     image_format=qcow2
 fi
 
-imgname=${name}-${build}-${image_type}.${basearch}.${image_format}
+imgname=${name}-${build}-${platform}.${basearch}.${image_format}
 imgpath=${PWD}/${imgname}
 
-# We do some extra handling of the rootfs here; it feeds into size estimation.
-rootfs_type=$(jq -re .rootfs < "${image_json}")
-
-deploy_via_container=""
-if jq -re '.["deploy-via-container"]' < "${image_json}"; then
-    deploy_via_container="true"
-fi
-# OStree container ociarchive file path
-ostree_container="${builddir}/$(meta_key images.ostree.path)"
-container_imgref=$(jq -r '.["container-imgref"]//""' < "${image_json}")
-if [ -z "${container_imgref}" ]; then
-    # If no container_imgref was set let's just set it to some professional
-    # looking default. The name of the ociarchive file should suffice.
-    container_imgref="ostree-image-signed:oci-archive:/$(basename "${ostree_container}")"
-fi
-
-# fs-verity requires block size = page size. We need to take that into account
-# in the disk size estimation due to higher fragmentation on larger blocks.
-BLKSIZE=""
-if [ "${rootfs_type}" = "ext4verity" ]; then
-    BLKSIZE="$(getconf PAGE_SIZE)"
-fi
-
-echo "Estimating disk size..."
-# The additional 35% here is obviously a hack, but we can't easily completely fill the filesystem,
-# and doing so has apparently negative performance implications.
-/usr/lib/coreos-assembler/estimate-commit-disk-size ${BLKSIZE:+--blksize ${BLKSIZE}} --repo "$ostree_repo" "$commit" --add-percent 35 > "$PWD/tmp/ostree-size.json"
-rootfs_size_mb="$(jq '."estimate-mb".final' "$PWD/tmp/ostree-size.json")"
-# The minimum size of a disk image we'll need will be the rootfs_size
-# estimate plus the size of the non-root partitions. We'll use this
-# size for the metal images, but for the IaaS/virt image we'll use
-# the size set in the configs since some of them have minimum sizes that
-# the platforms require and we want a "default" disk size that has some
-# free space.
-nonroot_partition_sizes=513
-# On s390x there is one more build - Secure Execution case, which has
-# different image layout. We add the sizes of the se and verity
-# partitions so that they don't "eat into" the 35% buffer (though note
-# this is all blown away on first boot anyway). For 's390x.mpp.yaml'
-# simplicity all s390x images have same size (of secex image).
-if [[ $basearch == "s390x" ]]; then
-    nonroot_partition_sizes=$((nonroot_partition_sizes + 200 + 128 + 256 + 1))
-fi
-metal_image_size_mb="$(( rootfs_size_mb + nonroot_partition_sizes ))"
-cloud_image_size_mb="$(jq -r ".size*1024" < "${image_json}")"
-echo "Disk sizes: metal: ${metal_image_size_mb}M (estimated), cloud: ${cloud_image_size_mb}M"
-
-set -x
-extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")"
-
-# Generate the JSON describing the disk we want to build
-image_dynamic_yaml="${tmp_builddir}/image-dynamic.yaml"
-image_dynamic_json="${tmp_builddir}/image-dynamic.json"
-image_for_disk_json="${tmp_builddir}/image-for-disk.json"
-cat >"${image_dynamic_yaml}" << EOF
-container-imgref: "${container_imgref}"
-deploy-via-container: "${deploy_via_container}"
-osname: "${name}"
-ostree-container: "${ostree_container}"
-ostree-ref: "${ref}"
-extra-kargs-string: "${extra_kargs}"
-image-type: "${image_type}"
-ostree-repo: "${ostree_repo}"
-metal-image-size: "${metal_image_size_mb}"
-cloud-image-size: "${cloud_image_size_mb}"
-# Note: this is only used in the secex case; there, the rootfs is
-# not the last partition on the disk so we need to explicitly size it
-rootfs-size: "${rootfs_size_mb}"
-EOF
-yaml2json "${image_dynamic_yaml}" "${image_dynamic_json}"
-cat "${image_json}" "${image_dynamic_json}" | jq -s add > "${image_for_disk_json}"
-platforms_json="${tmp_builddir}/platforms.json"
-yaml2json "${configdir}/platforms.yaml" "${platforms_json}"
-
 # In the jenkins pipelines we build the qemu image first and that operation
 # will do a lot of the same work required for later artifacts (metal, metal4k, etc)
 # so we want the cached output from that run to persist. The later artifacts get
 # built in parallel, so we need to be able to access the cache by multiple processes,
 # so for those we'll set `snapshot=on` so that each will get their own disk image.
 # This is OK because we don't checkpoint (cache) any of those stages.
-[ "${image_type}" == "qemu" ] && snapshot="off" || snapshot="on"
+[ "${platform}" == "qemu" ] && snapshot="off" || snapshot="on"
+outdir=$(mktemp -p "${tmp_builddir}" -d)
 runvm_with_cache_snapshot "$snapshot" -- /usr/lib/coreos-assembler/runvm-osbuild                    \
-            --config "${image_for_disk_json}"                                                       \
+            --config "${builddir}/runvm-osbuild-config.json"                                        \
             --mpp "/usr/lib/coreos-assembler/osbuild-manifests/coreos.osbuild.${basearch}.mpp.yaml" \
-            --filepath "${imgpath}"
+            --outdir "${outdir}"                                                                    \
+            --platform "${platform}"
+
+mv "${outdir}/${platform}/${platform}" "${imgpath}"
 
-if [[ "${image_type}" == "qemu-secex" ]]; then
+if [[ "${platform}" == "qemu-secex" ]]; then
     if [ ! -f "${genprotimgvm}" ]; then
         fatal "No genprotimgvm provided at ${genprotimgvm}"
     fi
 
     # Basic qemu args:
     qemu_args=(); blk_size="512"
-    [[ $image_type == metal4k ]] && blk_size="4096"
+    [[ $platform == metal4k ]] && blk_size="4096"
     qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${imgpath},cache=unsafe" \
         "-device" "virtio-blk,serial=target,drive=target,physical_block_size=${blk_size},logical_block_size=${blk_size}")
 
@@ -242,7 +158,9 @@ if [[ "${image_type}" == "qemu-secex" ]]; then
     genprotimg_dir=$(mktemp -p "${tmp_builddir}" -d)
     cp "${se_script_dir}/genprotimg-script.sh" "${se_script_dir}/post-script.sh" "${genprotimg_dir}"
     # Extra kargs with dm-verity hashes
-    secex_kargs="ignition.firstboot rootfs.roothash=$(<"${PWD}"/rootfs_hash) bootfs.roothash=$(<"${PWD}"/bootfs_hash)"
+    secex_kargs="ignition.firstboot"
+    secex_kargs+=" rootfs.roothash=$(<"${outdir}/${platform}/rootfs_hash")"
+    secex_kargs+=" bootfs.roothash=$(<"${outdir}/${platform}/bootfs_hash")"
     echo "${secex_kargs}" > "${genprotimg_dir}/parmfile"
     virt-make-fs --format=raw --type=ext4 "${genprotimg_dir}" "${genprotimg_img}"
     rm -rf "${genprotimg_dir}"
@@ -271,7 +189,7 @@ sha256=$(sha256sum_str < "${imgpath}")
 cosa meta --workdir "${workdir}" --build "${build}" --dump | python3 -c "
 import sys, json
 j = json.load(sys.stdin)
-j['images']['${image_type}'] = {
+j['images']['${platform}'] = {
     'path': '${imgname}',
     'sha256': '${sha256}',
     'size': $(stat -c '%s' "${imgpath}")
@@ -298,7 +216,7 @@ json.dump(j, sys.stdout, indent=4)
 fi
 
 # and now the crucial bits
-cosa meta --workdir "${workdir}" --build "${build}" --artifact "${image_type}" --artifact-json "$(readlink -f meta.json.new)"
+cosa meta --workdir "${workdir}" --build "${build}" --artifact "${platform}" --artifact-json "$(readlink -f meta.json.new)"
 /usr/lib/coreos-assembler/finalize-artifact "${imgpath}" "${builddir}/${imgname}"
 
 # Quiet for the rest of this so the last thing we see is a success message

src/osbuild-manifests/platform.applehv.ipp.yaml

@@ -65,5 +65,4 @@ pipelines:
         options:
           paths:
             - from: input://tree/disk.img
-              to:
-                mpp-format-string: 'tree:///{filename}'
+              to: tree:///applehv

src/osbuild-manifests/platform.gcp.ipp.yaml

@@ -85,5 +85,4 @@ pipelines:
               name:raw-gcp-image-tar:
                 file: disk.tar
         options:
-          filename:
-            mpp-format-string: '{filename}'
+          filename: gcp

src/osbuild-manifests/platform.hyperv.ipp.yaml

@@ -64,7 +64,6 @@ pipelines:
               name:raw-hyperv-image:
                 file: disk.img
         options:
-          filename:
-            mpp-format-string: '{filename}'
+          filename: hyperv
           format:
             type: vhdx

src/osbuild-manifests/platform.metal.ipp.yaml

@@ -90,8 +90,7 @@ pipelines:
         options:
           paths:
             - from: input://tree/disk.img
-              to:
-                mpp-format-string: 'tree:///{filename}'
+              to: tree:///metal
   - name: raw-metal4k-image
     build:
       mpp-format-string: '{buildroot}'
@@ -184,5 +183,4 @@ pipelines:
         options:
           paths:
             - from: input://tree/disk.img
-              to:
-                mpp-format-string: 'tree:///{filename}'
+              to: tree:///metal4k

src/osbuild-manifests/platform.qemu-secex.ipp.yaml

@@ -288,8 +288,7 @@ pipelines:
               name:raw-qemu-secex-image:
                 file: disk.img
         options:
-          filename:
-            mpp-format-string: '{filename}'
+          filename: qemu-secex
           format:
             type: qcow2
             compression: false

src/osbuild-manifests/platform.qemu.ipp.yaml

@@ -93,8 +93,7 @@ pipelines:
               name:raw-qemu-image:
                 file: disk.img
         options:
-          filename:
-            mpp-format-string: '{filename}'
+          filename: qemu
           format:
             type: qcow2
             compression: false