From 004ad9f2c9aeddf1ff2aac026a17849b99cd9ce9 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 31 Oct 2024 15:55:30 -0400 Subject: [PATCH 1/9] drop remnants of dasd image type It was determined that this was the exact same as the 4k disk image for s390x so we stopped building it. Let's remove the dead references from our code base. Closes https://github.com/coreos/coreos-assembler/issues/3766 --- src/cmd-buildextend-dasd | 1 - src/cmd-buildextend-metal | 11 +++-------- 2 files changed, 3 insertions(+), 9 deletions(-) delete mode 120000 src/cmd-buildextend-dasd diff --git a/src/cmd-buildextend-dasd b/src/cmd-buildextend-dasd deleted file mode 120000 index ad07b13c46..0000000000 --- a/src/cmd-buildextend-dasd +++ /dev/null @@ -1 +0,0 @@ -cmd-buildextend-metal \ No newline at end of file diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index c7ec9d8795..c12c4d55b7 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -14,7 +14,6 @@ image_suffix= case "$(basename "$0")" in "cmd-buildextend-metal") image_type=metal;; "cmd-buildextend-metal4k") image_type=metal4k;; - "cmd-buildextend-dasd") image_type=dasd;; "cmd-buildextend-qemu") image_type=qemu;; "cmd-buildextend-secex") secure_execution=1 @@ -86,10 +85,6 @@ case "$basearch" in *) fatal "$basearch is not supported for this command" ;; esac -if [[ "$basearch" != "s390x" && $image_type == dasd ]]; then - fatal "$basearch is not supported for building dasd images" -fi - # shellcheck disable=SC2031 export LIBGUESTFS_BACKEND=direct export IMAGE_TYPE="${image_type}" @@ -191,7 +186,7 @@ echo "Estimating disk size..." rootfs_size_mb="$(jq '."estimate-mb".final' "$PWD/tmp/ostree-size.json")" # The minimum size of a disk image we'll need will be the rootfs_size # estimate plus the size of the non-root partitions. We'll use this -# size for the metal/dasd images, but for the IaaS/virt image we'll use +# size for the metal images, but for the IaaS/virt image we'll use # the size set in the configs since some of them have minimum sizes that # the platforms require and we want a "default" disk size that has some # free space. @@ -218,8 +213,8 @@ extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra- qemu-img create -f ${image_format} "${path}.tmp" "${metal_image_size_mb}M" extra_target_device_opts="" -# we need 4096 block size for ECKD DASD and (obviously) metal4k -if [[ $image_type == dasd || $image_type == metal4k ]]; then +# we need 4096 block size for metal4k +if [[ $image_type == metal4k ]]; then extra_target_device_opts=",physical_block_size=4096,logical_block_size=4096" fi qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ From 00a1343331983ea646cddfef479e8c76ee9cdbff Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 31 Oct 2024 16:07:46 -0400 Subject: [PATCH 2/9] cmd-buildextend-metal: clean up disk_args This is dead code now since d37958a. --- src/cmd-buildextend-metal | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index c12c4d55b7..ae9fb81030 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -169,11 +169,9 @@ if [ "${rootfs_type}" = "ext4verity" ]; then BLKSIZE="$(getconf PAGE_SIZE)" fi -disk_args=() qemu_args=() # SecureExecution extra stuff if [[ $secure_execution -eq "1" ]]; then - disk_args+=("--with-secure-execution") if [ ! -f "${genprotimgvm}" ]; then fatal "No genprotimgvm provided at ${genprotimgvm}" fi @@ -203,10 +201,6 @@ metal_image_size_mb="$(( rootfs_size_mb + nonroot_partition_sizes ))" cloud_image_size_mb="$(jq -r ".size*1024" < "${image_json}")" echo "Disk sizes: metal: ${metal_image_size_mb}M (estimated), cloud: ${cloud_image_size_mb}M" -if [ "${image_type}" == metal4k ]; then - disk_args+=("--no-x86-bios-bootloader") -fi - set -x extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")" From b36f3cabf7041487f72d46811ce82b9dbed28cae Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 31 Oct 2024 16:11:56 -0400 Subject: [PATCH 3/9] cmd-buildextend-metal: remove qemu-img create Since d37958a this file that gets created is no longer used and gets overwritten by the file created from the OSBuild call. --- src/cmd-buildextend-metal | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index ae9fb81030..abef546642 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -204,8 +204,6 @@ echo "Disk sizes: metal: ${metal_image_size_mb}M (estimated), cloud: ${cloud_ima set -x extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")" -qemu-img create -f ${image_format} "${path}.tmp" "${metal_image_size_mb}M" - extra_target_device_opts="" # we need 4096 block size for metal4k if [[ $image_type == metal4k ]]; then From c7cb857dc572251b70a744cc445ed246d57adffb Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 31 Oct 2024 16:19:06 -0400 Subject: [PATCH 4/9] cmd-buildextend-metal: relocate some s390x only code Since d37958a this code is now only used in the s390x case. Let's move it down and colocate it with the rest of the related code. --- src/cmd-buildextend-metal | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index abef546642..62ac2b9112 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -169,14 +169,6 @@ if [ "${rootfs_type}" = "ext4verity" ]; then BLKSIZE="$(getconf PAGE_SIZE)" fi -qemu_args=() -# SecureExecution extra stuff -if [[ $secure_execution -eq "1" ]]; then - if [ ! -f "${genprotimgvm}" ]; then - fatal "No genprotimgvm provided at ${genprotimgvm}" - fi -fi - echo "Estimating disk size..." # The additional 35% here is obviously a hack, but we can't easily completely fill the filesystem, # and doing so has apparently negative performance implications. @@ -204,14 +196,6 @@ echo "Disk sizes: metal: ${metal_image_size_mb}M (estimated), cloud: ${cloud_ima set -x extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")" -extra_target_device_opts="" -# we need 4096 block size for metal4k -if [[ $image_type == metal4k ]]; then - extra_target_device_opts=",physical_block_size=4096,logical_block_size=4096" -fi -qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ - "-device" "virtio-blk,serial=target,drive=target${extra_target_device_opts}") - # Generate the JSON describing the disk we want to build image_dynamic_yaml="${tmp_builddir}/image-dynamic.yaml" image_dynamic_json="${tmp_builddir}/image-dynamic.json" @@ -257,6 +241,19 @@ runvm_with_cache_snapshot "$snapshot" -- /usr/lib/coreos-assembler/runvm-osbuild --filepath "${path}.tmp" "${osbuild_extra_args[@]}" if [[ $secure_execution -eq "1" ]]; then + if [ ! -f "${genprotimgvm}" ]; then + fatal "No genprotimgvm provided at ${genprotimgvm}" + fi + + # Basic qemu args: + qemu_args=(); extra_target_device_opts="" + # we need 4096 block size for metal4k + if [[ $image_type == metal4k ]]; then + extra_target_device_opts=",physical_block_size=4096,logical_block_size=4096" + fi + qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ + "-device" "virtio-blk,serial=target,drive=target${extra_target_device_opts}") + # SecureVM (holding Universal Key for all IBM Z Mainframes) requires scripts to execute genprotimg se_script_dir="/usr/lib/coreos-assembler/secex-genprotimgvm-scripts" genprotimg_img="${PWD}/secex-genprotimg.img" From 183842d2436dfdd0e91fd30e6765836dfc96a431 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 31 Oct 2024 16:28:25 -0400 Subject: [PATCH 5/9] cmd-buildextend-metal: simplify qemu_args a bit Mostly just trying to make it a little cleaner and eliminate the extra extra_target_device_opts long string. --- src/cmd-buildextend-metal | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 62ac2b9112..252f9f0905 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -246,13 +246,10 @@ if [[ $secure_execution -eq "1" ]]; then fi # Basic qemu args: - qemu_args=(); extra_target_device_opts="" - # we need 4096 block size for metal4k - if [[ $image_type == metal4k ]]; then - extra_target_device_opts=",physical_block_size=4096,logical_block_size=4096" - fi + qemu_args=(); blk_size="512" + [[ $image_type == metal4k ]] && blk_size="4096" qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ - "-device" "virtio-blk,serial=target,drive=target${extra_target_device_opts}") + "-device" "virtio-blk,serial=target,drive=target,physical_block_size=${blk_size},logical_block_size=${blk_size}") # SecureVM (holding Universal Key for all IBM Z Mainframes) requires scripts to execute genprotimg se_script_dir="/usr/lib/coreos-assembler/secex-genprotimgvm-scripts" From 4baa338ceb15acd09f167c805656beb59026d2bc Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 31 Oct 2024 16:24:41 -0400 Subject: [PATCH 6/9] cmd-buildextend-metal: rename path, img vars; don't use `.tmp` stub $path and $img were highly related. Here I rename $path to $imgpath and $img to $imgname to be more explicit and correct some usages. Previously they were being used interchangably because it just so happens `${PWD}/file` and `./file` point to the same place. Here I also stop using `${path}.tmp` everywhere and just use $imgpath because I don't really think the `.tmp` and the intermediate `finalize-artifact "${path}.tmp" "${path}"` are still needed. --- src/cmd-buildextend-metal | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 252f9f0905..8bad7fd51b 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -143,8 +143,8 @@ if [[ $image_type == qemu ]]; then image_format=qcow2 fi -img=${name}-${build}-${image_type}${image_suffix}.${basearch}.${image_format} -path=${PWD}/${img} +imgname=${name}-${build}-${image_type}${image_suffix}.${basearch}.${image_format} +imgpath=${PWD}/${imgname} # We do some extra handling of the rootfs here; it feeds into size estimation. rootfs_type=$(jq -re .rootfs < "${image_json}") @@ -202,7 +202,7 @@ image_dynamic_json="${tmp_builddir}/image-dynamic.json" image_for_disk_json="${tmp_builddir}/image-for-disk.json" cat >"${image_dynamic_yaml}" << EOF buildid: "${build}" -imgid: "${img}" +imgid: "${imgname}" ostree-commit: "${commit}" container-imgref: "${container_imgref}" deploy-via-container: "${deploy_via_container}" @@ -238,7 +238,7 @@ fi runvm_with_cache_snapshot "$snapshot" -- /usr/lib/coreos-assembler/runvm-osbuild \ --config "${image_for_disk_json}" \ --mpp "/usr/lib/coreos-assembler/osbuild-manifests/coreos.osbuild.${basearch}.mpp.yaml" \ - --filepath "${path}.tmp" "${osbuild_extra_args[@]}" + --filepath "${imgpath}" "${osbuild_extra_args[@]}" if [[ $secure_execution -eq "1" ]]; then if [ ! -f "${genprotimgvm}" ]; then @@ -248,7 +248,7 @@ if [[ $secure_execution -eq "1" ]]; then # Basic qemu args: qemu_args=(); blk_size="512" [[ $image_type == metal4k ]] && blk_size="4096" - qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ + qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${imgpath},cache=unsafe" \ "-device" "virtio-blk,serial=target,drive=target,physical_block_size=${blk_size},logical_block_size=${blk_size}") # SecureVM (holding Universal Key for all IBM Z Mainframes) requires scripts to execute genprotimg @@ -282,16 +282,14 @@ if [[ $secure_execution -eq "1" ]]; then exec 9>&- fi -/usr/lib/coreos-assembler/finalize-artifact "${path}.tmp" "${path}" - -sha256=$(sha256sum_str < "${img}") +sha256=$(sha256sum_str < "${imgpath}") cosa meta --workdir "${workdir}" --build "${build}" --dump | python3 -c " import sys, json j = json.load(sys.stdin) j['images']['${image_type}${image_suffix}'] = { - 'path': '${img}', + 'path': '${imgname}', 'sha256': '${sha256}', - 'size': $(stat -c '%s' "${img}") + 'size': $(stat -c '%s' "${imgpath}") } json.dump(j, sys.stdout, indent=4) " | jq -s add > "meta.json.new" @@ -316,10 +314,10 @@ fi # and now the crucial bits cosa meta --workdir "${workdir}" --build "${build}" --artifact "${image_type}" --artifact-json "$(readlink -f meta.json.new)" -/usr/lib/coreos-assembler/finalize-artifact "${img}" "${builddir}/${img}" +/usr/lib/coreos-assembler/finalize-artifact "${imgpath}" "${builddir}/${imgname}" # Quiet for the rest of this so the last thing we see is a success message set +x # clean up the tmpbuild rm -rf "${tmp_builddir}" -echo "Successfully generated: ${img}" +echo "Successfully generated: ${imgname}" From 87380b2466613bd1efa7a8676eb002b85c9856dd Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 31 Oct 2024 16:43:05 -0400 Subject: [PATCH 7/9] cmd-buildextend-metal: drop unused config parameters These aren't used in the runvm-osbuild flow, but were for create_disk.sh, which was dropped in d37958a. --- src/cmd-buildextend-metal | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 8bad7fd51b..ff05f41b39 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -201,9 +201,6 @@ image_dynamic_yaml="${tmp_builddir}/image-dynamic.yaml" image_dynamic_json="${tmp_builddir}/image-dynamic.json" image_for_disk_json="${tmp_builddir}/image-for-disk.json" cat >"${image_dynamic_yaml}" << EOF -buildid: "${build}" -imgid: "${imgname}" -ostree-commit: "${commit}" container-imgref: "${container_imgref}" deploy-via-container: "${deploy_via_container}" osname: "${name}" From 039ca2da1420c50d8f1c5d9afb7cadb59e5666b2 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 1 Nov 2024 11:50:01 -0400 Subject: [PATCH 8/9] simplify building the qemu-secex artifacts We shouldn't need to pass extra parameters around for qemu-secex and should be able to just ask for that "platform" to be produced from OSBuild without specifying more. This commit removes a bunch of extra variables that get set and passed around and also refactors the osbuild manifests to have qemu-secex treated more like the other platforms that we have in its own qemu-secex.ipp.yaml file. This commit also introduces a symlink cmd-buildextend-qemu-secex that points to cmd-buildextend-metal too (in addition to the existing cmd-buildextend-secex symlink). Since `qemu-secex` is the ID that is used in the meta.json I think we should try to stick with it more. --- src/cmd-buildextend-metal | 30 +- src/cmd-buildextend-qemu-secex | 1 + .../coreos.osbuild.s390x.mpp.yaml | 171 +--------- .../platform.qemu-secex.ipp.yaml | 320 ++++++++++++++++++ src/osbuild-manifests/platform.qemu.ipp.yaml | 114 +------ src/runvm-osbuild | 6 +- 6 files changed, 346 insertions(+), 296 deletions(-) create mode 120000 src/cmd-buildextend-qemu-secex create mode 100644 src/osbuild-manifests/platform.qemu-secex.ipp.yaml diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index ff05f41b39..d7c373c393 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -5,21 +5,14 @@ dn=$(dirname "$0") # shellcheck source=src/cmdlib.sh . "${dn}"/cmdlib.sh -# IBM SecureExecution -secure_execution= -image_suffix= - # This script is used for creating both the bare metal and the canonical VM # image (qemu). `buildextend-qemu` is a symlink to `buildextend-metal`. case "$(basename "$0")" in "cmd-buildextend-metal") image_type=metal;; "cmd-buildextend-metal4k") image_type=metal4k;; "cmd-buildextend-qemu") image_type=qemu;; - "cmd-buildextend-secex") - secure_execution=1 - image_type=qemu - image_suffix=-secex - ;; + "cmd-buildextend-qemu-secex") image_type=qemu-secex;; + "cmd-buildextend-secex") image_type=qemu-secex;; *) fatal "called as unexpected name $0";; esac @@ -112,9 +105,9 @@ trap 'rm -f ${build_semaphore}' EXIT # check if the image already exists in the meta.json if [ -z "${force}" ]; then - meta_img=$(meta_key "images.${image_type}${image_suffix}.path") + meta_img=$(meta_key "images.${image_type}.path") if [ "${meta_img}" != "None" ]; then - echo "${image_type}${image_suffix} image already exists:" + echo "${image_type} image already exists:" echo "$meta_img" exit 0 fi @@ -139,11 +132,11 @@ import_ostree_commit_for_build "${build}" image_json=${workdir}/tmp/image.json image_format=raw -if [[ $image_type == qemu ]]; then +if [[ "${image_type}" == "qemu" || "${image_type}" == "qemu-secex" ]]; then image_format=qcow2 fi -imgname=${name}-${build}-${image_type}${image_suffix}.${basearch}.${image_format} +imgname=${name}-${build}-${image_type}.${basearch}.${image_format} imgpath=${PWD}/${imgname} # We do some extra handling of the rootfs here; it feeds into size estimation. @@ -220,11 +213,6 @@ cat "${image_json}" "${image_dynamic_json}" | jq -s add > "${image_for_disk_json platforms_json="${tmp_builddir}/platforms.json" yaml2json "${configdir}/platforms.yaml" "${platforms_json}" -osbuild_extra_args=() -if [[ $secure_execution -eq "1" ]]; then - osbuild_extra_args+=("--secex" "1") -fi - # In the jenkins pipelines we build the qemu image first and that operation # will do a lot of the same work required for later artifacts (metal, metal4k, etc) # so we want the cached output from that run to persist. The later artifacts get @@ -235,9 +223,9 @@ fi runvm_with_cache_snapshot "$snapshot" -- /usr/lib/coreos-assembler/runvm-osbuild \ --config "${image_for_disk_json}" \ --mpp "/usr/lib/coreos-assembler/osbuild-manifests/coreos.osbuild.${basearch}.mpp.yaml" \ - --filepath "${imgpath}" "${osbuild_extra_args[@]}" + --filepath "${imgpath}" -if [[ $secure_execution -eq "1" ]]; then +if [[ "${image_type}" == "qemu-secex" ]]; then if [ ! -f "${genprotimgvm}" ]; then fatal "No genprotimgvm provided at ${genprotimgvm}" fi @@ -283,7 +271,7 @@ sha256=$(sha256sum_str < "${imgpath}") cosa meta --workdir "${workdir}" --build "${build}" --dump | python3 -c " import sys, json j = json.load(sys.stdin) -j['images']['${image_type}${image_suffix}'] = { +j['images']['${image_type}'] = { 'path': '${imgname}', 'sha256': '${sha256}', 'size': $(stat -c '%s' "${imgpath}") diff --git a/src/cmd-buildextend-qemu-secex b/src/cmd-buildextend-qemu-secex new file mode 120000 index 0000000000..c2fa2326d8 --- /dev/null +++ b/src/cmd-buildextend-qemu-secex @@ -0,0 +1 @@ +./cmd-buildextend-metal \ No newline at end of file diff --git a/src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml index a12d53d345..598debfaf1 100644 --- a/src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml @@ -17,8 +17,6 @@ mpp-vars: efi_system_size_mb: 127 se_size_mb: 200 boot_size_mb: 384 - root_size_mb: - mpp-format-int: $rootfs_size_mb boot_verity_size_mb: 128 root_verity_size_mb: 256 sector_size: 512 @@ -46,8 +44,6 @@ mpp-vars: # the host buildroot is the default if nothing is specified. # We're still defining it here in an attempt to be explicit. qemu_stage_buildroot: "" - # IBM Secure Execution - qemu_secex: $qemu_secex mpp-define-images: - id: image sector_size: @@ -83,40 +79,6 @@ mpp-define-images: - name: root type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 partnum: 4 - # Secure Execution image. It MUST contain same partitions as `image` plus 3 additional - - id: image_secex - sector_size: - mpp-format-int: "{sector_size}" - size: - mpp-format-string: "{metal_image_size_mb * 1024 * 1024}" - table: - uuid: 00000000-0000-4000-a000-000000000001 - label: gpt - partitions: - - name: se - type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 - partnum: 1 - size: - mpp-format-int: "{se_size_mb * 1024 * 1024 / sector_size}" - - name: boot - type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 - partnum: 3 - size: - mpp-format-int: "{boot_size_mb * 1024 * 1024 / sector_size}" - - name: root - type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 - partnum: 4 - size: - mpp-format-int: "{root_size_mb * 1024 * 1024 / sector_size}" - - name: boothash - partnum: 5 - size: - mpp-format-int: "{boot_verity_size_mb * 1024 * 1024 / sector_size}" - - name: roothash - type: B325BFBE-C7BE-4AB8-8357-139E652D2F6B - partnum: 6 - size: - mpp-format-int: "{root_verity_size_mb * 1024 * 1024 / sector_size}" pipelines: # If installing from container then let's pull the container file into a pipeline - name: oci-archive @@ -196,13 +158,6 @@ pipelines: # filesystem by OSTree (boot -> .) that makes it so that /boot paths # will always work. bootprefix: true - # If on s390x with secex then mkdir for filesytem labeled `se`, where `sdboot` image gets stored - - mpp-if: qemu_secex != '' - then: - type: org.osbuild.mkdir - options: - paths: - - path: /se - type: org.osbuild.ignition # Deploy via OSTree repo if specified, otherwise ociarchive or container. - mpp-if: ostree_repo != '' @@ -530,131 +485,9 @@ pipelines: source: mount deployment: default: true - # IBM Secure Execution (secex) image has special layout - - name: raw-secex-image - build: - mpp-format-string: '{buildroot}' - stages: - - type: org.osbuild.truncate - options: - filename: disk.img - size: - mpp-format-string: '{image_secex.size}' - - type: org.osbuild.sfdisk - devices: - device: - type: org.osbuild.loopback - options: - filename: disk.img - options: - mpp-format-json: '{image_secex.layout}' - - type: org.osbuild.mkfs.ext4 - devices: - device: - type: org.osbuild.loopback - options: - filename: disk.img - start: - mpp-format-int: '{image_secex.layout[''se''].start}' - size: - mpp-format-int: '{image_secex.layout[''se''].size}' - lock: true - options: - uuid: random - label: - mpp-format-string: '{sd_fs_label}' - - type: org.osbuild.mkfs.ext4 - devices: - device: - type: org.osbuild.loopback - options: - filename: disk.img - start: - mpp-format-int: '{image_secex.layout[''boot''].start}' - size: - mpp-format-int: '{image_secex.layout[''boot''].size}' - lock: true - options: - uuid: - mpp-format-string: '{boot_fs_uuid}' - label: - mpp-format-string: '{boot_fs_label}' - # Set manually the metadata_csum_seed ext4 option otherwise changing the - # filesystem UUID while it's mounted doesn't work. Can remove this when - # metadata_csum_seed is default in RHEL, which can be checked by looking - # in /etc/mke2fs.conf. - metadata_csum_seed: true - - type: org.osbuild.mkfs.xfs - devices: - device: - type: org.osbuild.loopback - options: - filename: disk.img - start: - mpp-format-int: '{image_secex.layout[''root''].start}' - size: - mpp-format-int: '{image_secex.layout[''root''].size}' - lock: true - options: - uuid: - mpp-format-string: '{root_fs_uuid}' - label: - mpp-format-string: '{root_fs_label}' - - type: org.osbuild.copy - inputs: - tree: - type: org.osbuild.tree - origin: org.osbuild.pipeline - references: - - name:tree - options: - paths: - - from: input://tree/ - to: mount://root/ - devices: - disk: - type: org.osbuild.loopback - options: - filename: disk.img - partscan: true - mounts: - - name: root - type: org.osbuild.xfs - source: disk - partition: - mpp-format-int: '{image_secex.layout[''root''].partnum}' - target: / - - name: boot - type: org.osbuild.ext4 - source: disk - partition: - mpp-format-int: '{image_secex.layout[''boot''].partnum}' - target: /boot - - type: org.osbuild.chattr - options: - items: - mount://root/: - immutable: true - devices: - disk: - type: org.osbuild.loopback - options: - filename: disk.img - partscan: true - mounts: - - name: root - type: org.osbuild.xfs - source: disk - partition: - mpp-format-int: '{image_secex.layout[''root''].partnum}' - target: / - - name: ostree.deployment - type: org.osbuild.ostree.deployment - options: - source: mount - deployment: - default: true - mpp-import-pipelines: path: platform.metal.ipp.yaml - mpp-import-pipelines: path: platform.qemu.ipp.yaml + - mpp-import-pipelines: + path: platform.qemu-secex.ipp.yaml diff --git a/src/osbuild-manifests/platform.qemu-secex.ipp.yaml b/src/osbuild-manifests/platform.qemu-secex.ipp.yaml new file mode 100644 index 0000000000..f01a38f13b --- /dev/null +++ b/src/osbuild-manifests/platform.qemu-secex.ipp.yaml @@ -0,0 +1,320 @@ +# This file defines the artifact to be used for the s390x secex platform. +version: '2' +mpp-vars: + root_size_mb: + mpp-format-int: $rootfs_size_mb +mpp-define-images: + # Secure Execution image. It MUST contain same partitions as `image` plus 3 additional + - id: image_secex + sector_size: + mpp-format-int: "{sector_size}" + size: + mpp-format-string: "{metal_image_size_mb * 1024 * 1024}" + table: + uuid: 00000000-0000-4000-a000-000000000001 + label: gpt + partitions: + - name: se + type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + partnum: 1 + size: + mpp-format-int: "{se_size_mb * 1024 * 1024 / sector_size}" + - name: boot + type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + partnum: 3 + size: + mpp-format-int: "{boot_size_mb * 1024 * 1024 / sector_size}" + - name: root + type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + partnum: 4 + size: + mpp-format-int: "{root_size_mb * 1024 * 1024 / sector_size}" + - name: boothash + partnum: 5 + size: + mpp-format-int: "{boot_verity_size_mb * 1024 * 1024 / sector_size}" + - name: roothash + type: B325BFBE-C7BE-4AB8-8357-139E652D2F6B + partnum: 6 + size: + mpp-format-int: "{root_verity_size_mb * 1024 * 1024 / sector_size}" +pipelines: + # IBM Secure Execution (secex) image has special layout + - name: raw-secex-image + build: + mpp-format-string: '{buildroot}' + stages: + - type: org.osbuild.truncate + options: + filename: disk.img + size: + mpp-format-string: '{image_secex.size}' + - type: org.osbuild.sfdisk + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + options: + mpp-format-json: '{image_secex.layout}' + - type: org.osbuild.mkfs.ext4 + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image_secex.layout[''boot''].start}' + size: + mpp-format-int: '{image_secex.layout[''boot''].size}' + lock: true + options: + uuid: + mpp-format-string: '{boot_fs_uuid}' + label: + mpp-format-string: '{boot_fs_label}' + # Set manually the metadata_csum_seed ext4 option otherwise changing the + # filesystem UUID while it's mounted doesn't work. Can remove this when + # metadata_csum_seed is default in RHEL, which can be checked by looking + # in /etc/mke2fs.conf. + metadata_csum_seed: true + - type: org.osbuild.mkfs.xfs + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image_secex.layout[''root''].start}' + size: + mpp-format-int: '{image_secex.layout[''root''].size}' + lock: true + options: + uuid: + mpp-format-string: '{root_fs_uuid}' + label: + mpp-format-string: '{root_fs_label}' + - type: org.osbuild.mkfs.ext4 + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image_secex.layout[''se''].start}' + size: + mpp-format-int: '{image_secex.layout[''se''].size}' + lock: true + options: + uuid: random + label: + mpp-format-string: '{sd_fs_label}' + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:tree + options: + paths: + - from: input://tree/ + to: mount://root/ + devices: + disk: + type: org.osbuild.loopback + options: + filename: disk.img + partscan: true + mounts: + - name: root + type: org.osbuild.xfs + source: disk + partition: + mpp-format-int: '{image_secex.layout[''root''].partnum}' + target: / + - name: boot + type: org.osbuild.ext4 + source: disk + partition: + mpp-format-int: '{image_secex.layout[''boot''].partnum}' + target: /boot + - type: org.osbuild.chattr + options: + items: + mount://root/: + immutable: true + devices: + disk: + type: org.osbuild.loopback + options: + filename: disk.img + partscan: true + mounts: + - name: root + type: org.osbuild.xfs + source: disk + partition: + mpp-format-int: '{image_secex.layout[''root''].partnum}' + target: / + - name: ostree.deployment + type: org.osbuild.ostree.deployment + options: + source: mount + deployment: + default: true + # For secex mkdir for filesytem labeled `se`, where `sdboot` image gets stored + - type: org.osbuild.mkdir + options: + paths: + - path: mount://root/se + devices: + disk: + type: org.osbuild.loopback + options: + filename: disk.img + partscan: true + mounts: + - name: root + type: org.osbuild.xfs + source: disk + partition: + mpp-format-int: '{image_secex.layout[''root''].partnum}' + target: / + - name: raw-qemu-secex-image + build: + mpp-format-string: '{buildroot}' + stages: + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:raw-secex-image + options: + paths: + - from: input://tree/disk.img + to: tree:///disk.img + # Increase the size to the cloud image size + - type: org.osbuild.truncate + options: + filename: disk.img + size: + mpp-format-string: "{cloud_image_size_mb * 1024 * 1024}" + # Still use `qemu` as the platform name here even though this + # is the `qemu-secex` artifact we are creating. similar to how + # `metal4k` still uses `metal` as its platform. + - type: org.osbuild.coreos.platform + options: + platform: qemu + devices: + disk: + type: org.osbuild.loopback + options: + filename: disk.img + partscan: true + mounts: + - name: root + type: org.osbuild.xfs + source: disk + partition: + mpp-format-int: '{image.layout[''root''].partnum}' + target: / + - name: ostree.deployment + type: org.osbuild.ostree.deployment + options: + source: mount + deployment: + default: true + - name: boot + type: org.osbuild.ext4 + source: disk + partition: + mpp-format-int: '{image.layout[''boot''].partnum}' + target: /boot + # For secex setup dm-verity for 'boot' + - type: org.osbuild.dmverity + options: + root_hash_file: "bootfs_hash" + devices: + data_device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image_secex.layout[''boot''].start}' + size: + mpp-format-int: '{image_secex.layout[''boot''].size}' + hash_device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image_secex.layout[''boothash''].start}' + size: + mpp-format-int: '{image_secex.layout[''boothash''].size}' + # For secex setup dm-verity for 'root' + - type: org.osbuild.dmverity + options: + root_hash_file: "rootfs_hash" + devices: + data_device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image_secex.layout[''root''].start}' + size: + mpp-format-int: '{image_secex.layout[''root''].size}' + hash_device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image_secex.layout[''roothash''].start}' + size: + mpp-format-int: '{image_secex.layout[''roothash''].size}' + - name: qemu-secex + build: + mpp-format-string: '{qemu_stage_buildroot}' + stages: + - type: org.osbuild.qemu + inputs: + image: + type: org.osbuild.files + origin: org.osbuild.pipeline + references: + name:raw-qemu-secex-image: + file: disk.img + options: + filename: + mpp-format-string: '{filename}' + format: + type: qcow2 + compression: false + compat: '1.1' + # For secex export hash for 'boot' + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:raw-qemu-secex-image + options: + paths: + - from: input://tree/bootfs_hash + to: tree:///bootfs_hash + # For secex export hash for 'root' + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:raw-qemu-secex-image + options: + paths: + - from: input://tree/rootfs_hash + to: tree:///rootfs_hash diff --git a/src/osbuild-manifests/platform.qemu.ipp.yaml b/src/osbuild-manifests/platform.qemu.ipp.yaml index d7441f946d..13d0c96b36 100644 --- a/src/osbuild-manifests/platform.qemu.ipp.yaml +++ b/src/osbuild-manifests/platform.qemu.ipp.yaml @@ -5,31 +5,17 @@ pipelines: build: mpp-format-string: '{buildroot}' stages: - - mpp-if: qemu_secex == '' - then: - type: org.osbuild.copy - inputs: - tree: - type: org.osbuild.tree - origin: org.osbuild.pipeline - references: - - name:raw-image - options: - paths: - - from: input://tree/disk.img - to: tree:///disk.img - else: - type: org.osbuild.copy - inputs: - tree: - type: org.osbuild.tree - origin: org.osbuild.pipeline - references: - - name:raw-secex-image - options: - paths: - - from: input://tree/disk.img - to: tree:///disk.img + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:raw-image + options: + paths: + - from: input://tree/disk.img + to: tree:///disk.img # Increase the size to the cloud image size - type: org.osbuild.truncate options: @@ -64,9 +50,9 @@ pipelines: partition: mpp-format-int: '{image.layout[''boot''].partnum}' target: /boot - # If on s390x without secex then run zipl, which must run after the kernel + # If on s390x then run zipl, which must run after the kernel # arguments get finalized in the coreos.platform stage above - - mpp-if: arch == 's390x' and qemu_secex == '' + - mpp-if: arch == 's390x' then: type: org.osbuild.zipl.inst options: @@ -94,52 +80,6 @@ pipelines: partition: mpp-format-int: '{image.layout[''boot''].partnum}' target: /boot - # If on s390x with secex then setup dm-verity for 'boot' - - mpp-if: arch == 's390x' and qemu_secex != '' - then: - type: org.osbuild.dmverity - options: - root_hash_file: "bootfs_hash" - devices: - data_device: - type: org.osbuild.loopback - options: - filename: disk.img - start: - mpp-format-int: '{image_secex.layout[''boot''].start}' - size: - mpp-format-int: '{image_secex.layout[''boot''].size}' - hash_device: - type: org.osbuild.loopback - options: - filename: disk.img - start: - mpp-format-int: '{image_secex.layout[''boothash''].start}' - size: - mpp-format-int: '{image_secex.layout[''boothash''].size}' - # If on s390x with secex then setup dm-verity for 'root' - - mpp-if: arch == 's390x' and qemu_secex != '' - then: - type: org.osbuild.dmverity - options: - root_hash_file: "rootfs_hash" - devices: - data_device: - type: org.osbuild.loopback - options: - filename: disk.img - start: - mpp-format-int: '{image_secex.layout[''root''].start}' - size: - mpp-format-int: '{image_secex.layout[''root''].size}' - hash_device: - type: org.osbuild.loopback - options: - filename: disk.img - start: - mpp-format-int: '{image_secex.layout[''roothash''].start}' - size: - mpp-format-int: '{image_secex.layout[''roothash''].size}' - name: qemu build: mpp-format-string: '{qemu_stage_buildroot}' @@ -159,31 +99,3 @@ pipelines: type: qcow2 compression: false compat: '1.1' - # If on s390x with secex then export hash for 'boot' - - mpp-if: arch == 's390x' and qemu_secex != '' - then: - type: org.osbuild.copy - inputs: - tree: - type: org.osbuild.tree - origin: org.osbuild.pipeline - references: - - name:raw-qemu-image - options: - paths: - - from: input://tree/bootfs_hash - to: tree:///bootfs_hash - # If on s390x with secex then export hash for 'root' - - mpp-if: arch == 's390x' and qemu_secex != '' - then: - type: org.osbuild.copy - inputs: - tree: - type: org.osbuild.tree - origin: org.osbuild.pipeline - references: - - name:raw-qemu-image - options: - paths: - - from: input://tree/rootfs_hash - to: tree:///rootfs_hash diff --git a/src/runvm-osbuild b/src/runvm-osbuild index ceb12ba755..4cedd7090f 100755 --- a/src/runvm-osbuild +++ b/src/runvm-osbuild @@ -11,7 +11,6 @@ Options: --help: show this help --mpp: the path to the OSBuild mpp.yaml file --filepath: where to write the created image file - --secex: Build qemu-secex image You probably don't want to run this script by hand. This script is run as part of 'coreos-assembler build'. @@ -32,7 +31,6 @@ getconfig_def() { jq -re .\""$k"\"//\""${default}"\" < "${config}" } -secex="" while [ $# -gt 0 ]; do flag="${1}"; shift; @@ -41,7 +39,6 @@ do --help) usage; exit;; --mpp) mppyaml="${1}"; shift;; --filepath) filepath="${1}"; shift;; - --secex) secex="${1}"; shift;; *) echo "${flag} is not understood."; usage; exit 10;; esac; done @@ -102,7 +99,6 @@ osbuild-mpp \ -D metal_image_size_mb="${metal_image_size_mb}" \ -D cloud_image_size_mb="${cloud_image_size_mb}" \ -D rootfs_size_mb="${rootfs_size_mb}" \ - -D qemu_secex=\""${secex}"\" \ "${mppyaml}" "${processed_json}" # Build the image @@ -121,7 +117,7 @@ osbuild \ mv "${outdir}/${platform}/${filename}" "${filepath}" # In case of IBM Secure Execution there are more artifacts -if [ -n "${secex}" ]; then +if [ "${platform}" == 'qemu-secex' ]; then dir=$(dirname "${filepath}") mv "${outdir}/${platform}/bootfs_hash" "${dir}" mv "${outdir}/${platform}/rootfs_hash" "${dir}" From 343224459626c0de8b5330a0fc92e87fee73c10e Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 1 Nov 2024 12:06:52 -0400 Subject: [PATCH 9/9] osbuild: backport patch for mkdir That allows making directories on mount:// targets. https://github.com/osbuild/osbuild/pull/1904 --- build.sh | 4 +- ...kdir-support-creating-dirs-on-mounts.patch | 109 ++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 src/0001-org.osbuild.mkdir-support-creating-dirs-on-mounts.patch diff --git a/build.sh b/build.sh index 9ec701a0b9..65b3f0dc49 100755 --- a/build.sh +++ b/build.sh @@ -173,7 +173,9 @@ patch_osbuild() { mv /usr/bin/osbuild-mpp /usr/lib/osbuild/tools/ # Now all the software is under the /usr/lib/osbuild dir and we can patch - patch -d /usr/lib/osbuild -p1 < /usr/lib/coreos-assembler/0001-hacks-for-coreos-selinux-issues.patch + cat /usr/lib/coreos-assembler/0001-hacks-for-coreos-selinux-issues.patch \ + /usr/lib/coreos-assembler/0001-org.osbuild.mkdir-support-creating-dirs-on-mounts.patch \ + | patch -d /usr/lib/osbuild -p1 # And then move the files back; supermin appliance creation will need it back # in the places delivered by the RPM. diff --git a/src/0001-org.osbuild.mkdir-support-creating-dirs-on-mounts.patch b/src/0001-org.osbuild.mkdir-support-creating-dirs-on-mounts.patch new file mode 100644 index 0000000000..eb915e259b --- /dev/null +++ b/src/0001-org.osbuild.mkdir-support-creating-dirs-on-mounts.patch @@ -0,0 +1,109 @@ +From 362a1ea2485ea2c49e6c250a0446bd5a33b2062c Mon Sep 17 00:00:00 2001 +From: Nikita Dubrovskii +Date: Mon, 30 Sep 2024 15:46:31 +0200 +Subject: [PATCH] org.osbuild.mkdir: support creating dirs on mounts + +This allows creating new directories on mounts: +``` +- type: org.osbuild.mkdir + options: + paths: + - path: mount:///boot/efi + devices: + disk: ... + mounts: + - name: boot + target: /boot + ... +``` +--- + stages/org.osbuild.mkdir | 22 ++++++++++++---------- + stages/org.osbuild.mkdir.meta.json | 21 ++++++++++++++++++--- + 2 files changed, 30 insertions(+), 13 deletions(-) + +diff --git a/stages/org.osbuild.mkdir b/stages/org.osbuild.mkdir +index f04549f6..d2d11a7a 100755 +--- a/stages/org.osbuild.mkdir ++++ b/stages/org.osbuild.mkdir +@@ -3,23 +3,26 @@ import os + import sys + + import osbuild.api +-from osbuild.util.path import in_tree ++from osbuild.util import parsing + + +-def main(tree, options): ++def main(args): ++ options = args["options"] ++ + for item in options["paths"]: + path = item["path"] + mode = item.get("mode", 0o777) + parents = item.get("parents", False) + exist_ok = item.get("exist_ok", False) + +- if not path.startswith("/"): +- print("WARNING: relative path used, this is discouraged!") +- +- target = os.path.join(tree, path.lstrip("/")) +- if not in_tree(target, tree): +- raise ValueError(f"path {path} not in tree") ++ if "://" not in path: ++ if not path.startswith("/"): ++ print("WARNING: relative path used, this is discouraged!") ++ path = f"tree:///{path}" ++ else: ++ path = f"tree://{path}" + ++ target = parsing.parse_location(path, args) + if parents: + os.makedirs(target, mode=mode, exist_ok=exist_ok) + else: +@@ -33,5 +36,4 @@ def main(tree, options): + + + if __name__ == "__main__": +- args = osbuild.api.arguments() +- sys.exit(main(args["tree"], args["options"])) ++ sys.exit(main(osbuild.api.arguments())) +diff --git a/stages/org.osbuild.mkdir.meta.json b/stages/org.osbuild.mkdir.meta.json +index 5534120a..6cebaaf5 100644 +--- a/stages/org.osbuild.mkdir.meta.json ++++ b/stages/org.osbuild.mkdir.meta.json +@@ -1,5 +1,5 @@ + { +- "summary": "Create directories within the tree.", ++ "summary": "Create directories within the tree or mount.", + "description": [ + "Can create one or more directories, optionally also the", + "intermediate directories. The stage can gracefully handle", +@@ -31,8 +31,23 @@ + ], + "properties": { + "path": { +- "type": "string", +- "pattern": "^\\/?(?!\\.\\.)((?!\\/\\.\\.\\/).)+$" ++ "anyOf": [ ++ { ++ "type": "string", ++ "description": "Target path, if a tree", ++ "pattern": "^\\/?(?!\\.\\.)((?!\\/\\.\\.\\/).)+$" ++ }, ++ { ++ "type": "string", ++ "description": "Target path, if a mount", ++ "pattern": "^mount://.+" ++ }, ++ { ++ "type": "string", ++ "description": "Target path, if a tree", ++ "pattern": "^tree://.+" ++ } ++ ] + }, + "mode": { + "type": "number", +-- +2.47.0 +