rework osbuild s390x; drop dead code from recent cleanups

build.sh

@@ -173,7 +173,9 @@ patch_osbuild() {
     mv /usr/bin/osbuild-mpp /usr/lib/osbuild/tools/
 
     # Now all the software is under the /usr/lib/osbuild dir and we can patch
-    patch -d /usr/lib/osbuild -p1 < /usr/lib/coreos-assembler/0001-hacks-for-coreos-selinux-issues.patch
+    cat /usr/lib/coreos-assembler/0001-hacks-for-coreos-selinux-issues.patch                      \
+        /usr/lib/coreos-assembler/0001-org.osbuild.mkdir-support-creating-dirs-on-mounts.patch    \
+        | patch -d /usr/lib/osbuild -p1
 
     # And then move the files back; supermin appliance creation will need it back
     # in the places delivered by the RPM.

src/0001-org.osbuild.mkdir-support-creating-dirs-on-mounts.patch

@@ -0,0 +1,109 @@
+From 362a1ea2485ea2c49e6c250a0446bd5a33b2062c Mon Sep 17 00:00:00 2001
+From: Nikita Dubrovskii <[email protected]>
+Date: Mon, 30 Sep 2024 15:46:31 +0200
+Subject: [PATCH] org.osbuild.mkdir: support creating dirs on mounts
+
+This allows creating new directories on mounts:
+```
+- type: org.osbuild.mkdir
+  options:
+    paths:
+      - path: mount:///boot/efi
+  devices:
+    disk: ...
+  mounts:
+    - name: boot
+      target: /boot
+      ...
+```
+---
+ stages/org.osbuild.mkdir           | 22 ++++++++++++----------
+ stages/org.osbuild.mkdir.meta.json | 21 ++++++++++++++++++---
+ 2 files changed, 30 insertions(+), 13 deletions(-)
+
+diff --git a/stages/org.osbuild.mkdir b/stages/org.osbuild.mkdir
+index f04549f6..d2d11a7a 100755
+--- a/stages/org.osbuild.mkdir
++++ b/stages/org.osbuild.mkdir
+@@ -3,23 +3,26 @@ import os
+ import sys
+
+ import osbuild.api
+-from osbuild.util.path import in_tree
++from osbuild.util import parsing
+
+
+-def main(tree, options):
++def main(args):
++    options = args["options"]
++
+     for item in options["paths"]:
+         path = item["path"]
+         mode = item.get("mode", 0o777)
+         parents = item.get("parents", False)
+         exist_ok = item.get("exist_ok", False)
+
+-        if not path.startswith("/"):
+-            print("WARNING: relative path used, this is discouraged!")
+-
+-        target = os.path.join(tree, path.lstrip("/"))
+-        if not in_tree(target, tree):
+-            raise ValueError(f"path {path} not in tree")
++        if "://" not in path:
++            if not path.startswith("/"):
++                print("WARNING: relative path used, this is discouraged!")
++                path = f"tree:///{path}"
++            else:
++                path = f"tree://{path}"
+
++        target = parsing.parse_location(path, args)
+         if parents:
+             os.makedirs(target, mode=mode, exist_ok=exist_ok)
+         else:
+@@ -33,5 +36,4 @@ def main(tree, options):
+
+
+ if __name__ == "__main__":
+-    args = osbuild.api.arguments()
+-    sys.exit(main(args["tree"], args["options"]))
++    sys.exit(main(osbuild.api.arguments()))
+diff --git a/stages/org.osbuild.mkdir.meta.json b/stages/org.osbuild.mkdir.meta.json
+index 5534120a..6cebaaf5 100644
+--- a/stages/org.osbuild.mkdir.meta.json
++++ b/stages/org.osbuild.mkdir.meta.json
+@@ -1,5 +1,5 @@
+ {
+-  "summary": "Create directories within the tree.",
++  "summary": "Create directories within the tree or mount.",
+   "description": [
+     "Can create one or more directories, optionally also the",
+     "intermediate directories. The stage can gracefully handle",
+@@ -31,8 +31,23 @@
+             ],
+             "properties": {
+               "path": {
+-                "type": "string",
+-                "pattern": "^\\/?(?!\\.\\.)((?!\\/\\.\\.\\/).)+$"
++                "anyOf": [
++                  {
++                    "type": "string",
++                    "description": "Target path, if a tree",
++                    "pattern": "^\\/?(?!\\.\\.)((?!\\/\\.\\.\\/).)+$"
++                  },
++                  {
++                    "type": "string",
++                    "description": "Target path, if a mount",
++                    "pattern": "^mount://.+"
++                  },
++                  {
++                    "type": "string",
++                    "description": "Target path, if a tree",
++                    "pattern": "^tree://.+"
++                  }
++                ]
+               },
+               "mode": {
+                 "type": "number",
+-- 
+2.47.0
+

src/cmd-buildextend-metal

@@ -5,22 +5,14 @@ dn=$(dirname "$0")
 # shellcheck source=src/cmdlib.sh
 . "${dn}"/cmdlib.sh
 
-# IBM SecureExecution
-secure_execution=
-image_suffix=
-
 # This script is used for creating both the bare metal and the canonical VM
 # image (qemu). `buildextend-qemu` is a symlink to `buildextend-metal`.
 case "$(basename "$0")" in
     "cmd-buildextend-metal") image_type=metal;;
     "cmd-buildextend-metal4k") image_type=metal4k;;
-    "cmd-buildextend-dasd") image_type=dasd;;
     "cmd-buildextend-qemu") image_type=qemu;;
-    "cmd-buildextend-secex")
-        secure_execution=1
-        image_type=qemu
-        image_suffix=-secex
-        ;;
+    "cmd-buildextend-qemu-secex") image_type=qemu-secex;;
+    "cmd-buildextend-secex") image_type=qemu-secex;;
     *) fatal "called as unexpected name $0";;
 esac
 
@@ -86,10 +78,6 @@ case "$basearch" in
     *) fatal "$basearch is not supported for this command" ;;
 esac
 
-if [[ "$basearch" != "s390x" && $image_type == dasd ]]; then
-    fatal "$basearch is not supported for building dasd images"
-fi
-
 # shellcheck disable=SC2031
 export LIBGUESTFS_BACKEND=direct
 export IMAGE_TYPE="${image_type}"
@@ -117,9 +105,9 @@ trap 'rm -f ${build_semaphore}' EXIT
 
 # check if the image already exists in the meta.json
 if [ -z "${force}" ]; then
-    meta_img=$(meta_key "images.${image_type}${image_suffix}.path")
+    meta_img=$(meta_key "images.${image_type}.path")
     if [ "${meta_img}" != "None" ]; then
-        echo "${image_type}${image_suffix} image already exists:"
+        echo "${image_type} image already exists:"
         echo "$meta_img"
         exit 0
     fi
@@ -144,12 +132,12 @@ import_ostree_commit_for_build "${build}"
 image_json=${workdir}/tmp/image.json
 
 image_format=raw
-if [[ $image_type == qemu ]]; then
+if [[ "${image_type}" == "qemu" || "${image_type}" == "qemu-secex" ]]; then
     image_format=qcow2
 fi
 
-img=${name}-${build}-${image_type}${image_suffix}.${basearch}.${image_format}
-path=${PWD}/${img}
+imgname=${name}-${build}-${image_type}.${basearch}.${image_format}
+imgpath=${PWD}/${imgname}
 
 # We do some extra handling of the rootfs here; it feeds into size estimation.
 rootfs_type=$(jq -re .rootfs < "${image_json}")
@@ -174,24 +162,14 @@ if [ "${rootfs_type}" = "ext4verity" ]; then
     BLKSIZE="$(getconf PAGE_SIZE)"
 fi
 
-disk_args=()
-qemu_args=()
-# SecureExecution extra stuff
-if [[ $secure_execution -eq "1" ]]; then
-    disk_args+=("--with-secure-execution")
-    if [ ! -f "${genprotimgvm}" ]; then
-        fatal "No genprotimgvm provided at ${genprotimgvm}"
-    fi
-fi
-
 echo "Estimating disk size..."
 # The additional 35% here is obviously a hack, but we can't easily completely fill the filesystem,
 # and doing so has apparently negative performance implications.
 /usr/lib/coreos-assembler/estimate-commit-disk-size ${BLKSIZE:+--blksize ${BLKSIZE}} --repo "$ostree_repo" "$commit" --add-percent 35 > "$PWD/tmp/ostree-size.json"
 rootfs_size_mb="$(jq '."estimate-mb".final' "$PWD/tmp/ostree-size.json")"
 # The minimum size of a disk image we'll need will be the rootfs_size
 # estimate plus the size of the non-root partitions. We'll use this
-# size for the metal/dasd images, but for the IaaS/virt image we'll use
+# size for the metal images, but for the IaaS/virt image we'll use
 # the size set in the configs since some of them have minimum sizes that
 # the platforms require and we want a "default" disk size that has some
 # free space.
@@ -208,31 +186,14 @@ metal_image_size_mb="$(( rootfs_size_mb + nonroot_partition_sizes ))"
 cloud_image_size_mb="$(jq -r ".size*1024" < "${image_json}")"
 echo "Disk sizes: metal: ${metal_image_size_mb}M (estimated), cloud: ${cloud_image_size_mb}M"
 
-if [ "${image_type}" == metal4k ]; then
-    disk_args+=("--no-x86-bios-bootloader")
-fi
-
 set -x
 extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")"
 
-qemu-img create -f ${image_format} "${path}.tmp" "${metal_image_size_mb}M"
-
-extra_target_device_opts=""
-# we need 4096 block size for ECKD DASD and (obviously) metal4k
-if [[ $image_type == dasd || $image_type == metal4k ]]; then
-  extra_target_device_opts=",physical_block_size=4096,logical_block_size=4096"
-fi
-qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \
-              "-device" "virtio-blk,serial=target,drive=target${extra_target_device_opts}")
-
 # Generate the JSON describing the disk we want to build
 image_dynamic_yaml="${tmp_builddir}/image-dynamic.yaml"
 image_dynamic_json="${tmp_builddir}/image-dynamic.json"
 image_for_disk_json="${tmp_builddir}/image-for-disk.json"
 cat >"${image_dynamic_yaml}" << EOF
-buildid: "${build}"
-imgid: "${img}"
-ostree-commit: "${commit}"
 container-imgref: "${container_imgref}"
 deploy-via-container: "${deploy_via_container}"
 osname: "${name}"
@@ -252,11 +213,6 @@ cat "${image_json}" "${image_dynamic_json}" | jq -s add > "${image_for_disk_json
 platforms_json="${tmp_builddir}/platforms.json"
 yaml2json "${configdir}/platforms.yaml" "${platforms_json}"
 
-osbuild_extra_args=()
-if [[ $secure_execution -eq "1" ]]; then
-    osbuild_extra_args+=("--secex" "1")
-fi
-
 # In the jenkins pipelines we build the qemu image first and that operation
 # will do a lot of the same work required for later artifacts (metal, metal4k, etc)
 # so we want the cached output from that run to persist. The later artifacts get
@@ -267,9 +223,19 @@ fi
 runvm_with_cache_snapshot "$snapshot" -- /usr/lib/coreos-assembler/runvm-osbuild                    \
             --config "${image_for_disk_json}"                                                       \
             --mpp "/usr/lib/coreos-assembler/osbuild-manifests/coreos.osbuild.${basearch}.mpp.yaml" \
-            --filepath "${path}.tmp" "${osbuild_extra_args[@]}"
+            --filepath "${imgpath}"
+
+if [[ "${image_type}" == "qemu-secex" ]]; then
+    if [ ! -f "${genprotimgvm}" ]; then
+        fatal "No genprotimgvm provided at ${genprotimgvm}"
+    fi
+
+    # Basic qemu args:
+    qemu_args=(); blk_size="512"
+    [[ $image_type == metal4k ]] && blk_size="4096"
+    qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${imgpath},cache=unsafe" \
+        "-device" "virtio-blk,serial=target,drive=target,physical_block_size=${blk_size},logical_block_size=${blk_size}")
 
-if [[ $secure_execution -eq "1" ]]; then
     # SecureVM (holding Universal Key for all IBM Z Mainframes) requires scripts to execute genprotimg
     se_script_dir="/usr/lib/coreos-assembler/secex-genprotimgvm-scripts"
     genprotimg_img="${PWD}/secex-genprotimg.img"
@@ -301,16 +267,14 @@ if [[ $secure_execution -eq "1" ]]; then
     exec 9>&-
 fi
 
-/usr/lib/coreos-assembler/finalize-artifact "${path}.tmp" "${path}"
-
-sha256=$(sha256sum_str < "${img}")
+sha256=$(sha256sum_str < "${imgpath}")
 cosa meta --workdir "${workdir}" --build "${build}" --dump | python3 -c "
 import sys, json
 j = json.load(sys.stdin)
-j['images']['${image_type}${image_suffix}'] = {
-    'path': '${img}',
+j['images']['${image_type}'] = {
+    'path': '${imgname}',
     'sha256': '${sha256}',
-    'size': $(stat -c '%s' "${img}")
+    'size': $(stat -c '%s' "${imgpath}")
 }
 json.dump(j, sys.stdout, indent=4)
 " | jq -s add > "meta.json.new"
@@ -335,10 +299,10 @@ fi
 
 # and now the crucial bits
 cosa meta --workdir "${workdir}" --build "${build}" --artifact "${image_type}" --artifact-json "$(readlink -f meta.json.new)"
-/usr/lib/coreos-assembler/finalize-artifact "${img}" "${builddir}/${img}"
+/usr/lib/coreos-assembler/finalize-artifact "${imgpath}" "${builddir}/${imgname}"
 
 # Quiet for the rest of this so the last thing we see is a success message
 set +x
 # clean up the tmpbuild
 rm -rf "${tmp_builddir}"
-echo "Successfully generated: ${img}"
+echo "Successfully generated: ${imgname}"

src/cmd-buildextend-qemu-secex

@@ -0,0 +1 @@
+./cmd-buildextend-metal