Generate images for 512B and 4096B sector sizes

.github/workflows/debos.yml

@@ -62,43 +62,46 @@ jobs:
       # operating system from the github runner one; install debos
       - name: Setup build environment
         run: |
-           set -x
-           # privileged container as debos will use mounts
-           sudo incus init "${INCUS_IMAGE}" "${INCUS_NAME}" \
-               -c security.privileged=true -c security.nesting=true
-           sudo incus start "${INCUS_NAME}"
-           # wait for network to be up (prior to running apt)
-           sudo incus exec "${INCUS_NAME}" \
+          set -x
+          # privileged container as debos will use mounts
+          sudo incus init "${INCUS_IMAGE}" "${INCUS_NAME}" \
+              -c security.privileged=true -c security.nesting=true
+          sudo incus start "${INCUS_NAME}"
+          # wait for network to be up (prior to running apt)
+          sudo incus exec "${INCUS_NAME}" \
               /usr/lib/systemd/systemd-networkd-wait-online
-           (
-               # these commands are run inside the container
-               cat <<EOF
-               apt update
-               apt -y upgrade
-               apt -y full-upgrade
-               apt -y install debos
-           EOF
-           ) | sudo incus exec "${INCUS_NAME}" -- sh
+
+          # commands are piped to be run inside the container
+          sudo incus exec "${INCUS_NAME}" -- sh -e -x <<EOF
+          apt update
+          apt -y upgrade
+          apt -y full-upgrade
+          apt -y install debos
+          EOF
 
       - name: Build debos recipe
         run: |
           set -x
           # mount current directory under /build
           sudo incus config device add "${INCUS_NAME}" build-dir \
               disk "source=${PWD}" path=/build shift=true
-          (
-              # these commands are run inside the container
-              cat <<EOF
-              cd /build
-              # debos tries KVM and UML as backends, and falls back to building
-              # directly on the host, but that requires loop devices; use
-              # qemu backend explicitly even if it's slower
-              # qemu backend also requires to set scratchsize, otherwise
-              # the whole build is done from memory and the out of memory
-              # killer gets triggered
-              debos -b qemu --scratchsize 4GiB qualcomm-linux-debian.yaml
+
+          # commands are piped to be run inside the container
+          sudo incus exec "${INCUS_NAME}" -- sh -e -x <<EOF
+          cd /build
+          # start by building the root filesystem
+          debos qualcomm-linux-debian-rootfs.yaml
+          # debos tries KVM and UML as backends, and falls back to
+          # building directly on the host, but that requires loop
+          # devices; use qemu backend explicitly even if it's slower
+          # qemu backend also requires to set scratchsize, otherwise the
+          # whole build is done from memory and the out of memory killer
+          # gets triggered
+          debos -b qemu --scratchsize 4GiB -t imagetype:ufs \
+              qualcomm-linux-debian-image.yaml
+          debos -b qemu --scratchsize 4GiB -t imagetype:sdcard \
+              qualcomm-linux-debian-image.yaml
           EOF
-          ) | sudo incus exec "${INCUS_NAME}" -- sh
 
       - name: Upload artifacts to fileserver
         run: |
@@ -113,7 +116,8 @@ jobs:
           dir="${FILESERVER_DIR}/${id}"
           mkdir -vp "${dir}"
           # copy output files
-          cp -v disk.img "${dir}"
+          cp -v disk-ufs.img.gz "${dir}"
+          cp -v disk-sdcard.img.gz "${dir}"
           # instruct fileserver to publish this directory
           url="${FILESERVER_URL}/${id}/"
           curl -X POST -H 'Accept: text/event-stream' "${url}"

README.md

@@ -13,15 +13,19 @@ main: Primary development branch. Contributors should develop submissions based
 ## Requirements
 
 [debos](https://github.com/go-debos/debos) is required to build the debos recipes. Recent debos packages should be available in Debian and Ubuntu repositories; there are 
-[debos installation instructions](https://github.com/go-debos/debos?tab=readme-ov-file#installation-from-source-under-debian) on the project's page, notably for Docker images and to build debos from source.
+[debos installation instructions](https://github.com/go-debos/debos?tab=readme-ov-file#installation-from-source-under-debian) on the project's page, notably for Docker images and to build debos from source. Make sure to use at least version 1.1.5 which supports setting the sector size.
 
 [qdl](https://github.com/linux-msm/qdl) is typically used for flashing. While recent versions are available in Debian and Ubuntu, make sure you have at least version 2.1 as it contains important fixes.
 
 ## Usage
 
 To build a disk image, run debos as follows:
 ```bash
-debos debos-recipes/qualcomm-linux-debian.yaml
+# build a root filesystem tarball
+debos debos-recipes/qualcomm-linux-debian-rootfs.yaml
+
+# build a disk image from the root filesystem
+debos debos-recipes/qualcomm-linux-debian-image.yaml
 ```
 
 ### Build backends
@@ -30,21 +34,25 @@ By default, debos will try to pick a fast build backend; it will try to use its
 
 To build large images, the debos resource defaults might not be sufficient. Consider raising the default debos memory and scratchsize settings. This should provide a good set of minimum defaults:
 ```bash
-debos --fakemachine-backend qemu --memory 1GiB --scratchsize 4GiB debos-recipes/qualcomm-linux-debian.yaml
+debos --fakemachine-backend qemu --memory 1GiB --scratchsize 4GiB debos-recipes/qualcomm-linux-debian-image.yaml
 ```
 
 ### Build options
 
-A few options are provided in the debos recipe:
-- image: set the output disk image filename; default: `disk.img`
-- imagesize: set the output disk image size; default: `4GiB`
-- dtb: override the firmware provided device tree with one from the linux kernel, e.g. `qcom/qcs6490-rb3gen2.dtb`; default: don't override
-- xfcedesktop: install a Xfce desktop environment; default: console only environment
+A few options are provided in the debos recipes; for the root filesystem recipe:
 - experimentalkernel: update the linux kernel to the version from experimental; default: don't update the kernel
+- xfcedesktop: install a Xfce desktop environment; default: console only environment
+
+For the image recipe:
+- dtb: override the firmware provided device tree with one from the linux kernel, e.g. `qcom/qcs6490-rb3gen2.dtb`; default: don't override
+- imagetype: either `ufs` (the default) or (`sdcard`); UFS uses 4096 bytes sectors and SD card 512 bytes
+- image: set the output disk image filename; default is to suffix it with the imagetype: `disk-ufs.img`
+- imagesize: set the output disk image size; default: `4GiB`
 
 These can be passed as follows:
 ```bash
-debos -t dtb:qcom/qcs6490-rb3gen2.dtb -t xfcedesktop:true -t experimentalkernel:true debos-recipes/qualcomm-linux-debian.yaml
+debos -t xfcedesktop:true -t experimentalkernel:true debos-recipes/qualcomm-linux-debian-rootfs.yaml
+debos -t dtb:qcom/qcs6490-rb3gen2.dtb debos-recipes/qualcomm-linux-debian-image.yaml
 ```
 
 ## Flashing Instructions
@@ -54,16 +62,12 @@ Once a disk image is created, it is suitable for putting on an SD card, albeit m
 
 These images don't currently ambition to provide early boot assets such as boot firmware or data for other partitions containing board specific configuration or coprocessor firmware. Instead, start by provisioning an image with these early boot assets, such as the Yocto-based Qualcomm Linux images, and then flashing a debos generated image on top. Standalone, ready to flash (but probably not Debian based) images of the boot assets are planned to be made available publicly – stay tuned!
 
-Depending on the target board and target boot media, it's also necessary to use the right sector size for the image: typically 512B vs 4096B. SD cards and eMMC typically use the historical 512B sector size, while UFS storage uses 4096B sector size. debos has just gained support for configurable sector sizes, but that requires building it from source; alernatively, you can post-process the image with a conversation script as explained below.
+Depending on the target board and target boot media, it's also necessary to use the right sector size for the image: typically 512B vs 4096B. SD cards and eMMC typically use the historical 512B sector size, while UFS storage uses 4096B sector size. debos has gained support for configurable sector sizes in version 1.1.5.
 
 ### RB3 Gen2 instructions
 
 The RB3 Gen2 board boots from UFS by default. To flash a disk image to the UFS storage of the RB3 Gen2 board:
 1. provision some known good early boot assets by flashing the Yocto edition of [Qualcomm Linux](https://www.qualcomm.com/developer/software/qualcomm-linux)
-1. unless you've got a recent debos that supports creating images with a 4096B sector size, convert the debos disk image from 512B to 4096B sector sizes; this sample script can be used as a workaround until [debos gains support for setting the sector size](https://github.com/go-debos/debos/issues/537) but it's a britle approach which requires root, the workaround script is also full of hardcoded expectations and might need local tweaks:
-    ```bash
-    sudo scripts/workaround-convert-sector-size disk.img disk-4096.img 4096
-    ```
 1. create a `rawprogram-ufs.xml` file instructing QDL to flash to the first UFS LUN (LUN0):
     ```xml
     <?xml version="1.0" ?>

debos-recipes/qualcomm-linux-debian-image.yaml

@@ -0,0 +1,101 @@
+{{- $dtb := or .dtb "firmware" }}
+{{- $imagesize := or .imagesize "4GiB" }}
+{{- $imagetype := or .imagetype "ufs" }}
+{{- $image := or .image (printf "disk-%s.img" $imagetype) }}
+
+architecture: arm64
+sectorsize: {{if eq $imagetype "ufs"}} 4096 {{else}} 512 {{end}}
+
+actions:
+  - action: unpack
+    description: Unpack root filesystem
+    compression: gz
+    file: rootfs.tar.gz
+
+{{- if ne $dtb "firmware" }}
+  - action: run
+    description: Use OS device tree override (board specific)
+    chroot: true
+    command: |
+      set -eu
+      mkdir -p /etc/kernel
+      echo {{ $dtb }} >/etc/kernel/devicetree
+{{- end }}
+
+  # parttype values are from:
+  # https://www.freedesktop.org/wiki/Specifications/DiscoverablePartitionsSpec/
+  - action: image-partition
+    description: Create an image file
+    imagename: {{ $image }}
+    imagesize: {{ $imagesize }}
+    partitiontype: gpt
+    partitions:
+      - name: esp
+        partlabel: ESP
+        flags: [ boot, esp ]
+        # EFI System Partition
+        parttype: c12a7328-f81f-11d2-ba4b-00a0c93ec93b
+        fs: vfat
+        start: 4MiB
+        # size of 512MiB
+        end: 516MiB
+        fsck: true
+      - name: root
+        # Root Partition (64-bit ARM/AArch64)
+        parttype: b921b045-1df0-41c3-af44-4c6f280d3fae
+        fs: ext4
+        # 4MiB after ESP
+        start: 520MiB
+        end: 100%
+        fsck: true
+    mountpoints:
+      - mountpoint: /boot/efi/
+        partition: esp
+      - mountpoint: /
+        partition: root
+
+  # XXX these kernel options might be specific to a kernel version or board
+  - action: filesystem-deploy
+    setup-fstab: true
+    append-kernel-cmdline: clk_ignore_unused pd_ignore_unused audit=0 deferred_probe_timeout=30
+
+  - action: apt
+    recommends: true
+    description: Make system bootable with systemd-boot
+    packages:
+      - systemd-boot
+      # TODO investigate why systemd-boot Recommends: shim-signed which
+      # Depends on grub packages
+      - shim-signed-
+
+  - action: run
+    description: Create task to grow root filesystem on first boot
+    chroot: true
+    command: |
+      set -eu
+      # for growpart
+      apt -y install cloud-guest-utils
+      # extract the dev of the root filesystem from fstab; change UUID=id into
+      # /dev/disk/by-uuid/id
+      ROOTFS_DEV=$(
+          awk '$2 == "/" {gsub("UUID=", "/dev/disk/by-uuid/"); print $1}' \
+              /etc/fstab
+          )
+      # generate systemd unit to grow the second partition with above UUID
+      cat >/etc/systemd/system/debos-grow-rootfs.service <<EOF
+      [Unit]
+      Description=Grow root partition and filesystem (debos)
+      [Service]
+      Type=oneshot
+      ExecStartPre=-sh -c 'growpart /dev/\`lsblk -n -o pkname $ROOTFS_DEV\` 2'
+      ExecStart=resize2fs $ROOTFS_DEV
+      [Install]
+      WantedBy=default.target
+      EOF
+      # enable unit
+      systemctl enable debos-grow-rootfs
+
+  - action: run
+    postprocess: true
+    command: gzip -f "${ARTIFACTDIR}/{{ $image }}"
+

debos-recipes/qualcomm-linux-debian.yaml → debos-recipes/qualcomm-linux-debian-rootfs.yaml

@@ -1,6 +1,3 @@
-{{- $image := or .image "disk.img" }}
-{{- $imagesize := or .imagesize "4GiB" }}
-{{- $dtb := or .dtb "firmware" }}
 {{- $xfcedesktop := or .xfcedesktop "false" }}
 {{- $experimentalkernel := or .experimentalkernel "false" }}
 
@@ -154,16 +151,6 @@ actions:
           update-locale LANG=en_US.UTF-8
       fi
 
-{{- if ne $dtb "firmware" }}
-  - action: run
-    description: Use OS device tree override (board specific)
-    chroot: true
-    command: |
-      set -eu
-      mkdir -p /etc/kernel
-      echo {{ $dtb }} >/etc/kernel/devicetree
-{{- end }}
-
   - action: apt
     description: Install kernel and firmware packages
     recommends: true
@@ -197,76 +184,7 @@ actions:
           /etc/apt/sources.list.d/debian-experimental.sources
 {{- end }}
 
-  # parttype values are from:
-  # https://www.freedesktop.org/wiki/Specifications/DiscoverablePartitionsSpec/
-  - action: image-partition
-    description: Create an image file
-    imagename: {{ $image }}
-    imagesize: {{ $imagesize }}
-    partitiontype: gpt
-    partitions:
-      - name: esp
-        partlabel: ESP
-        flags: [ boot, esp ]
-        # EFI System Partition
-        parttype: c12a7328-f81f-11d2-ba4b-00a0c93ec93b
-        fs: vfat
-        start: 4MiB
-        # size of 512MiB
-        end: 516MiB
-        fsck: true
-      - name: root
-        # Root Partition (64-bit ARM/AArch64)
-        parttype: b921b045-1df0-41c3-af44-4c6f280d3fae
-        fs: ext4
-        # 4MiB after ESP
-        start: 520MiB
-        end: 100%
-        fsck: true
-    mountpoints:
-      - mountpoint: /boot/efi/
-        partition: esp
-      - mountpoint: /
-        partition: root
-
-  # XXX these kernel options might be specific to a kernel version or board
-  - action: filesystem-deploy
-    setup-fstab: true
-    append-kernel-cmdline: clk_ignore_unused pd_ignore_unused audit=0 deferred_probe_timeout=30
-
-  - action: apt
-    recommends: true
-    description: Make system bootable with systemd-boot
-    packages:
-      - systemd-boot
-      # TODO investigate why systemd-boot Recommends: shim-signed which
-      # Depends on grub packages
-      - shim-signed-
-
-  - action: run
-    description: Create task to grow root filesystem on first boot
-    chroot: true
-    command: |
-      set -eu
-      # for growpart
-      apt -y install cloud-guest-utils
-      # extract the dev of the root filesystem from fstab; change UUID=id into
-      # /dev/disk/by-uuid/id
-      ROOTFS_DEV=$(
-          awk '$2 == "/" {gsub("UUID=", "/dev/disk/by-uuid/"); print $1}' \
-              /etc/fstab
-          )
-      # generate systemd unit to grow the second partition with above UUID
-      cat >/etc/systemd/system/debos-grow-rootfs.service <<EOF
-      [Unit]
-      Description=Grow root partition and filesystem (debos)
-      [Service]
-      Type=oneshot
-      ExecStartPre=-sh -c 'growpart /dev/\`lsblk -n -o pkname $ROOTFS_DEV\` 2'
-      ExecStart=resize2fs $ROOTFS_DEV
-      [Install]
-      WantedBy=default.target
-      EOF
-      # enable unit
-      systemctl enable debos-grow-rootfs
+  - action: pack
+    file: rootfs.tar.gz
+    compression: gz