Merge branch 'main' into satish-debug

Signed-off-by: Satish Mhaske <[email protected]>

Author: sampra2025

.github/workflows/debos.yml

@@ -24,9 +24,7 @@ defaults:
     working-directory: debos-recipes
 
 env:
-  BUILD_ID: ${{ github.repository }}-${{ github.run_id }}-${{ github.run_attempt }}
-  FILESERVER_DIR: /efs/qli/metaqcom/gh-runners/quic-yocto/builds
-  FILESERVER_URL: "https://artifactory-qdc-global.qualcomm.com/artifactory/pdlm-qcomlinux-generic-lv-fed/QualcommLinuxPrd/qualcomm-linux/meta-qcom/${BUILD_ID}"
+  FILESERVER_URL: https://quic-yocto-fileserver-1029608027416.us-central1.run.app
 
 # cancel in progress builds for this workflow triggered by the same ref
 concurrency:
@@ -39,7 +37,8 @@ jobs:
     container:
       image: debian:trixie
       volumes:
-        - /efs/qli/metaqcom/gh-runners/quic-yocto/builds:/efs/qli/metaqcom/gh-runners/quic-yocto/builds
+        - /srv/gh-runners/quic-yocto/builds:/fileserver-builds
+      options: --privileged
     steps:
       - uses: actions/checkout@v4
         with:
@@ -49,35 +48,50 @@ jobs:
       # avoid an automated update while we're building
       - name: Update OS packages
         run: |
-          set -ex
+          set -x
           apt update
           apt -y upgrade
           apt -y full-upgrade
-          apt -y install debos
+          # debos is needed to build recipes, mtools is needed for the flash
+          # recipe
+          apt -y install debos mtools
 
-      # create a fresh container build environment to decouple the build
-      # operating system from the github runner one; install debos
-      - name: Setup build environment
+    run: |
+          set -x
+          # start by building the root filesystem
+          debos qualcomm-linux-debian-rootfs.yaml
+          # debos tries KVM and UML as backends, and falls back to
+          # building directly on the host, but that requires loop
+          # devices; use qemu backend explicitly even if it's slower
+          # qemu backend also requires to set scratchsize, otherwise the
+          # whole build is done from memory and the out of memory killer
+          # gets triggered
+          debos -b qemu --scratchsize 4GiB -t imagetype:ufs \
+              qualcomm-linux-debian-image.yaml
+          debos -b qemu --scratchsize 4GiB -t imagetype:sdcard \
+              qualcomm-linux-debian-image.yaml
+          # build flashable files
+          debos qualcomm-linux-debian-flash.yaml
 
+  - name: Stage image for publishing
         run: |
-          set -ex
-          # debos tries KVM and UML as backends, and falls back to building
-          # directly on the host, but that requires loop devices; use
-          # qemu backend explicitly even if it's slower
-          # qemu backend also requires to set scratchsize, otherwise
-          # the whole build is done from memory and the out of memory
-          # killer gets triggered
-          debos -b qemu --scratchsize 4GiB qualcomm-linux-debian.yaml
-
-      - name: Stage image for publishing
-        run: |
-          set -ex
+          set -x
+          # curl will be used to talk to fileserver; should be installed by
+          # default
+          apt -y install curl
           # github runs are only unique per repository and may also be re-run;
           # create an unique id with repository, run id, and run attempt
           id="${GITHUB_REPOSITORY}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
           # create a directory for the current run
-          dir="${FILESERVER_DIR}/${id}"
+          dir="/fileserver-builds/${id}"
           mkdir -vp "${dir}"
-          # Files placed under ${dir} will be published internally
-          cp -v disk.img "${dir}"
+          # copy output files
+          cp -av rootfs.tar.gz "${dir}"
+          cp -av dtbs.tar.gz "${dir}"
+          cp -av disk-ufs.img.gz "${dir}"
+          cp -av disk-sdcard.img.gz "${dir}"
+          tar -cvf "${dir}"/flash.tar.gz disk-ufs.img1 disk-ufs.img2 flash_*
+          # instruct fileserver to publish this directory
+          url="${FILESERVER_URL}/${id}/"
+          curl -X POST -H 'Accept: text/event-stream' "${url}"
 

README.md

@@ -13,71 +13,97 @@ main: Primary development branch. Contributors should develop submissions based
 ## Requirements
 
 [debos](https://github.com/go-debos/debos) is required to build the debos recipes. Recent debos packages should be available in Debian and Ubuntu repositories; there are 
-[debos installation instructions](https://github.com/go-debos/debos?tab=readme-ov-file#installation-from-source-under-debian) on the project's page, notably for Docker images and to build debos from source.
+[debos installation instructions](https://github.com/go-debos/debos?tab=readme-ov-file#installation-from-source-under-debian) on the project's page, notably for Docker images and to build debos from source. Make sure to use at least version 1.1.5 which supports setting the sector size.
 
-[qdl](https://github.com/linux-msm/qdl) is typically used for flashing. While recent versions are available in Debian and Ubuntu, make sure you have at least version 2.1 as it contains important fixes.
+[qdl](https://github.com/linux-msm/qdl) is typically used for flashing. While recent versions are available in Debian and Ubuntu, make sure to use at least version 2.1 as it contains important fixes.
 
 ## Usage
 
-To build a disk image, run debos as follows:
+To build flashable assets, run debos as follows:
 ```bash
-debos debos-recipes/qualcomm-linux-debian.yaml
+# build tarballs of the root filesystem and DTBs
+debos debos-recipes/qualcomm-linux-debian-rootfs.yaml
+
+# build disk and filesystem images from the root filesystem; the default is to
+# build an UFS image
+debos debos-recipes/qualcomm-linux-debian-image.yaml
+
+# build flashable assets from the DTBs and UFS filesystem images; currently these
+# are only built for the RB3 Gen2 Vision Kit board
+debos debos-recipes/qualcomm-linux-debian-flash.yaml
 ```
 
-### Build backends
+### Debos tips
 
-By default, debos will try to pick a fast build backend; it will try to use its KVM backend ("-b kvm") when available, and otherwise an UML environment ("-b uml"). If none of these work, a solid backend is QEMU ("-b qemu"). Because the target images are arm64, this can be really slow, especially when building from another architecture such as amd64.
+By default, debos will try to pick a fast build backend; it will try to use its KVM backend ("-b kvm") when available, and otherwise an UML environment ("-b uml"). If none of these work, a solid backend is QEMU ("-b qemu"); because the target images are arm64, this can be really slow when building from another architecture such as amd64.
 
 To build large images, the debos resource defaults might not be sufficient. Consider raising the default debos memory and scratchsize settings. This should provide a good set of minimum defaults:
 ```bash
-debos --fakemachine-backend qemu --memory 1GiB --scratchsize 4GiB debos-recipes/qualcomm-linux-debian.yaml
+debos --fakemachine-backend qemu --memory 1GiB --scratchsize 4GiB debos-recipes/qualcomm-linux-debian-image.yaml
 ```
 
 ### Build options
 
-A few options are provided in the debos recipe:
-- image: set the output disk image filename; default: `disk.img`
-- imagesize: set the output disk image size; default: `4GiB`
-- dtb: override the firmware provided device tree with one from the linux kernel, e.g. `qcom/qcs6490-rb3gen2.dtb`; default: don't override
-- xfcedesktop: install a Xfce desktop environment; default: console only environment
+A few options are provided in the debos recipes; for the root filesystem recipe:
 - experimentalkernel: update the linux kernel to the version from experimental; default: don't update the kernel
+- xfcedesktop: install a Xfce desktop environment; default: console only environment
+
+For the image recipe:
+- dtb: override the firmware provided device tree with one from the linux kernel, e.g. `qcom/qcs6490-rb3gen2.dtb`; default: don't override
+- imagetype: either `ufs` (the default) or (`sdcard`); UFS images are named disk-ufs.img and use 4096 bytes sectors and SD card images are named disk-sdcard.img and use 512 bytes sectors
+- imagesize: set the output disk image size; default: `4GiB`
 
-These can be passed as follows:
+Here are some example invocations:
 ```bash
-debos -t dtb:qcom/qcs6490-rb3gen2.dtb -t xfcedesktop:true -t experimentalkernel:true debos-recipes/qualcomm-linux-debian.yaml
+# build the root filesystem with Xfce and a kernel from experimental
+debos -t xfcedesktop:true -t experimentalkernel:true debos-recipes/qualcomm-linux-debian-rootfs.yaml
+
+# build an image where systemd overrides the firmware device tree with the one
+# for RB3 Gen2
+debos -t dtb:qcom/qcs6490-rb3gen2.dtb debos-recipes/qualcomm-linux-debian-image.yaml
+
+# build an SD card image
+debos -t imagetype:sdcard debos-recipes/qualcomm-linux-debian-image.yaml
 ```
 
 ## Flashing Instructions
 ### Overview
 
-Once a disk image is created, it is suitable for putting on an SD card, albeit most Qualcomm boards boot from internal storage by default. The disk image can also be flashed on the internal storage of your board with [qdl](https://github.com/linux-msm/qdl).
+The `disk-sdcard.img` disk image can simply be written to a SD card, albeit most Qualcomm boards boot from internal storage by default. With an SD card, the board will use boot firmware from internal storage (eMMC or UFS) and do an EFI boot from the SD card if the firmware can't boot from internal storage.
 
-These images don't currently ambition to provide early boot assets such as boot firmware or data for other partitions containing board specific configuration or coprocessor firmware. Instead, start by provisioning an image with these early boot assets, such as the Yocto-based Qualcomm Linux images, and then flashing a debos generated image on top. Standalone, ready to flash (but probably not Debian based) images of the boot assets are planned to be made available publicly – stay tuned!
+If there is no need to update the boot firmware, the `disk-ufs.img` disk image can also be flashed on the first LUN of the internal UFS storage with [qdl](https://github.com/linux-msm/qdl). Create a `rawprogram-ufs.xml` file as follows:
+```xml
+<?xml version="1.0" ?>
+<data>
+  <program SECTOR_SIZE_IN_BYTES="4096" file_sector_offset="0" filename="disk-ufs.img" label="image" num_partition_sectors="0" partofsingleimage="false" physical_partition_number="0" start_sector="0"/>
+</data>
+```
+Put the board in "emergency download mode" (EDL; see next section) and run:
+```bash
+qdl --storage ufs prog_firehose_ddr.elf rawprogram-ufs.xml
+```
+Make sure to use `prog_firehose_ddr.elf` for the target platform, such as this [version from the QCM6490 boot binaries](https://softwarecenter.qualcomm.com/download/software/chip/qualcomm_linux-spf-1-0/qualcomm-linux-spf-1-0_test_device_public/r1.0_00058.0/qcm6490-le-1-0/common/build/ufs/bin/QCM6490_bootbinaries.zip).
 
-Depending on the target board and target boot media, it's also necessary to use the right sector size for the image: typically 512B vs 4096B. SD cards and eMMC typically use the historical 512B sector size, while UFS storage uses 4096B sector size. debos has just gained support for configurable sector sizes, but that requires building it from source; alernatively, you can post-process the image with a conversation script as explained below.
+To flash a complete set of assets on UFS internal storage, put the board in EDL mode and run:
+```bash
+# use the RB3 Gen2 Vision Kit flashable assets
+cd flash_rb3gen2-vision-kit
+qdl --storage ufs prog_firehose_ddr.elf rawprogram[0-9].xml patch[0-9].xml
+```
 
-### RB3 Gen2 instructions
+### Emergency Download Mode (EDL)
 
-The RB3 Gen2 board boots from UFS by default. To flash a disk image to the UFS storage of the RB3 Gen2 board:
-1. provision some known good early boot assets by flashing the Yocto edition of [Qualcomm Linux](https://www.qualcomm.com/developer/software/qualcomm-linux)
-1. unless you've got a recent debos that supports creating images with a 4096B sector size, convert the debos disk image from 512B to 4096B sector sizes; this sample script can be used as a workaround until [debos gains support for setting the sector size](https://github.com/go-debos/debos/issues/537) but it's a britle approach which requires root, the workaround script is also full of hardcoded expectations and might need local tweaks:
-    ```bash
-    sudo scripts/workaround-convert-sector-size disk.img disk-4096.img 4096
-    ```
-1. create a `rawprogram-ufs.xml` file instructing QDL to flash to the first UFS LUN (LUN0):
-    ```xml
-    <?xml version="1.0" ?>
-    <data>
-      <program SECTOR_SIZE_IN_BYTES="4096" file_sector_offset="0" filename="disk-4096.img" label="image" num_partition_sectors="0" partofsingleimage="false" physical_partition_number="0" start_sector="0"/>
-    </data>
-    ```
-1. put the board in "emergency download" mode (EDL) by removing any cable from the USB type-C port, and pressing the `F_DL` button while turning the power on
+In EDL mode, the board will receive a flashing program over its USB type-C cable, and that program will receive data to flash on the internal storage. This is a lower level mode than fastboot which is implemented by a higher-level bootloader.
+
+To enter EDL mode:
+1. remove power to the board
+1. remove any cable from the USB type-C port
+1. on some boards, it's necessary to set some DIP switches
+1. press the `F_DL` button while turning the power on
 1. connect a cable from the flashing host to the USB type-C port on the board
-1. run qdl to flash the image:
-    ```bash
-    qdl prog_firehose_ddr.elf rawprogram-ufs.xml
-    ```
-The `prog_firehose_ddr.elf` payload is part of the the Yocto Qualcomm Linux image download.
+1. run qdl to flash the board
+
+NB: It's also possible to run qdl from the host while the baord is not connected, and starting the board directly in EDL mode.
 
 ## Development
 

debos-recipes/qualcomm-linux-debian-flash.yaml

@@ -0,0 +1,125 @@
+architecture: arm64
+
+actions:
+  - action: download
+    description: Download qcom-ptool
+    url: https://github.com/qualcomm-linux/qcom-ptool/archive/refs/heads/main.tar.gz
+    name: qcom-ptool
+    filename: qcom-ptool.tar.gz
+    unpack: true
+
+  - action: download
+    description: Download QCM6490 boot binaries
+    url: https://softwarecenter.qualcomm.com/download/software/chip/qualcomm_linux-spf-1-0/qualcomm-linux-spf-1-0_test_device_public/r1.0_00058.0/qcm6490-le-1-0/common/build/ufs/bin/QCM6490_bootbinaries.zip
+    name: qcm6490_boot-binaries
+    filename: qcm6490_boot-binaries.zip
+
+  - action: download
+    description: Download RB3 Gen2 Vision Kit CDT
+    url: https://artifacts.codelinaro.org/artifactory/codelinaro-le/Qualcomm_Linux/QCS6490/cdt/rb3gen2-vision-kit.zip
+    name: rb3gen2-vision-kit_cdt
+    filename: rb3gen2-vision-kit_cdt.zip
+
+  - action: run
+    description: Generate flash directories for UFS boards
+    chroot: false
+    command: |
+      set -eux
+      # work dir that will be thrown away
+      mkdir -v build
+
+      # path to unpacked qcom-ptool tarball
+      QCOM_PTOOL="${ROOTDIR}/../qcom-ptool.tar.gz.d/qcom-ptool-main"
+
+      ## platform: QCM6490
+      # unpack boot binaries
+      unzip -j "${ROOTDIR}/../qcm6490_boot-binaries.zip" \
+          -d build/qcm6490_boot-binaries
+      # generate partition files
+      mkdir -v build/qcm6490_partitions
+      (
+          cd build/qcm6490_partitions
+          conf="${QCOM_PTOOL}/platforms/qcm6490/partitions.conf"
+          "${QCOM_PTOOL}/gen_partition.py" -i "$conf" \
+              -o ptool-partitions.xml
+          # partitions.conf sets --type=emmc, nand or ufs
+          if grep -F '^--disk --type=ufs ' "${conf}"; then
+              touch flash-ufs
+          fi
+          "${QCOM_PTOOL}/ptool.py" -x ptool-partitions.xml
+      )
+
+      ### board: RB3Gen2 Vision Kit
+      flash_dir="${ARTIFACTDIR}/flash_rb3gen2-vision-kit"
+      rm -rf "${flash_dir}"
+      mkdir -v "${flash_dir}"
+      # copy platform partition files
+      cp --preserve=mode,timestamps -v build/qcm6490_partitions/* \
+          "${flash_dir}"
+      # copy platform boot binaries; these shouldn't ship partition files, but
+      # make sure not to accidentally clobber any such file
+      find build/qcm6490_boot-binaries \
+          -not -name 'gpt_*' \
+          -not -name 'patch*.xml' \
+          -not -name 'rawprogram*.xml' \
+          -not -name 'wipe*.xml' \
+          -not -name 'zeros_*' \
+          \( \
+              -name Qualcomm-Technologies-Inc.-Proprietary \
+              -or -name 'prog_*' \
+              -or -name '*.bin' \
+              -or -name '*.elf' \
+              -or -name '*.fv' \
+              -or -name '*.mbn' \
+          \) \
+          -exec cp --preserve=mode,timestamps -v '{}' "${flash_dir}" \;
+      # unpack board CDT
+      unzip -j "${ROOTDIR}/../rb3gen2-vision-kit_cdt.zip" \
+          -d build/rb3gen2-vision-kit_cdt
+      # copy just the CDT data; no partition or flashing files
+      cp --preserve=mode,timestamps -v build/rb3gen2-vision-kit_cdt/cdt_vision_kit.bin \
+          "${flash_dir}"
+
+      # update flashing files for CDT
+      sed -i '/label="cdt"/s/filename=""/filename="cdt_vision_kit.bin"/' \
+          "${flash_dir}"/rawprogram*.xml
+
+      # generate a dtb.bin FAT partition with just a single dtb for the current
+      # board; long-term this should really be a set of dtbs and overlays as to
+      # share dtb.bin across boards
+      dtb_bin="${flash_dir}/dtb.bin"
+      rm -f "${dtb_bin}"
+      # dtb.bin is only used in UFS based boards at the moment and UFS uses a
+      # 4k sector size, so pass -S 4096
+      # in qcom-ptool/platforms/*/partitions.conf, dtb_a and _b partitions
+      # are provisioned with 64MiB; create a 4MiB FAT that will comfortably fit
+      # in these and hold the target device tree, which is 4096 KiB sized
+      # blocks for mkfs.vfat's last argument
+      mkfs.vfat -S 4096 -C "${dtb_bin}" 4096
+      # RB3 Gen2 Vision Kit will probably have a more specific DTB (see
+      # <[email protected]> on lore.kernel.org)
+      # but for now use the core kit one
+      dtb="qcom/qcs6490-rb3gen2.dtb"
+      # extract board device tree from the root filesystem provided tarball
+      tar -C build -xvf "${ARTIFACTDIR}/dtbs.tar.gz" "${dtb}"
+      # copy into the FAT as combined-dtb.dtb
+      mcopy -vmp -i "${dtb_bin}" "build/${dtb}" ::/combined-dtb.dtb
+
+      # (NB: flashing files already expect "dtb.bin" as a filename)
+
+      # update flashing files for ESP image;
+      # qcom-ptool/platforms/*/partitions.conf uses filename=efi.bin for the
+      # ESP partition on EFI capable platforms
+      sed -i '/label="efi"/s#filename="[^"]*"#filename="../disk-ufs.img1"#' \
+          "${flash_dir}"/rawprogram*.xml
+
+      # update flashing files for rootfs image;
+      # qcom-ptool/platforms/*/partitions.conf uses filename=rootfs.img for the
+      # rootfs partition
+      sed -i \
+          '/label="rootfs"/s#filename="[^"]*"#filename="../disk-ufs.img2"#' \
+          "${flash_dir}"/rawprogram*.xml
+
+      # cleanup
+      rm -rf build
+