Include SBOM generation into GCP images build pipeline (#1)

* Add initial sbom generation support

* Add sbom generation from collected data and upload artifact

* Update gcp 8 and 9 packer definitions

* Update paths around SBOM generation

* Run sbom_generator as sudo

Also, remove dump_repo_metadata role

* Restored commented actions

* Pull cloud-images-sbom-tools from AlmaLinux organization

Also, simplify installation of cloud-images-sbom-tools dependencies

---------

Signed-off-by: Jonathan Wright <[email protected]>
Co-authored-by: Jonathan Wright <[email protected]>

Author: javihernandez

.github/actions/shared-steps/action.yml

@@ -391,6 +391,25 @@ runs:
         # Install ansible
         sudo ${{ env.runner_os == 'ubuntu' && 'apt-get' || 'dnf -q' }} -y install ansible
 
+    - name: Clone SBOM tools
+      shell: bash
+      run: |
+        rm -rf sbom-tools
+        git clone --depth=1 https://github.com/AlmaLinux/cloud-images-sbom-tools.git sbom-tools
+
+    - name: Set up Python and install generator deps
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: sbom-tools/requirements.txt
+
+    - name: Create venv and install
+      shell: bash
+      run: |
+        python -m venv .venv-sbom
+        .venv-sbom/bin/pip install -r sbom-tools/requirements.txt
+
     - name: Initialize packer
       shell: bash
       run: sudo /usr/bin/packer init -upgrade .
@@ -406,7 +425,8 @@ runs:
     - name: Locate image file, generate checksum
       shell: bash
       run: |
-        # Locate image file, generate checksum
+        # Locate image file, generate checksum, rename repo metadata file
+        ls -la $(dirname '${{ env.output_mask }}')
         image_file=$(ls -1 ${{ env.output_mask }} | head -n 1)
         [ "x${image_file}" = "x" ] && false
         cd $(dirname ${image_file})
@@ -416,15 +436,15 @@ runs:
         echo "IMAGE_NAME=$(basename ${image_file})" >> $GITHUB_ENV
 
         # don't fail if this doesn't exist, we may not always generate it
-        sudo mv repo-metadata-*.txt $(basename ${image_file}).repo-metadata.txt || true
+        sudo mv sbom-data-*.json $(basename ${image_file}).sbom-data.json || true
 
     - id: 'google-auth-dev-images'
       if: env.IMAGE_TYPE == 'gcp'
       uses: 'google-github-actions/auth@v2'
       with:
         workload_identity_provider: 'projects/443728870479/locations/global/workloadIdentityPools/github-actions/providers/github'
         service_account: 'github-actions-cloud-images@almalinux-dev-images-469421.iam.gserviceaccount.com'
-    
+
     - name: 'Set up Google Cloud SDK'
       if: env.IMAGE_TYPE == 'gcp'
       uses: 'google-github-actions/[email protected]'
@@ -477,64 +497,23 @@ runs:
           -source_gcs_path="gs://almalinux-images-dev/" \
           vm-scripts/gcp/almalinux_${version_major}${{ inputs.arch == 'aarch64' && '_arm64' || '' }}.publish.json
 
-    - name: Test/check release and architecture, list installed packages in ${{ env.IMAGE_FILE }} cloud image
-      if: ${{ ! contains(inputs.type, 'vagrant') && ! contains(inputs.type, 'gcp') }}
+    # - name: 'Run Google cloud-image-testing tests (basic suite)'
+    #   if: env.IMAGE_TYPE == 'gcp' && inputs.run_test == 'true'
+    #   shell: bash
+    #   run: |
+    #     cd cloud-image-tests
+    #     ./bin/manager \
+    #       -local_path bin \
+    #       -project almalinux-image-testing-469421 \
+    #       -filter '^(cvm|livemigrate|suspendresume|loadbalancer|guestagent|hostnamevalidation|imageboot|licensevalidation|network|security|hotattach|lssd|disk|packagevalidation|ssh|metadata|vmspec)$' \
+    #       -images 'projects/almalinux-dev-images-469421/global/images/family/almalinux-${{ env.version_major }}${{ inputs.arch == 'aarch64' && '-arm64' || '' }}' \
+    #       -parallel_stagger 10s -parallel_count 20
+
+    - name: Generate SBOM
       shell: bash
       run: |
-        # List installed packages in ${{ env.IMAGE_FILE }} image
-
-        # Partition number with root file-system
-        case ${{ inputs.arch }} in
-          x86_64*) partition=4 ;;
-          aarch64*) partition=3 ;;
-          *) false ;;
-        esac
-
-        # Image file format: raw or qcow2
-        case ${{ inputs.type }} in
-          oci|gencloud|opennebula) format=qcow2 ;;
-          azure) format=raw ;;
-          *) false ;;
-        esac
-        rootfs_path=/mnt/rootfs
-        sudo mkdir -p ${rootfs_path}
-
-        # Install qemu-utils
-        sudo ${{ env.runner_os == 'ubuntu' && 'apt-get' || 'dnf -q' }} \
-          -y install \
-          ${{ env.runner_os == 'ubuntu' && 'qemu-utils' || 'qemu-img' }}
-
-        # Load nbd kernel module
-        sudo modprobe nbd max_part=8
-
-        # Make a copy of the image file
-        sudo cp ${{ env.IMAGE_FILE }} $(dirname ${rootfs_path})
-
-        # Attach the image file to the nbd device
-        sudo qemu-nbd \
-          --read-only \
-          --format=${format} \
-          --connect=/dev/nbd0 \
-          $(dirname ${rootfs_path})/$(basename ${{ env.IMAGE_FILE }}) \
-          && sleep 10 || false
-
-        # Mount need partition
-        sudo fdisk -l /dev/nbd0
-        sudo mount /dev/nbd0p${partition} ${rootfs_path} \
-          && sleep 10 || false
-
-        echo "[Debug] AlmaLinux release:"
-        grep '${{ env.RELEASE_STRING }}' ${rootfs_path}/etc/almalinux-release
-
-        echo "[Debug] System architecture:"
-        rpm --dbpath=${rootfs_path}/var/lib/rpm -q --qf='%{ARCH}\n' ${{ env.RELEASE_PACKAGE }} | grep '${{ env.alma_arch }}'
-
-        # Get installed packages list
-        sudo sh -c "rpm --dbpath=${rootfs_path}/var/lib/rpm -qa --queryformat '%{NAME}\n' | sort > ${{ env.IMAGE_FILE }}.txt"
-
-        [ -f ${{ env.IMAGE_FILE }}.txt ] \
-          && echo "got_pkgs_list=true" >> $GITHUB_ENV \
-          || echo "got_pkgs_list=false" >> $GITHUB_ENV
+        echo "Generating SBOM document of ${{ env.IMAGE_FILE }}"
+        sudo .venv-sbom/bin/python3 sbom-tools/sbom_generator.py "${{ env.IMAGE_NAME }}" "${{ env.IMAGE_FILE }}.sbom-data.json" "${{ env.IMAGE_FILE }}.sbom.spdx.json"
 
     - name: Test ${{ inputs.type }} ${{ inputs.variant }} image
       if: inputs.run_test == 'true' && contains(inputs.type, 'vagrant')
@@ -636,13 +615,22 @@ runs:
         path: ${{ env.IMAGE_FILE }}
 
     - uses: actions/upload-artifact@v4
-      name: Store repo metadata as artifact
-      id: repo-meta-artifact
+      name: Store collected sbom data as artifact
+      id: sbom-data-artifact
+      if: inputs.store_as_artifact == 'true'
+      with:
+        compression-level: 9
+        name: ${{ env.IMAGE_NAME }}.sbom-data.json
+        path: ${{ env.IMAGE_FILE }}.sbom-data.json
+
+    - uses: actions/upload-artifact@v4
+      name: Store SBOM as artifact
+      id: sbom-artifact
       if: inputs.store_as_artifact == 'true'
       with:
         compression-level: 9
-        name: ${{ env.IMAGE_NAME }}.repo-metadata.txt
-        path: ${{ env.IMAGE_FILE }}.repo-metadata.txt
+        name: ${{ env.IMAGE_NAME }}.sbom.spdx.json
+        path: ${{ env.IMAGE_FILE }}.sbom.spdx.json
 
     - uses: actions/upload-artifact@v4
       name: Store checksum as artifact

.github/workflows/test-gcp.yml

@@ -26,11 +26,11 @@ on:
           description: 'Image to test, overrides version_major to test a direct image instead.  Architecture must be set properly for the image being passed.  This must be a full path to a GCP image, for example, projects/almalinux-dev-images-469421/global/images/almalinux-9-v20230920'
           required: false
           default: ''
-        notify_mattermost:
-          description: "Send notification to Mattermost"
-          required: true
-          type: boolean
-          default: false
+        # notify_mattermost:
+        #   description: "Send notification to Mattermost"
+        #   required: true
+        #   type: boolean
+        #   default: false
 
 jobs:
   init-data:

almalinux-8-gcp.pkr.hcl

@@ -100,10 +100,10 @@ build {
     only = ["qemu.almalinux-8-gcp-aarch64"]
   }
 
-  # copy the repo metadata file into output
+  # copy SBOM metadata file into output
   post-processor "shell-local" {
     inline = [
-      "cp /tmp/repo-metadata-$PACKER_BUILD_NAME.txt output-$PACKER_BUILD_NAME/"
+      "cp /tmp/sbom-data-$PACKER_BUILD_NAME.json output-$PACKER_BUILD_NAME/"
     ]
   }
 

almalinux-9-gcp.pkr.hcl

@@ -100,10 +100,10 @@ build {
     only = ["qemu.almalinux-9-gcp-aarch64"]
   }
 
-  # copy the repo metadata file into output
+  # copy SBOM metadata file into output
   post-processor "shell-local" {
     inline = [
-      "cp /tmp/repo-metadata-$PACKER_BUILD_NAME.txt output-$PACKER_BUILD_NAME/"
+      "cp /tmp/sbom-data-$PACKER_BUILD_NAME.json output-$PACKER_BUILD_NAME/"
     ]
   }
 

almalinux_10_gcp.pkr.hcl

@@ -117,10 +117,10 @@ build {
     ]
   }
 
-  # copy the repo metadata file into output
+  # copy SBOM metadata file into output
   post-processor "shell-local" {
     inline = [
-      "cp /tmp/repo-metadata-$PACKER_BUILD_NAME.txt output-$PACKER_BUILD_NAME/"
+      "cp /tmp/sbom-data-$PACKER_BUILD_NAME.json output-$PACKER_BUILD_NAME/"
     ]
   }
 

ansible/roles/cleanup_vm/defaults/main.yml

@@ -1,2 +1,3 @@
 ---
 cleanup_ssh_host_keys: true
+collect_sbom_data: true

ansible/roles/cleanup_vm/tasks/main.yml

@@ -5,9 +5,10 @@
   changed_when: removeoldoutput.rc == 0
   ignore_errors: yes
 
-- name: Include repo metadata dump role (for SBOMs)
+- name: Include sbom_data role for SBOM data collection
   include_role:
-    name: dump_repo_metadata
+    name: sbom_data
+  when: collect_sbom_data | bool
 
 - name: Find persistent net rules
   ansible.builtin.find:

ansible/roles/sbom_data/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+- name: Copy SBOM data collector into the system
+  ansible.builtin.copy:
+    src: "{{ playbook_dir }}/../sbom-tools/sbom_data_collector.py"
+    dest: /dev/shm/sbom_data_collector.py
+  
+- name: Collect SBOM data from the system
+  ansible.builtin.shell: python3 /dev/shm/sbom_data_collector.py -o /dev/shm/sbom-data.json -v
+  register: sbom_data_collector
+  failed_when: false
+  
+- name: Write SBOM data to artifact file
+  ansible.builtin.fetch:
+    src: /dev/shm/sbom-data.json
+    dest: /tmp/sbom-data-{{ packer_build_name }}.json
+    flat: true
+  become: false
+  when: sbom_data_collector.changed