Include SBOM generation into GCP images build pipeline

.github/actions/shared-steps/action.yml

@@ -41,6 +41,11 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - uses: runs-on/action@v2
+      # only when in runs-on environment
+      if: env.RUNS_ON_VERSION != ''
+      with:
+        metrics: cpu,network,memory,disk,io
 
     - name: Runner OS, install extra packages
       shell: bash
@@ -73,14 +78,15 @@ runs:
         echo "version_major=${version_major}" >> $GITHUB_ENV
         echo "alma_arch=${alma_arch}" >> $GITHUB_ENV
 
-    - name: Prepare staff
+    - name: Prepare stuff
       shell: bash
       run: |
-        # Prepare staff
+        # Prepare stuff
+        packer_opts=
         case ${{ env.runner_os }} in
           ubuntu)
             # Packer options
-            packer_opts="-var ovmf_code=/usr/share/OVMF/OVMF_CODE_4M.fd -var ovmf_vars=/usr/share/OVMF/OVMF_VARS_4M.fd"
+            packer_opts="-var qemu_binary=/usr/bin/qemu-system-${{ inputs.arch }} -var ovmf_code=/usr/share/OVMF/OVMF_CODE_4M.fd -var ovmf_vars=/usr/share/OVMF/OVMF_VARS_4M.fd"
             ;;
           rhel)
             # Packer options
@@ -114,6 +120,10 @@ runs:
         # AWS S3 path to store images
         aws_s3_path=images/${{ env.version_major }}/${release}/${{ inputs.type }}/${{ env.TIME_STAMP }}
 
+        # tell packer we can use more cpu/ram if we're using runs-on
+        # which means we're using runs-on with metal instances
+        [[ ${{ env.RUNS_ON_VERSION }} != '' ]] && packer_opts="${packer_opts} -var cpus=$(($(nproc)-4)) -var memory_${{ env.alma_arch }}=32768"
+
         # Overriding packer source, image mask and S3 path where necessary
         case "${{ inputs.type }}${{ env.version_major }}" in
           azure8|azure9)
@@ -228,6 +238,17 @@ runs:
             output_mask=output-${packer_source}/AlmaLinux-*.${{ env.alma_arch }}*.qcow2
             packer_source=qemu.${packer_source}
             ;;
+          gcp8|gcp9)
+            output_mask=output-${packer_source}/AlmaLinux-*${version_major}*.${{ env.alma_arch }}.tar.gz
+            packer_source=qemu.${packer_source}
+            ;;
+          gcp10)
+            packer_source=almalinux_${{ env.version_major }}_${{ inputs.type }}_${{ env.alma_arch }}
+            [[ ${{ env.version_major }} == *"v2"* ]] && packer_source="${packer_source}_v2"
+            [[ ${{ inputs.variant }} == *"64k"* ]] && packer_source="almalinux_${{ env.version_major }}_${{ inputs.type }}_64k_${{ env.alma_arch }}"
+            output_mask=output-${packer_source}/AlmaLinux-*${version_major}*.${{ env.alma_arch }}.tar.gz
+            packer_source=qemu.${packer_source}
+            ;;
           *)
             output_mask=output-${output_mask}
             packer_source=qemu.${packer_source}
@@ -260,8 +281,24 @@ runs:
             ;;
         esac
 
+    - name: Remove KVM
+      if: inputs.type == 'vagrant_virtualbox' || inputs.type == 'vagrant_vmware'
+      shell: bash
+      run: |
+        # Remove KVM
+        case ${{ env.runner_os }} in
+          ubuntu)
+            sudo apt-get -y remove qemu-kvm
+            ;;
+          rhel)
+            sudo dnf -y -q remove qemu-kvm
+            ;;
+        esac
+        sudo rmmod kvm_amd || sudo rmmod kvm_intel || true
+        sudo rmmod kvm || true
+
     - name: Check nested virtualization support
-      if: inputs.arch == 'x86_64' && inputs.type != 'vagrant_virtualbox' && inputs.type != 'vagrant_vmware' && inputs.runner != 'aws-ec2'
+      if: inputs.arch == 'x86_64' && env.RUNS_ON_VERSION == '' && inputs.type != 'vagrant_virtualbox' && inputs.type != 'vagrant_vmware'
       shell: bash
       run: |
         # Check nested virtualization support
@@ -356,6 +393,26 @@ runs:
         # Install ansible
         sudo ${{ env.runner_os == 'ubuntu' && 'apt-get' || 'dnf -q' }} -y install ansible
 
+    - name: Clone SBOM tools
+      shell: bash
+      run: |
+        rm -rf sbom-tools
+        git clone --depth=1 https://github.com/javihernandez/cloud-images-sbom-tools.git sbom-tools
+
+    - name: Set up Python and install generator deps
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: sbom-tools/requirements.txt
+
+    - name: Create venv and install
+      shell: bash
+      run: |
+        python -m venv .venv-sbom
+        . .venv-sbom/bin/activate
+        pip install -r sbom-tools/requirements.txt
+
     - name: Initialize packer
       shell: bash
       run: sudo /usr/bin/packer init -upgrade .
@@ -365,12 +422,13 @@ runs:
       run: |
         # Build ${{ inputs.type }} image
         # PACKER_LOG=1
+        sudo systemctl start libvirtd
         sudo sh -c "/usr/bin/packer build ${{ env.PACKER_OPTS }} -only=${{ env.packer_source }} ."
 
     - name: Locate image file, generate checksum
       shell: bash
       run: |
-        # Locate image file, generate checksum
+        # Locate image file, generate checksum, rename repo metadata file
         ls -la $(dirname '${{ env.output_mask }}')
         image_file=$(ls -1 ${{ env.output_mask }} | head -n 1)
         [ "x${image_file}" = "x" ] && false
@@ -380,8 +438,85 @@ runs:
         echo "IMAGE_FILE=${image_file}" >> $GITHUB_ENV
         echo "IMAGE_NAME=$(basename ${image_file})" >> $GITHUB_ENV
 
-    # - name: Setup tmate session
-    #   uses: mxschmitt/action-tmate@v3
+        # don't fail if this doesn't exist, we may not always generate it
+        sudo mv sbom-data-*.json $(basename ${image_file}).sbom-data.json || true
+
+    - id: 'google-auth-dev-images'
+      if: env.IMAGE_TYPE == 'gcp'
+      uses: 'google-github-actions/auth@v2'
+      with:
+        workload_identity_provider: 'projects/443728870479/locations/global/workloadIdentityPools/github-actions/providers/github'
+        service_account: 'github-actions-cloud-images@almalinux-dev-images-469421.iam.gserviceaccount.com'
+
+    - name: 'Set up Google Cloud SDK'
+      if: env.IMAGE_TYPE == 'gcp'
+      uses: 'google-github-actions/[email protected]'
+
+    - name: 'Upload output to GCP storage bucket'
+      if: env.IMAGE_TYPE == 'gcp'
+      shell: bash
+      run: gcloud storage cp ${{ env.IMAGE_FILE }} gs://almalinux-images-dev/almalinux-${version_major}${{ inputs.arch == 'aarch64' && '-arm64' || '' }}-v$(date +'%Y%m%d')/root.tar.gz
+
+    - name: Clone gce_image_publish repo
+      if: env.IMAGE_TYPE == 'gcp'
+      uses: actions/checkout@v5
+      with:
+        path: compute-image-tools
+        repository: GoogleCloudPlatform/compute-image-tools
+        ref: "20250916.00"
+
+    - name: Build gce_image_publish tool
+      if: env.IMAGE_TYPE == 'gcp'
+      shell: bash
+      run: |
+        # we need golang
+        case ${{ env.runner_os }} in
+          ubuntu)
+            sudo apt update
+            sudo apt-get -y install golang-go
+            ;;
+          rhel)
+            sudo dnf -y -q install golang
+            ;;
+        esac
+        # print golang version for reference
+        go version
+        # Build gce_image_publish tool
+        cd compute-image-tools/cli_tools/gce_image_publish
+        go mod tidy
+        go install
+
+    - name: Create test image on GCP
+      if: env.IMAGE_TYPE == 'gcp'
+      shell: bash
+      run: |
+        /home/$USER/go/bin/gce_image_publish \
+          -var:environment=test \
+          -skip_confirmation \
+          -rollout_rate=0 \
+          -publish_project="almalinux-dev-images-469421" \
+          -work_project="almalinux-dev-images-469421" \
+          -replace \
+          -source_gcs_path="gs://almalinux-images-dev/" \
+          vm-scripts/gcp/almalinux_${version_major}${{ inputs.arch == 'aarch64' && '_arm64' || '' }}.publish.json
+
+    # - name: 'Run Google cloud-image-testing tests (basic suite)'
+    #   if: env.IMAGE_TYPE == 'gcp' && inputs.run_test == 'true'
+    #   shell: bash
+    #   run: |
+    #     cd cloud-image-tests
+    #     ./bin/manager \
+    #       -local_path bin \
+    #       -project almalinux-image-testing-469421 \
+    #       -filter '^(cvm|livemigrate|suspendresume|loadbalancer|guestagent|hostnamevalidation|imageboot|licensevalidation|network|security|hotattach|lssd|disk|packagevalidation|ssh|metadata|vmspec)$' \
+    #       -images 'projects/almalinux-dev-images-469421/global/images/family/almalinux-${{ env.version_major }}${{ inputs.arch == 'aarch64' && '-arm64' || '' }}' \
+    #       -parallel_stagger 10s -parallel_count 20
+
+    - name: Generate SBOM
+      shell: bash
+      run: |
+        echo "Generating SBOM document of ${{ env.IMAGE_FILE }}"
+        sudo .venv-sbom/bin/python3 sbom-tools/sbom_generator.py "${{ env.IMAGE_NAME }}" "${{ env.IMAGE_FILE }}.sbom-data.json" "${{ env.IMAGE_FILE }}.sbom.spdx.json"
 
     - name: Test ${{ inputs.type }} ${{ inputs.variant }} image
       # Skip testing for vagrant_virtualbox on GH runner because 'vugrant up' fails to connect to the newly created VM via ssh:
@@ -474,6 +609,24 @@ runs:
         name: ${{ env.IMAGE_NAME }}
         path: ${{ env.IMAGE_FILE }}
 
+    - uses: actions/upload-artifact@v4
+      name: Store collected sbom data as artifact
+      id: sbom-data-artifact
+      if: inputs.store_as_artifact == 'true'
+      with:
+        compression-level: 9
+        name: ${{ env.IMAGE_NAME }}.sbom-data.json
+        path: ${{ env.IMAGE_FILE }}.sbom-data.json
+
+    - uses: actions/upload-artifact@v4
+      name: Store SBOM as artifact
+      id: sbom-artifact
+      if: inputs.store_as_artifact == 'true'
+      with:
+        compression-level: 9
+        name: ${{ env.IMAGE_NAME }}.sbom.spdx.json
+        path: ${{ env.IMAGE_FILE }}.sbom.spdx.json
+
     - uses: actions/upload-artifact@v4
       name: Store checksum as artifact
       id: checksum-artifact
@@ -488,7 +641,7 @@ runs:
       id: pkglist-artifact
       if: inputs.store_as_artifact  == 'true' && env.got_pkgs_list == 'true'
       with:
-        compression-level: 1
+        compression-level: 9
         name: ${{ env.IMAGE_NAME }}.txt
         path: ${{ env.IMAGE_FILE }}.txt
 

.github/workflows/build.yml

@@ -30,6 +30,7 @@ on:
             - ALL
             - azure
             # - digitalocean        # TODO: require data to work with the cloud, such as: bucket, access key, secret key, etc.
+            - gcp
             - gencloud
             - oci
             - opennebula
@@ -114,6 +115,10 @@ jobs:
             VARIANTS_GH+=("azure-x86_64")
             VARIANTS_SH+=("azure-aarch64")
           fi
+          if [ "${{ inputs.image_type }}" = "gcp" -o "${{ inputs.image_type }}" = "ALL" ]; then
+            VARIANTS_GH+=("gcp-x86_64")
+            VARIANTS_SH+=("gcp-aarch64")
+          fi
           if [ "${{ inputs.image_type }}" = "digitalocean" -o "${{ inputs.image_type }}" = "ALL" ]; then
             if [[ "${{ inputs.version_major }}" != *"kitten"*  ]] && [[ "${{ inputs.version_major }}" != *"10"  ]]; then
               VARIANTS_GH+=("digitalocean-x86_64")
@@ -145,6 +150,14 @@ jobs:
               VARIANTS_SH+=("vagrant_vmware-x86_64") # VMware has networking issues on GitHub runners, so we use self-hosted runner
           fi
 
+          # Add SH values to matrix_gh if using runs-on
+          if [ "${{ github.repository_owner }}" == 'AlmaLinux' ]; then
+            for sh in "${VARIANTS_SH[@]}"; do
+              VARIANTS_GH+=("$sh")
+            done
+            unset VARIANTS_SH
+          fi
+
           [ ${#VARIANTS_GH[@]} -ne 0 ] && matrix_gh=$(printf '"%s",' "${VARIANTS_GH[@]}")
           matrix_gh=${matrix_gh%,}  # Remove the trailing comma
           echo matrix_gh=$(jq -c <<< [${matrix_gh}]) >> $GITHUB_OUTPUT
@@ -173,26 +186,34 @@ jobs:
 
   build-gh-hosted:
     name: ${{ matrix.variant }} ${{ matrix.matrix_gh }} image
+    permissions:
+      id-token: write
+      contents: read
     needs: [init-data]
     if: ${{ needs.init-data.outputs.matrix_gh != '[]' }}
-    runs-on: ubuntu-24.04
+    # use runs-on runners if within the almalinux org, otherwise GH runners"
+    runs-on: "${{ github.repository_owner == 'AlmaLinux' && format('runs-on={0}/family=c7i.metal-24xl+c7a.metal-48xl+*8gd.metal*/image=ubuntu24-full-{2}', github.run_id, matrix.variant, contains(matrix.matrix_gh, 'aarch64') && 'arm64' || 'x64') || 'ubuntu-24.04' }}"
     strategy:
       fail-fast: false
       matrix:
-        variant: ${{ fromJSON(format('["{0}"]', ( inputs.version_major == '10-kitten' || inputs.version_major == '10' ) && format('{0}", "{0}-v2', inputs.version_major) || inputs.version_major )) }}
+        variant: ${{ fromJSON(format('["{0}"]', ( (inputs.version_major == '10-kitten' || inputs.version_major == '10') && !(contains(needs.init-data.outputs.matrix_gh, 'aarch64') ) ) && format('{0}", "{0}-v2', inputs.version_major) || inputs.version_major )) }}
         matrix_gh: ${{ fromJSON(needs.init-data.outputs.matrix_gh) }}
         exclude:
           - matrix_gh: 'azure-x86_64'
             variant: '10-kitten-v2'
           - matrix_gh: 'oci-x86_64'
             variant: '10-kitten-v2'
+          - matrix_gh: 'gcp-x86_64'
+            variant: '10-kitten-v2'
           # Kitten x86_64_v2 Vagrant for VirtualBox stuck on "Waiting for SSH to become available"
           - matrix_gh: 'vagrant_virtualbox-x86_64'
             variant: '10-kitten-v2'
           - matrix_gh: 'digitalocean-x86_64'
             variant: '10-kitten-v2'
           - matrix_gh: 'azure-x86_64'
             variant: '10-v2'
+          - matrix_gh: 'gcp-x86_64'
+            variant: '10-v2'
           - matrix_gh: 'oci-x86_64'
             variant: '10-v2'
           - matrix_gh: 'digitalocean-x86_64'
@@ -211,7 +232,7 @@ jobs:
           echo "ARCH=$arch" >> $GITHUB_ENV
 
       - name: Checkout ${{ github.action_repository }}
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - uses: ./.github/actions/shared-steps
         name: ${{ matrix.variant }} ${{ matrix.matrix_gh }} image
@@ -232,15 +253,25 @@ jobs:
           store_as_artifact: ${{ inputs.store_as_artifact }}
           upload_to_s3: ${{ inputs.upload_to_s3 }}
           notify_mattermost: ${{ inputs.notify_mattermost }}
-          run_test: true # Do image simple testing and generate installed packages list (vagrant_* only)
+          run_test: true # Do image simple testing and generate installed packages list (vagrant_* and GCP only)
           # runner: ${{ github.repository_owner == 'AlmaLinux' && 'aws-ec2' || 'gh_hosted' }}
           runner: gh_hosted
         env:
           PACKER_GITHUB_API_TOKEN: ${{ secrets.GIT_HUB_TOKEN }}
 
+
+
+
+
+
+
+
+### Everything below is for self-hosted runners only ###
+
   start-self-hosted-runner:
     name: ${{ matrix.variant }} ${{ matrix.matrix_sh }} runner
-    if: ${{ inputs.self-hosted && needs.init-data.outputs.matrix_sh != '[]' }}
+    # If we're in the almalinux org we use runs-on for self-hosted
+    if: ${{ github.repository_owner != 'AlmaLinux' && inputs.self-hosted && needs.init-data.outputs.matrix_sh != '[]' }}
     runs-on: ubuntu-24.04
     needs: [init-data]
     strategy:
@@ -316,6 +347,8 @@ jobs:
         exclude:
           - matrix_sh: 'oci-aarch64'
             variant: '10-kitten'
+          - matrix_sh: 'oci-aarch64'
+            variant: '10'
           - matrix_sh: 'oci-aarch64'
             variant: '9-64k'
           - matrix_sh: 'oci-aarch64'