feat: add missing build-time SBOMs

airflow/Dockerfile

@@ -12,6 +12,7 @@ FROM stackable/image/statsd_exporter AS statsd_exporter-builder
 FROM stackable/image/vector AS airflow-build-image
 
 ARG PRODUCT
+ARG STATSD_EXPORTER
 ARG PYTHON
 ARG TARGETARCH
 
@@ -38,20 +39,37 @@ RUN microdnf update && \
         python${PYTHON}-pip \
         python${PYTHON}-wheel \
         # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
-        unixODBC && \
+        unixODBC \
+        # Needed to modify the SBOM
+        jq && \
     microdnf clean all && \
     rm -rf /var/cache/yum
 
-RUN python${PYTHON} -m venv --system-site-packages /stackable/app && \
-    source /stackable/app/bin/activate && \
-    pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt && \
-    # Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
-    pip install --no-cache-dir s3fs cyclonedx-bom && \
-    cyclonedx-py environment --schema-version 1.5 --outfile /stackable/airflow-${PRODUCT}.cdx.json
+RUN <<EOF
+python${PYTHON} -m venv --system-site-packages /stackable/app
+
+source /stackable/app/bin/activate
+
+pip install --no-cache-dir --upgrade pip
+pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt
+# Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
+pip install --no-cache-dir s3fs==2024.9.0 cyclonedx-bom==5.0.0
+
+# Create the SBOM for Airflow
+# Important: All `pip install` commands must be above this line, otherwise the SBOM will be incomplete
+cyclonedx-py environment --schema-version 1.5 --outfile /tmp/sbom.json
+
+# Break circular dependencies by removing the apache-airflow dependency from the providers
+jq '.dependencies |= map(if .ref | test("^apache-airflow-providers-") then
+    .dependsOn |= map(select(. != "apache-airflow=='${PRODUCT}'"))
+else
+    .
+end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT}.cdx.json
+EOF
 
 WORKDIR /stackable
 COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
+COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${STATSD_EXPORTER}.cdx.json
 
 FROM stackable/image/vector AS airflow-main-image
 

kafka/Dockerfile

@@ -11,16 +11,33 @@ ARG OPA_AUTHORIZER
 ARG JMX_EXPORTER
 ARG STACKABLE_USER_UID
 
+RUN <<EOF
+microdnf update
+
+# patch: Required for the apply-patches.sh script
+microdnf install \
+patch
+
+microdnf clean all
+rm -rf /var/cache/yum
+EOF
+
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/apply_patches.sh /stackable/kafka-${PRODUCT}-src/patches/apply_patches.sh
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/kafka-${PRODUCT}-src/patches/${PRODUCT}
+
 RUN curl "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT}-src.tgz" | tar -xzC . && \
     cd kafka-${PRODUCT}-src && \
+    ./patches/apply_patches.sh ${PRODUCT} && \
     # TODO: Try to install gradle via package manager (if possible) instead of fetching it from the internet
     # We don't specify "-x test" to skip the tests, as we might bump some Kafka internal dependencies in the future and
     # it's a good idea to run the tests in this case.
     ./gradlew clean releaseTarGz && \
+    ./gradlew cyclonedxBom && \
     tar -xf core/build/distributions/kafka_${SCALA}-${PRODUCT}.tgz -C /stackable && \
+    cp build/reports/bom.json /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json && \
     rm -rf /stackable/kafka_${SCALA}-${PRODUCT}/site-docs/ && \
     rm -rf /stackable/kafka-${PRODUCT}-src
 
@@ -55,6 +72,7 @@ LABEL name="Apache Kafka" \
 COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo
 COPY --chown=${STACKABLE_USER_UID}:0 kafka/licenses /licenses
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}/kafka_${SCALA}-${PRODUCT}.cdx.json
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stackable/jmx/
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT}
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses

kafka/stackable/patches/3.7.1/001-cyclonedx-plugin.patch

@@ -0,0 +1,52 @@
+diff --git a/build.gradle b/build.gradle
+index 32e6e8f..13a0def 100644
+--- a/build.gradle
++++ b/build.gradle
+@@ -48,6 +48,47 @@ plugins {
+   // artifacts - see https://github.com/johnrengelman/shadow/issues/901
+   id 'com.github.johnrengelman.shadow' version '8.1.0' apply false
+   id 'com.diffplug.spotless' version '6.14.0' apply false // 6.14.1 and newer require Java 11 at compile time, so we can't upgrade until AK 4.0
++  id 'org.cyclonedx.bom' version '1.10.0'
++}
++
++cyclonedxBom {
++    // Specified the type of project being built. Defaults to 'library'
++    projectType = "application"
++    // Specified the version of the CycloneDX specification to use. Defaults to '1.5'
++    schemaVersion = "1.5"
++    // Boms destination directory. Defaults to 'build/reports'
++    destination = file("build/reports")
++    // The file name for the generated BOMs (before the file format suffix). Defaults to 'bom'
++    outputName = "bom"
++    // The file format generated, can be xml, json or all for generating both. Defaults to 'all'
++    outputFormat = "json"
++    includeConfigs = ["runtimeClasspath"]
++    // Exclude test components. This list needs to be checked and, if it changed, updated for every new Kafka version.
++    // The list can be obtained by running `gradle projects | grep upgrade-system-tests`
++    skipProjects = [
++      'upgrade-system-tests-0100',
++      'upgrade-system-tests-0101',
++      'upgrade-system-tests-0102',
++      'upgrade-system-tests-0110',
++      'upgrade-system-tests-10',
++      'upgrade-system-tests-11',
++      'upgrade-system-tests-20',
++      'upgrade-system-tests-21',
++      'upgrade-system-tests-22',
++      'upgrade-system-tests-23',
++      'upgrade-system-tests-24',
++      'upgrade-system-tests-25',
++      'upgrade-system-tests-26',
++      'upgrade-system-tests-27',
++      'upgrade-system-tests-28',
++      'upgrade-system-tests-30',
++      'upgrade-system-tests-31',
++      'upgrade-system-tests-32',
++      'upgrade-system-tests-33',
++      'upgrade-system-tests-34',
++      'upgrade-system-tests-35',
++      'upgrade-system-tests-36'
++    ]
+ }
+
+ ext {

kafka/stackable/patches/3.8.0/001-cyclonedx-plugin.patch

@@ -0,0 +1,53 @@
+diff --git a/build.gradle b/build.gradle
+index 92082fe..e3d6c72 100644
+--- a/build.gradle
++++ b/build.gradle
+@@ -48,6 +48,48 @@ plugins {
+   // artifacts - see https://github.com/johnrengelman/shadow/issues/901
+   id 'com.github.johnrengelman.shadow' version '8.1.0' apply false
+   id 'com.diffplug.spotless' version '6.14.0' apply false // 6.14.1 and newer require Java 11 at compile time, so we can't upgrade until AK 4.0
++  id 'org.cyclonedx.bom' version '1.10.0'
++}
++
++cyclonedxBom {
++    // Specified the type of project being built. Defaults to 'library'
++    projectType = "application"
++    // Specified the version of the CycloneDX specification to use. Defaults to '1.5'
++    schemaVersion = "1.5"
++    // Boms destination directory. Defaults to 'build/reports'
++    destination = file("build/reports")
++    // The file name for the generated BOMs (before the file format suffix). Defaults to 'bom'
++    outputName = "bom"
++    // The file format generated, can be xml, json or all for generating both. Defaults to 'all'
++    outputFormat = "json"
++    includeConfigs = ["runtimeClasspath"]
++    // Exclude test components. This list needs to be checked and, if it changed, updated for every new Kafka version.
++    // The list can be obtained by running `gradle projects | grep upgrade-system-tests`
++    skipProjects = [
++      'upgrade-system-tests-0100',
++      'upgrade-system-tests-0101',
++      'upgrade-system-tests-0102',
++      'upgrade-system-tests-0110',
++      'upgrade-system-tests-10',
++      'upgrade-system-tests-11',
++      'upgrade-system-tests-20',
++      'upgrade-system-tests-21',
++      'upgrade-system-tests-22',
++      'upgrade-system-tests-23',
++      'upgrade-system-tests-24',
++      'upgrade-system-tests-25',
++      'upgrade-system-tests-26',
++      'upgrade-system-tests-27',
++      'upgrade-system-tests-28',
++      'upgrade-system-tests-30',
++      'upgrade-system-tests-31',
++      'upgrade-system-tests-32',
++      'upgrade-system-tests-33',
++      'upgrade-system-tests-34',
++      'upgrade-system-tests-35',
++      'upgrade-system-tests-36',
++      'upgrade-system-tests-37'
++    ]
+ }
+
+ ext {

kafka/stackable/patches/apply_patches.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Enable error handling and unset variable checking
+set -eu
+set -o pipefail
+
+# Check if $1 (VERSION) is provided
+if [ -z "${1-}" ]; then
+  echo "Please provide a value for VERSION as the first argument."
+  exit 1
+fi
+
+VERSION="$1"
+PATCH_DIR="patches/$VERSION"
+
+# Check if version-specific patches directory exists
+if [ ! -d "$PATCH_DIR" ]; then
+  echo "Patches directory '$PATCH_DIR' does not exist."
+  exit 1
+fi
+
+# Create an array to hold the patches in sorted order
+declare -a patch_files=()
+
+echo "Applying patches from ${PATCH_DIR}" now
+
+# Read the patch files into the array
+while IFS= read -r -d $'\0' file; do
+  patch_files+=("$file")
+done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV)
+
+echo "Found ${#patch_files[@]} patches, applying now"
+
+# Iterate through sorted patch files
+for patch_file in "${patch_files[@]}"; do
+  echo "Applying $patch_file"
+  # We can not use Git here, as we are not within a Git repo
+  patch --directory "." --strip=1 < "$patch_file" || {
+    echo "Failed to apply $patch_file"
+    exit 1
+  }
+done
+
+echo "All patches applied successfully."

opa/Dockerfile

@@ -64,22 +64,36 @@ ARG TARGETOS
 ENV GOARCH=$TARGETARCH
 ENV GOOS=$TARGETOS
 
-# go - used to build OPA
 # gzip, tar - used to unpack the OPA source
+# git - needed by the cyclonedx-gomod tool to determine the version of OPA
+# golang - used to build OPA
 RUN microdnf update && \
     microdnf install \
-    go \
+    git \
+    golang \
     gzip \
     tar && \
     microdnf clean all
 
+# We use version 1.7.0, since a newer version of cyclonedx-gomod is not compatible with the version of Golang (>= 1.23.1)
+RUN go install github.com/CycloneDX/cyclonedx-gomod/cmd/[email protected]
 RUN curl "https://repo.stackable.tech/repository/packages/opa/opa_${PRODUCT}.tar.gz" -o opa.tar.gz && \
     tar -zxvf opa.tar.gz && \
-    mv opa-${PRODUCT} opa
+    mv "opa-${PRODUCT}" opa
 
 WORKDIR /opa
 
-RUN go build -o opa -buildmode=exe
+RUN <<EOF
+# Unfortunately, we need to create a dummy Git repository to allow cyclonedx-gomod to determine the version of OPA
+git init
+git add go.mod
+git config --global user.email "[email protected]"
+git config --global user.name "dummy"
+git commit -m "dummy"
+git tag "${PRODUCT}"
+go build -o opa -buildmode=exe
+~/go/bin/cyclonedx-gomod app -json -output-version 1.5 -output "opa_${PRODUCT}.cdx.json" -packages -files
+EOF
 
 FROM stackable/image/vector
 
@@ -98,6 +112,7 @@ LABEL name="Open Policy Agent" \
 COPY opa/licenses /licenses
 
 COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /opa/opa /stackable/opa/opa
+COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /opa/opa_${PRODUCT}.cdx.json /stackable/opa/
 COPY --from=opa-bundle-builder --chown=${STACKABLE_USER_UID}:0 /opa-bundle-builder/target/release/stackable-opa-bundle-builder /stackable/opa-bundle-builder
 COPY --from=multilog-builder --chown=${STACKABLE_USER_UID}:0 /daemontools/admin/daemontools/command/multilog /stackable/multilog
 

statsd_exporter/Dockerfile

@@ -11,19 +11,33 @@ microdnf update
 
 # Tar and gzip are used to unpack the statsd_exporter source
 # Golang is used to build statsd_exporter
+# Git is needed by the cyclonedx-gomod tool to determine the version of statsd_exporter
 microdnf install \
   tar \
   gzip \
+  git \
   golang
 
 microdnf clean all
 rm -rf /var/cache/yum
 
 export GOPATH=/go_cache
+# We use version 1.7.0, since a newer version of cyclonedx-gomod is not compatible with the version of Golang (>= 1.23.1)
+go install github.com/CycloneDX/cyclonedx-gomod/cmd/[email protected]
+
 curl "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT}.src.tar.gz" | tar -xzC .
 (
   cd "statsd_exporter-${PRODUCT}" || exit
+
+  # Unfortunately, we need to create a dummy Git repository to allow cyclonedx-gomod to determine the version of statsd_exporter
+  git init
+  git add go.mod
+  git config --global user.email "[email protected]"
+  git config --global user.name "dummy"
+  git commit -m "dummy"
+  git tag "${PRODUCT}"
   go build -o ../statsd_exporter
+  $GOPATH/bin/cyclonedx-gomod app -json -output-version 1.5 -output ../statsd_exporter-${PRODUCT}.cdx.json -packages -files
 )
 rm -rf "statsd_exporter-${PRODUCT}"
 EOF