feat: generate SBOMs at build time for OPA, statsd_exporter and kafka

Author: dervoeti

kafka/Dockerfile

@@ -1,34 +1,49 @@
-# syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5
-# check=error=true
+# syntax=docker/dockerfile:1.8.1@sha256:e87caa74dcb7d46cd820352bfea12591f3dba3ddc4285e19c7dcd13359f7cefd
 
 FROM stackable/image/kcat AS kcat
 
-FROM stackable/image/java-devel AS kafka-builder
+FROM stackable/image/java-devel as kafka-builder
 
 ARG PRODUCT
 ARG SCALA
 ARG OPA_AUTHORIZER
 ARG JMX_EXPORTER
 
+RUN <<EOF
+microdnf update
+
+# patch: Required for the apply-patches.sh script
+microdnf install \
+patch
+
+microdnf clean all
+rm -rf /var/cache/yum
+EOF
+
 USER stackable
 WORKDIR /stackable
 
-RUN curl "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT}-src.tgz" | tar -xzC . && \
-    cd kafka-${PRODUCT}-src && \
+COPY --chown=stackable:stackable kafka/stackable/patches/apply_patches.sh /stackable/kafka-${PRODUCT}-src/patches/apply_patches.sh
+COPY --chown=stackable:stackable kafka/stackable/patches/${PRODUCT} /stackable/kafka-${PRODUCT}-src/patches/${PRODUCT}
+
+RUN curl --fail -L "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT}-src.tgz" | tar -xzC .
+RUN cd kafka-${PRODUCT}-src && \
+    ./patches/apply_patches.sh ${PRODUCT} && \
     # TODO: Try to install gradle via package manager (if possible) instead of fetching it from the internet
     # We don't specify "-x test" to skip the tests, as we might bump some Kafka internal dependencies in the future and
     # it's a good idea to run the tests in this case.
     ./gradlew clean releaseTarGz && \
+    ./gradlew cyclonedxBom && \
     tar -xf core/build/distributions/kafka_${SCALA}-${PRODUCT}.tgz -C /stackable && \
+    cp build/reports/bom.json /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json && \
     rm -rf /stackable/kafka_${SCALA}-${PRODUCT}/site-docs/ && \
     rm -rf /stackable/kafka-${PRODUCT}-src
 
-# TODO (@NickLarsenNZ): Compile from source: https://github.com/StyraInc/opa-kafka-plugin
-RUN curl https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
+RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
     -o /stackable/kafka_${SCALA}-${PRODUCT}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
 
 COPY --chown=stackable:stackable kafka/stackable/jmx/ /stackable/jmx/
-RUN curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
+RUN curl --fail -L https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
     -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
     chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
     ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar
@@ -69,8 +84,6 @@ LABEL name="Apache Kafka" \
 COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo
 RUN microdnf update && \
     microdnf install \
-    # needed by kcat for kerberos
-    cyrus-sasl-gssapi \
     # Can be removed once listener-operator integration is used
     kubectl && \
     microdnf clean all && \
@@ -85,6 +98,7 @@ COPY --chown=stackable:stackable kafka/licenses /licenses
 # We copy opa-authorizer.jar and jmx-exporter through the builder image to have an absolutely minimal final image
 # (e.g. we don't even need curl in it).
 COPY --chown=stackable:stackable --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
+COPY --chown=stackable:stackable --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json
 COPY --chown=stackable:stackable --from=kafka-builder /stackable/jmx/ /stackable/jmx/
 COPY --chown=stackable:stackable --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT}
 COPY --chown=stackable:stackable --from=kcat /licenses /licenses

kafka/stackable/patches/3.7.1/001-cyclonedx-plugin.patch

@@ -0,0 +1,27 @@
+diff --git a/build.gradle b/build.gradle
+index 32e6e8f..7bfe6c2 100644
+--- a/build.gradle
++++ b/build.gradle
+@@ -48,6 +48,22 @@ plugins {
+   // artifacts - see https://github.com/johnrengelman/shadow/issues/901
+   id 'com.github.johnrengelman.shadow' version '8.1.0' apply false
+   id 'com.diffplug.spotless' version '6.14.0' apply false // 6.14.1 and newer require Java 11 at compile time, so we can't upgrade until AK 4.0
++  id 'org.cyclonedx.bom' version '1.9.0'
++}
++
++cyclonedxBom {
++    // Specified the type of project being built. Defaults to 'library'
++    projectType = "application"
++    // Specified the version of the CycloneDX specification to use. Defaults to '1.5'
++    schemaVersion = "1.5"
++    // Boms destination directory. Defaults to 'build/reports'
++    destination = file("build/reports")
++    // The file name for the generated BOMs (before the file format suffix). Defaults to 'bom'
++    outputName = "bom"
++    // The file format generated, can be xml, json or all for generating both. Defaults to 'all'
++    outputFormat = "json"
++	// Fixes: https://github.com/gradle/gradle/issues/6854
++	skipConfigs = ["incrementalScalaAnalysisFortest", "incrementalScalaAnalysisFormain", "compileClasspath", "testCompileClasspath"]
+ }
+ 
+ ext {

kafka/stackable/patches/apply_patches.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Enable error handling and unset variable checking
+set -eu
+set -o pipefail
+
+# Check if $1 (VERSION) is provided
+if [ -z "${1-}" ]; then
+  echo "Please provide a value for VERSION as the first argument."
+  exit 1
+fi
+
+VERSION="$1"
+PATCH_DIR="patches/$VERSION"
+
+# Check if version-specific patches directory exists
+if [ ! -d "$PATCH_DIR" ]; then
+  echo "Patches directory '$PATCH_DIR' does not exist."
+  exit 1
+fi
+
+# Create an array to hold the patches in sorted order
+declare -a patch_files=()
+
+echo "Applying patches from ${PATCH_DIR}" now
+
+# Read the patch files into the array
+while IFS= read -r -d $'\0' file; do
+  patch_files+=("$file")
+done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV)
+
+echo "Found ${#patch_files[@]} patches, applying now"
+
+# Iterate through sorted patch files
+for patch_file in "${patch_files[@]}"; do
+  echo "Applying $patch_file"
+  # We can not use Git here, as we are not within a Git repo
+  patch --directory "." --strip=1 < "$patch_file" || {
+    echo "Failed to apply $patch_file"
+    exit 1
+  }
+done
+
+echo "All patches applied successfully."

opa/Dockerfile

@@ -65,22 +65,36 @@ ARG TARGETOS
 ENV GOARCH=$TARGETARCH
 ENV GOOS=$TARGETOS
 
-# go - used to build OPA
 # gzip, tar - used to unpack the OPA source
+# git - needed by the cyclonedx-gomod tool to determine the version of OPA
+# golang - used to build OPA
 RUN microdnf update && \
-    microdnf install \
-    go \
-    gzip \
-    tar && \
-    microdnf clean all
-
-RUN curl "https://repo.stackable.tech/repository/packages/opa/opa_${PRODUCT}.tar.gz" -o opa.tar.gz && \
+microdnf install \
+gzip \
+git \
+golang \
+tar && \
+microdnf clean all
+
+# We use version 1.7.0, since a newer version of cyclonedx-gomod is not compatible with the version of Golang (>= 1.23.1)
+RUN go install github.com/CycloneDX/cyclonedx-gomod/cmd/[email protected]
+RUN curl --fail -L "https://repo.stackable.tech/repository/packages/opa/opa_${PRODUCT}.tar.gz" -o opa.tar.gz && \
     tar -zxvf opa.tar.gz && \
-    mv opa-${PRODUCT} opa
+    mv "opa-${PRODUCT}" opa
 
 WORKDIR /opa
 
-RUN go build -o opa -buildmode=exe
+RUN <<EOF
+# Unfortunately, we need to create a dummy Git repository to allow cyclonedx-gomod to determine the version of OPA
+git init
+git add go.mod
+git config --global user.email "[email protected]"
+git config --global user.name "dummy"
+git commit -m "dummy"
+git tag "${PRODUCT}"
+go build -o opa -buildmode=exe
+~/go/bin/cyclonedx-gomod app -json -output-version 1.5 -output "opa_${PRODUCT}.cdx.json" -packages -files
+EOF
 
 FROM stackable/image/vector
 
@@ -107,7 +121,8 @@ COPY opa/licenses /licenses
 USER stackable
 WORKDIR /stackable/opa
 
-COPY --from=opa-builder /opa/opa /stackable/opa/opa
+COPY --from=opa-builder /opa/opa_${PRODUCT}.cdx.json /stackable/opa/
+COPY --from=opa-builder /opa/opa /stackable/opa/
 COPY --from=opa-bundle-builder --chown=stackable:stackable /opa-bundle-builder/target/release/stackable-opa-bundle-builder /stackable/opa-bundle-builder
 COPY --from=multilog-builder --chown=stackable:stackable /daemontools/admin/daemontools/command/multilog /stackable/multilog
 

statsd_exporter/Dockerfile

@@ -11,19 +11,33 @@ microdnf update
 
 # Tar and gzip are used to unpack the statsd_exporter source
 # Golang is used to build statsd_exporter
+# Git is needed by the cyclonedx-gomod tool to determine the version of statsd_exporter
 microdnf install \
   tar \
   gzip \
+  git \
   golang
 
 microdnf clean all
 rm -rf /var/cache/yum
 
 export GOPATH=/go_cache
-curl "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT}.src.tar.gz" | tar -xzC .
+go install github.com/CycloneDX/cyclonedx-gomod/cmd/[email protected]
+EOF
+
+RUN --mount=type=cache,id=go-statsd-exporter,uid=1000,target=/go_cache <<EOF
+curl --fail -L "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT}.src.tar.gz" | tar -xzC .
 (
   cd "statsd_exporter-${PRODUCT}" || exit
+
+  # Unfortunately, we need to create a dummy Git repository to allow cyclonedx-gomod to determine the version of statsd_exporter
+  git init
+  git add go.mod
+  git config --global user.email "[email protected]"
+  git config --global user.name "dummy"
+  git commit -m "dummy"
+  git tag "${PRODUCT}"
   go build -o ../statsd_exporter
+  /go_cache/bin/cyclonedx-gomod app -json -output-version 1.5 -output ../statsd_exporter-${PRODUCT}.cdx.json -packages -files
 )
-rm -rf "statsd_exporter-${PRODUCT}"
 EOF