Skip to content

Commit 094be62

Browse files
dervoetisiegfriedweber
dervoeti
and
siegfriedweber
authored
feat: add missing build-time SBOMs (#895)
* feat: generate SBOMs at build time for OPA, statsd_exporter and kafka * fix: remove circular dependencies in Airflow SBOM * fix: kafka: ignore test components in SBOM * fix: kafka: missing patchfile for kafka 3.8.0 * fix: no need to cleanup builder image Co-authored-by: Siegfried Weber <[email protected]> * fix: add comment about cyclonedx-gomod to statsd_exporter as well Co-authored-by: Siegfried Weber <[email protected]> * fix: undo merge errors * fix: casing to make linter happy * fix: indenting and alphabetical sorting of packages Co-authored-by: Siegfried Weber <[email protected]> * fix: re-added line to remove sourcecode after build of statsd_exporter * fix: merge RUN layers in statsd_exporter * fix: place SBOM files closer to the application they are for * fix: update gradle cyclonedx plugin to version 1.10.0 * fix: remove unnecessary curl flags, because we have a curlrc file * fix: fixed variable substitution * feat: pinned versions of python packages * fix: fixes to adapt upstream changes * fix: use GOPATH for invoking cyclonedx-gomod * feat: add comment on how to obtain skipped projects in Kafka build --------- Co-authored-by: Siegfried Weber <[email protected]>
1 parent 45cbe54 commit 094be62

File tree

7 files changed

+226
-12
lines changed

7 files changed

+226
-12
lines changed

‎airflow/Dockerfile‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ FROM stackable/image/statsd_exporter AS statsd_exporter-builder
1212
FROM stackable/image/vector AS airflow-build-image
1313

1414
ARG PRODUCT
15+
ARG STATSD_EXPORTER
1516
ARG PYTHON
1617
ARG TARGETARCH
1718

@@ -38,20 +39,37 @@ RUN microdnf update && \
3839
python${PYTHON}-pip \
3940
python${PYTHON}-wheel \
4041
# The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
41-
unixODBC && \
42+
unixODBC \
43+
# Needed to modify the SBOM
44+
jq && \
4245
microdnf clean all && \
4346
rm -rf /var/cache/yum
4447

45-
RUN python${PYTHON} -m venv --system-site-packages /stackable/app && \
46-
source /stackable/app/bin/activate && \
47-
pip install --no-cache-dir --upgrade pip && \
48-
pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt && \
49-
# Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
50-
pip install --no-cache-dir s3fs cyclonedx-bom && \
51-
cyclonedx-py environment --schema-version 1.5 --outfile /stackable/airflow-${PRODUCT}.cdx.json
48+
RUN <<EOF
49+
python${PYTHON} -m venv --system-site-packages /stackable/app
50+
51+
source /stackable/app/bin/activate
52+
53+
pip install --no-cache-dir --upgrade pip
54+
pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt
55+
# Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
56+
pip install --no-cache-dir s3fs==2024.9.0 cyclonedx-bom==5.0.0
57+
58+
# Create the SBOM for Airflow
59+
# Important: All `pip install` commands must be above this line, otherwise the SBOM will be incomplete
60+
cyclonedx-py environment --schema-version 1.5 --outfile /tmp/sbom.json
61+
62+
# Break circular dependencies by removing the apache-airflow dependency from the providers
63+
jq '.dependencies |= map(if .ref | test("^apache-airflow-providers-") then
64+
.dependsOn |= map(select(. != "apache-airflow=='${PRODUCT}'"))
65+
else
66+
.
67+
end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT}.cdx.json
68+
EOF
5269

5370
WORKDIR /stackable
5471
COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
72+
COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${STATSD_EXPORTER}.cdx.json
5573

5674
FROM stackable/image/vector AS airflow-main-image
5775

‎kafka/Dockerfile‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,33 @@ ARG OPA_AUTHORIZER
1111
ARG JMX_EXPORTER
1212
ARG STACKABLE_USER_UID
1313

14+
RUN <<EOF
15+
microdnf update
16+
17+
# patch: Required for the apply-patches.sh script
18+
microdnf install \
19+
patch
20+
21+
microdnf clean all
22+
rm -rf /var/cache/yum
23+
EOF
24+
1425
USER ${STACKABLE_USER_UID}
1526
WORKDIR /stackable
1627

28+
COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/apply_patches.sh /stackable/kafka-${PRODUCT}-src/patches/apply_patches.sh
29+
COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/kafka-${PRODUCT}-src/patches/${PRODUCT}
30+
1731
RUN curl "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT}-src.tgz" | tar -xzC . && \
1832
cd kafka-${PRODUCT}-src && \
33+
./patches/apply_patches.sh ${PRODUCT} && \
1934
# TODO: Try to install gradle via package manager (if possible) instead of fetching it from the internet
2035
# We don't specify "-x test" to skip the tests, as we might bump some Kafka internal dependencies in the future and
2136
# it's a good idea to run the tests in this case.
2237
./gradlew clean releaseTarGz && \
38+
./gradlew cyclonedxBom && \
2339
tar -xf core/build/distributions/kafka_${SCALA}-${PRODUCT}.tgz -C /stackable && \
40+
cp build/reports/bom.json /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json && \
2441
rm -rf /stackable/kafka_${SCALA}-${PRODUCT}/site-docs/ && \
2542
rm -rf /stackable/kafka-${PRODUCT}-src
2643

@@ -55,6 +72,7 @@ LABEL name="Apache Kafka" \
5572
COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo
5673
COPY --chown=${STACKABLE_USER_UID}:0 kafka/licenses /licenses
5774
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
75+
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}/kafka_${SCALA}-${PRODUCT}.cdx.json
5876
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stackable/jmx/
5977
COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT}
6078
COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses

‎kafka/stackable/patches/3.7.1/001-cyclonedx-plugin.patch‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
diff --git a/build.gradle b/build.gradle
2+
index 32e6e8f..13a0def 100644
3+
--- a/build.gradle
4+
+++ b/build.gradle
5+
@@ -48,6 +48,47 @@ plugins {
6+
// artifacts - see https://github.com/johnrengelman/shadow/issues/901
7+
id 'com.github.johnrengelman.shadow' version '8.1.0' apply false
8+
id 'com.diffplug.spotless' version '6.14.0' apply false // 6.14.1 and newer require Java 11 at compile time, so we can't upgrade until AK 4.0
9+
+ id 'org.cyclonedx.bom' version '1.10.0'
10+
+}
11+
+
12+
+cyclonedxBom {
13+
+ // Specified the type of project being built. Defaults to 'library'
14+
+ projectType = "application"
15+
+ // Specified the version of the CycloneDX specification to use. Defaults to '1.5'
16+
+ schemaVersion = "1.5"
17+
+ // Boms destination directory. Defaults to 'build/reports'
18+
+ destination = file("build/reports")
19+
+ // The file name for the generated BOMs (before the file format suffix). Defaults to 'bom'
20+
+ outputName = "bom"
21+
+ // The file format generated, can be xml, json or all for generating both. Defaults to 'all'
22+
+ outputFormat = "json"
23+
+ includeConfigs = ["runtimeClasspath"]
24+
+ // Exclude test components. This list needs to be checked and, if it changed, updated for every new Kafka version.
25+
+ // The list can be obtained by running `gradle projects | grep upgrade-system-tests`
26+
+ skipProjects = [
27+
+ 'upgrade-system-tests-0100',
28+
+ 'upgrade-system-tests-0101',
29+
+ 'upgrade-system-tests-0102',
30+
+ 'upgrade-system-tests-0110',
31+
+ 'upgrade-system-tests-10',
32+
+ 'upgrade-system-tests-11',
33+
+ 'upgrade-system-tests-20',
34+
+ 'upgrade-system-tests-21',
35+
+ 'upgrade-system-tests-22',
36+
+ 'upgrade-system-tests-23',
37+
+ 'upgrade-system-tests-24',
38+
+ 'upgrade-system-tests-25',
39+
+ 'upgrade-system-tests-26',
40+
+ 'upgrade-system-tests-27',
41+
+ 'upgrade-system-tests-28',
42+
+ 'upgrade-system-tests-30',
43+
+ 'upgrade-system-tests-31',
44+
+ 'upgrade-system-tests-32',
45+
+ 'upgrade-system-tests-33',
46+
+ 'upgrade-system-tests-34',
47+
+ 'upgrade-system-tests-35',
48+
+ 'upgrade-system-tests-36'
49+
+ ]
50+
}
51+
52+
ext {

‎kafka/stackable/patches/3.8.0/001-cyclonedx-plugin.patch‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
diff --git a/build.gradle b/build.gradle
2+
index 92082fe..e3d6c72 100644
3+
--- a/build.gradle
4+
+++ b/build.gradle
5+
@@ -48,6 +48,48 @@ plugins {
6+
// artifacts - see https://github.com/johnrengelman/shadow/issues/901
7+
id 'com.github.johnrengelman.shadow' version '8.1.0' apply false
8+
id 'com.diffplug.spotless' version '6.14.0' apply false // 6.14.1 and newer require Java 11 at compile time, so we can't upgrade until AK 4.0
9+
+ id 'org.cyclonedx.bom' version '1.10.0'
10+
+}
11+
+
12+
+cyclonedxBom {
13+
+ // Specified the type of project being built. Defaults to 'library'
14+
+ projectType = "application"
15+
+ // Specified the version of the CycloneDX specification to use. Defaults to '1.5'
16+
+ schemaVersion = "1.5"
17+
+ // Boms destination directory. Defaults to 'build/reports'
18+
+ destination = file("build/reports")
19+
+ // The file name for the generated BOMs (before the file format suffix). Defaults to 'bom'
20+
+ outputName = "bom"
21+
+ // The file format generated, can be xml, json or all for generating both. Defaults to 'all'
22+
+ outputFormat = "json"
23+
+ includeConfigs = ["runtimeClasspath"]
24+
+ // Exclude test components. This list needs to be checked and, if it changed, updated for every new Kafka version.
25+
+ // The list can be obtained by running `gradle projects | grep upgrade-system-tests`
26+
+ skipProjects = [
27+
+ 'upgrade-system-tests-0100',
28+
+ 'upgrade-system-tests-0101',
29+
+ 'upgrade-system-tests-0102',
30+
+ 'upgrade-system-tests-0110',
31+
+ 'upgrade-system-tests-10',
32+
+ 'upgrade-system-tests-11',
33+
+ 'upgrade-system-tests-20',
34+
+ 'upgrade-system-tests-21',
35+
+ 'upgrade-system-tests-22',
36+
+ 'upgrade-system-tests-23',
37+
+ 'upgrade-system-tests-24',
38+
+ 'upgrade-system-tests-25',
39+
+ 'upgrade-system-tests-26',
40+
+ 'upgrade-system-tests-27',
41+
+ 'upgrade-system-tests-28',
42+
+ 'upgrade-system-tests-30',
43+
+ 'upgrade-system-tests-31',
44+
+ 'upgrade-system-tests-32',
45+
+ 'upgrade-system-tests-33',
46+
+ 'upgrade-system-tests-34',
47+
+ 'upgrade-system-tests-35',
48+
+ 'upgrade-system-tests-36',
49+
+ 'upgrade-system-tests-37'
50+
+ ]
51+
}
52+
53+
ext {

‎kafka/stackable/patches/apply_patches.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/usr/bin/env bash
2+
3+
# Enable error handling and unset variable checking
4+
set -eu
5+
set -o pipefail
6+
7+
# Check if $1 (VERSION) is provided
8+
if [ -z "${1-}" ]; then
9+
echo "Please provide a value for VERSION as the first argument."
10+
exit 1
11+
fi
12+
13+
VERSION="$1"
14+
PATCH_DIR="patches/$VERSION"
15+
16+
# Check if version-specific patches directory exists
17+
if [ ! -d "$PATCH_DIR" ]; then
18+
echo "Patches directory '$PATCH_DIR' does not exist."
19+
exit 1
20+
fi
21+
22+
# Create an array to hold the patches in sorted order
23+
declare -a patch_files=()
24+
25+
echo "Applying patches from ${PATCH_DIR}" now
26+
27+
# Read the patch files into the array
28+
while IFS= read -r -d $'\0' file; do
29+
patch_files+=("$file")
30+
done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV)
31+
32+
echo "Found ${#patch_files[@]} patches, applying now"
33+
34+
# Iterate through sorted patch files
35+
for patch_file in "${patch_files[@]}"; do
36+
echo "Applying $patch_file"
37+
# We can not use Git here, as we are not within a Git repo
38+
patch --directory "." --strip=1 < "$patch_file" || {
39+
echo "Failed to apply $patch_file"
40+
exit 1
41+
}
42+
done
43+
44+
echo "All patches applied successfully."

‎opa/Dockerfile‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,22 +64,36 @@ ARG TARGETOS
6464
ENV GOARCH=$TARGETARCH
6565
ENV GOOS=$TARGETOS
6666

67-
# go - used to build OPA
6867
# gzip, tar - used to unpack the OPA source
68+
# git - needed by the cyclonedx-gomod tool to determine the version of OPA
69+
# golang - used to build OPA
6970
RUN microdnf update && \
7071
microdnf install \
71-
go \
72+
git \
73+
golang \
7274
gzip \
7375
tar && \
7476
microdnf clean all
7577

78+
# We use version 1.7.0, since a newer version of cyclonedx-gomod is not compatible with the version of Golang (>= 1.23.1)
79+
RUN go install github.com/CycloneDX/cyclonedx-gomod/cmd/[email protected]
7680
RUN curl "https://repo.stackable.tech/repository/packages/opa/opa_${PRODUCT}.tar.gz" -o opa.tar.gz && \
7781
tar -zxvf opa.tar.gz && \
78-
mv opa-${PRODUCT} opa
82+
mv "opa-${PRODUCT}" opa
7983

8084
WORKDIR /opa
8185

82-
RUN go build -o opa -buildmode=exe
86+
RUN <<EOF
87+
# Unfortunately, we need to create a dummy Git repository to allow cyclonedx-gomod to determine the version of OPA
88+
git init
89+
git add go.mod
90+
git config --global user.email "[email protected]"
91+
git config --global user.name "dummy"
92+
git commit -m "dummy"
93+
git tag "${PRODUCT}"
94+
go build -o opa -buildmode=exe
95+
~/go/bin/cyclonedx-gomod app -json -output-version 1.5 -output "opa_${PRODUCT}.cdx.json" -packages -files
96+
EOF
8397

8498
FROM stackable/image/vector
8599

@@ -98,6 +112,7 @@ LABEL name="Open Policy Agent" \
98112
COPY opa/licenses /licenses
99113

100114
COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /opa/opa /stackable/opa/opa
115+
COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /opa/opa_${PRODUCT}.cdx.json /stackable/opa/
101116
COPY --from=opa-bundle-builder --chown=${STACKABLE_USER_UID}:0 /opa-bundle-builder/target/release/stackable-opa-bundle-builder /stackable/opa-bundle-builder
102117
COPY --from=multilog-builder --chown=${STACKABLE_USER_UID}:0 /daemontools/admin/daemontools/command/multilog /stackable/multilog
103118

‎statsd_exporter/Dockerfile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,33 @@ microdnf update
1111

1212
# Tar and gzip are used to unpack the statsd_exporter source
1313
# Golang is used to build statsd_exporter
14+
# Git is needed by the cyclonedx-gomod tool to determine the version of statsd_exporter
1415
microdnf install \
1516
tar \
1617
gzip \
18+
git \
1719
golang
1820

1921
microdnf clean all
2022
rm -rf /var/cache/yum
2123

2224
export GOPATH=/go_cache
25+
# We use version 1.7.0, since a newer version of cyclonedx-gomod is not compatible with the version of Golang (>= 1.23.1)
26+
go install github.com/CycloneDX/cyclonedx-gomod/cmd/[email protected]
27+
2328
curl "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT}.src.tar.gz" | tar -xzC .
2429
(
2530
cd "statsd_exporter-${PRODUCT}" || exit
31+
32+
# Unfortunately, we need to create a dummy Git repository to allow cyclonedx-gomod to determine the version of statsd_exporter
33+
git init
34+
git add go.mod
35+
git config --global user.email "[email protected]"
36+
git config --global user.name "dummy"
37+
git commit -m "dummy"
38+
git tag "${PRODUCT}"
2639
go build -o ../statsd_exporter
40+
$GOPATH/bin/cyclonedx-gomod app -json -output-version 1.5 -output ../statsd_exporter-${PRODUCT}.cdx.json -packages -files
2741
)
2842
rm -rf "statsd_exporter-${PRODUCT}"
2943
EOF

0 commit comments

Comments
 (0)