Merge branch 'main' into hbase/entrypoint

Author: razvan

CHANGELOG.md

@@ -31,6 +31,7 @@ All notable changes to this project will be documented in this file.
 - Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872]).
 - java: migrate to temurin jdk/jre ([#894]).
 - tools: bump kubectl to `1.31.1` and jq to `1.7.1` ([#896]).
+- Make username, user id, group id configurable, use numeric ids everywhere, change group of all files to 0 ([#849], [#890]).
 
 ### Removed
 
@@ -66,6 +67,7 @@ All notable changes to this project will be documented in this file.
 [#822]: https://github.com/stackabletech/docker-images/pull/822
 [#846]: https://github.com/stackabletech/docker-images/pull/846
 [#848]: https://github.com/stackabletech/docker-images/pull/848
+[#849]: https://github.com/stackabletech/docker-images/pull/849
 [#851]: https://github.com/stackabletech/docker-images/pull/851
 [#852]: https://github.com/stackabletech/docker-images/pull/852
 [#853]: https://github.com/stackabletech/docker-images/pull/853
@@ -80,6 +82,7 @@ All notable changes to this project will be documented in this file.
 [#880]: https://github.com/stackabletech/docker-images/pull/880
 [#881]: https://github.com/stackabletech/docker-images/pull/881
 [#882]: https://github.com/stackabletech/docker-images/pull/882
+[#890]: https://github.com/stackabletech/docker-images/pull/890
 [#894]: https://github.com/stackabletech/docker-images/pull/894
 [#896]: https://github.com/stackabletech/docker-images/pull/896
 

airflow/Dockerfile

@@ -12,6 +12,7 @@ FROM stackable/image/statsd_exporter AS statsd_exporter-builder
 FROM stackable/image/vector AS airflow-build-image
 
 ARG PRODUCT
+ARG STATSD_EXPORTER
 ARG PYTHON
 ARG TARGETARCH
 
@@ -38,20 +39,37 @@ RUN microdnf update && \
         python${PYTHON}-pip \
         python${PYTHON}-wheel \
         # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
-        unixODBC && \
+        unixODBC \
+        # Needed to modify the SBOM
+        jq && \
     microdnf clean all && \
     rm -rf /var/cache/yum
 
-RUN python${PYTHON} -m venv --system-site-packages /stackable/app && \
-    source /stackable/app/bin/activate && \
-    pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt && \
-    # Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
-    pip install --no-cache-dir s3fs cyclonedx-bom && \
-    cyclonedx-py environment --schema-version 1.5 --outfile /stackable/airflow-${PRODUCT}.cdx.json
+RUN <<EOF
+python${PYTHON} -m venv --system-site-packages /stackable/app
+
+source /stackable/app/bin/activate
+
+pip install --no-cache-dir --upgrade pip
+pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt
+# Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
+pip install --no-cache-dir s3fs==2024.9.0 cyclonedx-bom==5.0.0
+
+# Create the SBOM for Airflow
+# Important: All `pip install` commands must be above this line, otherwise the SBOM will be incomplete
+cyclonedx-py environment --schema-version 1.5 --outfile /tmp/sbom.json
+
+# Break circular dependencies by removing the apache-airflow dependency from the providers
+jq '.dependencies |= map(if .ref | test("^apache-airflow-providers-") then
+    .dependsOn |= map(select(. != "apache-airflow=='${PRODUCT}'"))
+else
+    .
+end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT}.cdx.json
+EOF
 
 WORKDIR /stackable
 COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
+COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${STATSD_EXPORTER}.cdx.json
 
 FROM stackable/image/vector AS airflow-main-image
 

druid/Dockerfile

@@ -120,8 +120,8 @@ ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid
 # Force to overwrite the existing 'run-druid'
 ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

hadoop/Dockerfile

@@ -169,8 +169,8 @@ find . -name 'hadoop-*tests.jar' -type f -delete
 # It is so non-root users (as we are) can mount a FUSE device and let other users access it
 echo "user_allow_other" > /etc/fuse.conf
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

hbase/Dockerfile

@@ -355,8 +355,8 @@ ln --symbolic --logical --verbose "/stackable/hbase-${PRODUCT}" /stackable/hbase
 ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}" /stackable/hbase-operator-tools
 ln --symbolic --logical --verbose "/stackable/phoenix/phoenix-server-hbase-${HBASE_PROFILE}.jar" "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar"
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

hello-world/Dockerfile

@@ -22,8 +22,8 @@ rm -rf /var/cache/yum
 
 curl "https://repo.stackable.tech/repository/packages/hello-world/hello-world-${PRODUCT}.jar" -o /stackable/hello-world.jar
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

hive/Dockerfile

@@ -103,40 +103,47 @@ LABEL io.openshift.tags="ubi9,stackable,hive,sdp"
 LABEL io.k8s.description="${DESCRIPTION}"
 LABEL io.k8s.display-name="${NAME}"
 
-RUN <<EOF
-microdnf update
-microdnf clean all
-rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
-rm -rf /var/cache/yum
-EOF
-
-USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin
-RUN ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore
 
 # It is useful to see which version of Hadoop is used at a glance
 # Therefore the use of the full name here
 # TODO: Do we really need all of Hadoop in here?
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP}
-RUN ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop
+
+RUN <<EOF
+microdnf update
+microdnf clean all
+rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
+rm -rf /var/cache/yum
+
+ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore
+ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop
 
 # The next two sections for S3 and Azure use hardcoded version numbers on purpose instead of wildcards
 # This way the build will fail should one of the files not be available anymore in a later Hadoop version!
 
 # Add S3 Support for Hive (support for s3a://)
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar /stackable/hive-metastore/lib/
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/hive-metastore/lib/
 
 # Add Azure ABFS support (support for abfs://)
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stackable/hive-metastore/lib/
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/
+
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
+chown -R ${STACKABLE_USER_UID}:0 /stackable
+chmod -R g=u /stackable
+EOF
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx
 COPY hive/licenses /licenses
 
+USER ${STACKABLE_USER_UID}
+
 ENV HADOOP_HOME=/stackable/hadoop
 ENV HIVE_HOME=/stackable/hive-metastore
 ENV PATH="${PATH}":/stackable/hadoop/bin:/stackable/hive-metastore/bin

kafka-testing-tools/Dockerfile

@@ -8,6 +8,7 @@ FROM stackable/image/stackable-base AS final
 ARG PRODUCT
 ARG KCAT
 ARG RELEASE
+ARG STACKABLE_USER_UID
 
 LABEL name="Kafka Testing Tools" \
       maintainer="[email protected]" \
@@ -29,11 +30,10 @@ RUN microdnf install  \
     && rm -rf /var/cache/yum
 
 # Store kcat version with binary name and add softlink
-COPY --chown=stackable:stackable --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT}
 RUN ln -s /stackable/kcat-${KCAT} /stackable/kcat
-COPY --chown=stackable:stackable --from=kcat /licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses
 
-
-COPY --chown=stackable:stackable kafka-testing-tools/licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 kafka-testing-tools/licenses /licenses
 
 ENTRYPOINT ["/stackable/kcat"]

kafka/Dockerfile

@@ -9,53 +9,56 @@ ARG PRODUCT
 ARG SCALA
 ARG OPA_AUTHORIZER
 ARG JMX_EXPORTER
+ARG STACKABLE_USER_UID
 
-USER stackable
+RUN <<EOF
+microdnf update
+
+# patch: Required for the apply-patches.sh script
+microdnf install \
+patch
+
+microdnf clean all
+rm -rf /var/cache/yum
+EOF
+
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/apply_patches.sh /stackable/kafka-${PRODUCT}-src/patches/apply_patches.sh
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/kafka-${PRODUCT}-src/patches/${PRODUCT}
+
 RUN curl "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT}-src.tgz" | tar -xzC . && \
     cd kafka-${PRODUCT}-src && \
+    ./patches/apply_patches.sh ${PRODUCT} && \
     # TODO: Try to install gradle via package manager (if possible) instead of fetching it from the internet
     # We don't specify "-x test" to skip the tests, as we might bump some Kafka internal dependencies in the future and
     # it's a good idea to run the tests in this case.
     ./gradlew clean releaseTarGz && \
+    ./gradlew cyclonedxBom && \
     tar -xf core/build/distributions/kafka_${SCALA}-${PRODUCT}.tgz -C /stackable && \
+    cp build/reports/bom.json /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json && \
     rm -rf /stackable/kafka_${SCALA}-${PRODUCT}/site-docs/ && \
     rm -rf /stackable/kafka-${PRODUCT}-src
 
 # TODO (@NickLarsenNZ): Compile from source: https://github.com/StyraInc/opa-kafka-plugin
 RUN curl https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
     -o /stackable/kafka_${SCALA}-${PRODUCT}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
 
-COPY --chown=stackable:stackable kafka/stackable/jmx/ /stackable/jmx/
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/
 RUN curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
     -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
     chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
     ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar
 
-# For earlier versions this script removes the .class file that contains the
-# vulnerable code.
-# TODO: This can be restricted to target only versions which do not honor the environment
-#   varible that has been set above but this has not currently been implemented
-COPY shared/log4shell.sh /bin
-RUN /bin/log4shell.sh /stackable/kafka_${SCALA}-${PRODUCT}
-
-# Ensure no vulnerable files are left over
-# This will currently report vulnerable files being present, as it also alerts on
-# SocketNode.class, which we do not remove with our scripts.
-# Further investigation will be needed whether this should also be removed.
-COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64
-COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64
-COPY shared/log4shell_scanner /bin/log4shell_scanner
-RUN /bin/log4shell_scanner s /stackable/kafka_${SCALA}-${PRODUCT}
-# ===
 
 FROM stackable/image/java-base AS final
 
 ARG RELEASE
 ARG PRODUCT
 ARG SCALA
 ARG KCAT
+ARG STACKABLE_USER_UID
 
 LABEL name="Apache Kafka" \
       maintainer="[email protected]" \
@@ -67,32 +70,39 @@ LABEL name="Apache Kafka" \
 
 # This is needed for kubectl
 COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo
-RUN microdnf update && \
-    microdnf install \
-    # needed by kcat for kerberos
-    cyrus-sasl-gssapi \
-    # Can be removed once listener-operator integration is used
-    kubectl && \
-    microdnf clean all && \
-    rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt && \
-    rm -rf /var/cache/yum
-
-USER stackable
-WORKDIR /stackable
-
-COPY --chown=stackable:stackable kafka/licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}/kafka_${SCALA}-${PRODUCT}.cdx.json
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stackable/jmx/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses
 
-# We copy opa-authorizer.jar and jmx-exporter through the builder image to have an absolutely minimal final image
-# (e.g. we don't even need curl in it).
-COPY --chown=stackable:stackable --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
-COPY --chown=stackable:stackable --from=kafka-builder /stackable/jmx/ /stackable/jmx/
-COPY --chown=stackable:stackable --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT}
-COPY --chown=stackable:stackable --from=kcat /licenses /licenses
+WORKDIR /stackable
 
-RUN ln -s /stackable/bin/kcat-${KCAT} /stackable/bin/kcat && \
-    # kcat was located in /stackable/kcat - legacy
-    ln -s /stackable/bin/kcat /stackable/kcat && \
-    ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka
+RUN <<EOF
+microdnf update
+# cyrus-sasl-gssapi: needed by kcat for kerberos
+# kubectl: Can be removed once listener-operator integration is used
+microdnf install \
+  cyrus-sasl-gssapi \
+  kubectl
+
+microdnf clean all
+rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
+rm -rf /var/cache/yum
+
+ln -s /stackable/bin/kcat-${KCAT} /stackable/bin/kcat
+# kcat was located in /stackable/kcat - legacy
+ln -s /stackable/bin/kcat /stackable/kcat
+ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka
+
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
+chown -R ${STACKABLE_USER_UID}:0 /stackable
+chmod -R g=u /stackable
+EOF
+
+USER ${STACKABLE_USER_UID}
 
 ENV PATH="${PATH}:/stackable/bin:/stackable/kafka/bin"
 

kafka/stackable/patches/3.7.1/001-cyclonedx-plugin.patch

@@ -0,0 +1,52 @@
+diff --git a/build.gradle b/build.gradle
+index 32e6e8f..13a0def 100644
+--- a/build.gradle
++++ b/build.gradle
+@@ -48,6 +48,47 @@ plugins {
+   // artifacts - see https://github.com/johnrengelman/shadow/issues/901
+   id 'com.github.johnrengelman.shadow' version '8.1.0' apply false
+   id 'com.diffplug.spotless' version '6.14.0' apply false // 6.14.1 and newer require Java 11 at compile time, so we can't upgrade until AK 4.0
++  id 'org.cyclonedx.bom' version '1.10.0'
++}
++
++cyclonedxBom {
++    // Specified the type of project being built. Defaults to 'library'
++    projectType = "application"
++    // Specified the version of the CycloneDX specification to use. Defaults to '1.5'
++    schemaVersion = "1.5"
++    // Boms destination directory. Defaults to 'build/reports'
++    destination = file("build/reports")
++    // The file name for the generated BOMs (before the file format suffix). Defaults to 'bom'
++    outputName = "bom"
++    // The file format generated, can be xml, json or all for generating both. Defaults to 'all'
++    outputFormat = "json"
++    includeConfigs = ["runtimeClasspath"]
++    // Exclude test components. This list needs to be checked and, if it changed, updated for every new Kafka version.
++    // The list can be obtained by running `gradle projects | grep upgrade-system-tests`
++    skipProjects = [
++      'upgrade-system-tests-0100',
++      'upgrade-system-tests-0101',
++      'upgrade-system-tests-0102',
++      'upgrade-system-tests-0110',
++      'upgrade-system-tests-10',
++      'upgrade-system-tests-11',
++      'upgrade-system-tests-20',
++      'upgrade-system-tests-21',
++      'upgrade-system-tests-22',
++      'upgrade-system-tests-23',
++      'upgrade-system-tests-24',
++      'upgrade-system-tests-25',
++      'upgrade-system-tests-26',
++      'upgrade-system-tests-27',
++      'upgrade-system-tests-28',
++      'upgrade-system-tests-30',
++      'upgrade-system-tests-31',
++      'upgrade-system-tests-32',
++      'upgrade-system-tests-33',
++      'upgrade-system-tests-34',
++      'upgrade-system-tests-35',
++      'upgrade-system-tests-36'
++    ]
+ }
+ 
+ ext {