Merge branch 'main' into feature/gaiaX-oidc

Author: Maleware

.gitattributes

@@ -0,0 +1 @@
+nix/** linguist-generated

.scripts/get_manifest_digest.sh

@@ -16,7 +16,7 @@ All notable changes to this project will be documented in this file.
 - trino: Add version `455` ([#822]).
 - trino-cli: Add version `455` ([#822]).
 - spark: Add version `3.5.2` ([#848]).
-- statsd-exporter: Bump version to 0.27.1 ([#866], [#879]).
+- statsd-exporter: Bump version to `0.27.1` ([#866], [#879]).
 - hadoop: Add patch "HADOOP-18516: Support Fixed SAS Token for ABFS Authentication" ([#852]).
 - hbase: Add hadoop-azure.jar to the lib directory to support the Azure Blob Filesystem and
   the Azure Data Lake Storage ([#853]).
@@ -27,8 +27,11 @@ All notable changes to this project will be documented in this file.
 
 - ci: Rename local actions, adjust action inputs and outputs, add definition
   README file ([#819]).
-- Update cargo-cyclonedx to 0.5.5 and build CycloneDX 1.5 files ([#783])
-- Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872])
+- Update cargo-cyclonedx to 0.5.5 and build CycloneDX 1.5 files ([#783]).
+- Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872]).
+- java: migrate to temurin jdk/jre ([#894]).
+- tools: bump kubectl to `1.31.1` and jq to `1.7.1` ([#896]).
+- Make username, user id, group id configurable, use numeric ids everywhere, change group of all files to 0 ([#849], [#890]).
 
 ### Removed
 
@@ -38,13 +41,14 @@ All notable changes to this project will be documented in this file.
 - kafka: Remove versions `3.4.1`, `3.6.1`, `3.6.2` ([#813]).
 - trino: Remove versions `414`, `442` ([#822]).
 - trino-cli: Remove version `451` ([#822]).
-- hbase: Remove 2.4.17 ([#846]).
-- omid: Remove 1.1.0 and 1.1.1 ([#846]).
-- spark: Remove 3.4.2 and 3.4.3 ([#848]).
-- statsd-exporter: Remove 0.26.1 ([#866]).
-- superset: Remove 2.1.3, 3.1.0 and 3.1.3 ([#866]).
-- zookeeper: Remove 3.8.4 ([#851]).
-- nifi: Remove 1.21.0 and 1.25.0 ([#868]).
+- hbase: Remove `2.4.17` ([#846]).
+- omid: Remove `1.1.0` and `1.1.1` ([#846]).
+- spark: Remove `3.4.2` and `3.4.3` ([#848]).
+- statsd-exporter: Remove `0.26.1` ([#866]).
+- superset: Remove `2.1.3`, `3.1.0` and `3.1.3` ([#866]).
+- zookeeper: Remove `3.8.4` ([#851]).
+- nifi: Remove `1.21.0` and `1.25.0` ([#868]).
+- druid: Remove `28.0.1` ([#880]).
 
 ### Fixed
 
@@ -63,6 +67,7 @@ All notable changes to this project will be documented in this file.
 [#822]: https://github.com/stackabletech/docker-images/pull/822
 [#846]: https://github.com/stackabletech/docker-images/pull/846
 [#848]: https://github.com/stackabletech/docker-images/pull/848
+[#849]: https://github.com/stackabletech/docker-images/pull/849
 [#851]: https://github.com/stackabletech/docker-images/pull/851
 [#852]: https://github.com/stackabletech/docker-images/pull/852
 [#853]: https://github.com/stackabletech/docker-images/pull/853
@@ -74,8 +79,12 @@ All notable changes to this project will be documented in this file.
 [#877]: https://github.com/stackabletech/docker-images/pull/877
 [#878]: https://github.com/stackabletech/docker-images/pull/878
 [#879]: https://github.com/stackabletech/docker-images/pull/879
+[#880]: https://github.com/stackabletech/docker-images/pull/880
 [#881]: https://github.com/stackabletech/docker-images/pull/881
 [#882]: https://github.com/stackabletech/docker-images/pull/882
+[#890]: https://github.com/stackabletech/docker-images/pull/890
+[#894]: https://github.com/stackabletech/docker-images/pull/894
+[#896]: https://github.com/stackabletech/docker-images/pull/896
 
 ## [24.7.0] - 2024-07-24
 

.scripts/get_repo_digest.sh

@@ -12,6 +12,7 @@ FROM stackable/image/statsd_exporter AS statsd_exporter-builder
 FROM stackable/image/vector AS airflow-build-image
 
 ARG PRODUCT
+ARG STATSD_EXPORTER
 ARG PYTHON
 ARG TARGETARCH
 
@@ -38,22 +39,39 @@ RUN microdnf update && \
         python${PYTHON}-pip \
         python${PYTHON}-wheel \
         # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
-        unixODBC && \
+        unixODBC \
+        # Needed to modify the SBOM
+        jq && \
     microdnf clean all && \
     rm -rf /var/cache/yum
 
-RUN python${PYTHON} -m venv --system-site-packages /stackable/app && \
-    source /stackable/app/bin/activate && \
-    pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt && \
-    # Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
-    pip install --no-cache-dir s3fs cyclonedx-bom && \
-    # Needed for OIDC
-    pip install --no-cache-dir Flask_OIDC==2.2.0 Flask-OpenID==1.3.1 && \
-    cyclonedx-py environment --schema-version 1.5 --outfile /stackable/airflow-${PRODUCT}.cdx.json
+RUN <<EOF
+python${PYTHON} -m venv --system-site-packages /stackable/app
+
+source /stackable/app/bin/activate
+
+pip install --no-cache-dir --upgrade pip
+pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT} --constraint /tmp/constraints.txt
+# Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
+pip install --no-cache-dir s3fs==2024.9.0 cyclonedx-bom==5.0.0
+# Needed for OIDC
+pip install --no-cache-dir Flask_OIDC==2.2.0 Flask-OpenID==1.3.1
+
+# Create the SBOM for Airflow
+# Important: All `pip install` commands must be above this line, otherwise the SBOM will be incomplete
+cyclonedx-py environment --schema-version 1.5 --outfile /tmp/sbom.json
+
+# Break circular dependencies by removing the apache-airflow dependency from the providers
+jq '.dependencies |= map(if .ref | test("^apache-airflow-providers-") then
+    .dependsOn |= map(select(. != "apache-airflow=='${PRODUCT}'"))
+else
+    .
+end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT}.cdx.json
+EOF
 
 WORKDIR /stackable
 COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
+COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${STATSD_EXPORTER}.cdx.json
 
 FROM stackable/image/vector AS airflow-main-image
 

CHANGELOG.md

@@ -120,8 +120,8 @@ ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid
 # Force to overwrite the existing 'run-druid'
 ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

airflow/Dockerfile

@@ -5,16 +5,6 @@
         "java-devel": "11",
         "authorizer": "0.5.0",
     },
-    {
-        "product": "28.0.1",
-        # Java 17 should be fully supported as of 27.0.0 https://github.com/apache/druid/releases#27.0.0-highlights-java-17-support
-        # Did not work in a quick test due to reflection error:
-        # Caused by: java.lang.reflect.InaccessibleObjectException: Unable to make protected final java.lang.Class
-        # java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int,java.security.ProtectionDomain) throws java.lang.ClassFormatError
-        "java-base": "11",
-        "java-devel": "11",
-        "authorizer": "0.5.0",
-    },
     {
         "product": "30.0.0",
         # https://druid.apache.org/docs/30.0.0/operations/java/

druid/Dockerfile

@@ -169,8 +169,8 @@ find . -name 'hadoop-*tests.jar' -type f -delete
 # It is so non-root users (as we are) can mount a FUSE device and let other users access it
 echo "user_allow_other" > /etc/fuse.conf
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

druid/versions.py

@@ -354,8 +354,8 @@ ln --symbolic --logical --verbose "/stackable/hbase-${PRODUCT}" /stackable/hbase
 ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}" /stackable/hbase-operator-tools
 ln --symbolic --logical --verbose "/stackable/phoenix/phoenix-server-hbase-${HBASE_PROFILE}.jar" "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar"
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

hadoop/Dockerfile

@@ -22,8 +22,8 @@ rm -rf /var/cache/yum
 
 curl "https://repo.stackable.tech/repository/packages/hello-world/hello-world-${PRODUCT}.jar" -o /stackable/hello-world.jar
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF