Merge branch 'main' into chore/vector-24.11

Author: siegfriedweber

.hadolint.yaml

@@ -30,7 +30,26 @@ ignored:
   # Reason: I've yet to see this being useful, where this happens we usually have no way to actually provide the file
   - SC1091
 
+  # Expressions don't expand in single quotes, use double quotes for that.
+  # https://www.shellcheck.net/wiki/SC2016
+  # Reason: Ignoring because envsubst requires environment variable names
+  # as parameter. These environment variables must not be expanded, e.g.:
+  # envsubst '${PRODUCT}:${HBASE_OPERATOR_TOOLS}' < /stackable/bin/hbck2.env
+  - SC2016
+
   # Use cd ... || exit in case cd fails.
   # https://github.com/koalaman/shellcheck/wiki/SC2164
   # Reason: Ignoring because we inherit SHELL from the base image which contains "-e" for bash
   - SC2164
+
+  # In POSIX sh, [[ ]] is undefined.
+  # https://www.shellcheck.net/wiki/SC3010
+  # Reason: Ignoring because we inherit SHELL from the base image which
+  # sets the default shell to Bash where [[ ]] is defined.
+  - SC3010
+
+  # In POSIX sh, string replacement is undefined.
+  # https://www.shellcheck.net/wiki/SC3060
+  # Reason: Ignoring because we inherit SHELL from the base image which
+  # sets the default shell to Bash where string replacement is supported.
+  - SC3060

CHANGELOG.md

@@ -15,7 +15,10 @@ All notable changes to this project will be documented in this file.
 - trino: Add version `455` ([#822]).
 - trino-cli: Add version `455` ([#822]).
 - spark: Add version `3.5.2` ([#848]).
+- statsd-exporter: Add version 0.27.0 ([#866]).
 - hadoop: Add patch "HADOOP-18516: Support Fixed SAS Token for ABFS Authentication" ([#852]).
+- hbase: Add hadoop-azure.jar to the lib directory to support the Azure Blob Filesystem and
+  the Azure Data Lake Storage ([#853]).
 
 ### Changed
 
@@ -34,6 +37,8 @@ All notable changes to this project will be documented in this file.
 - hbase: Remove 2.4.17 ([#846]).
 - omid: Remove 1.1.0 and 1.1.1 ([#846]).
 - spark: Remove 3.4.2 and 3.4.3 ([#848]).
+- statsd-exporter: Remove 0.26.1 ([#866]).
+- superset: Remove 2.1.3, 3.1.0 and 3.1.3 ([#866]).
 - zookeeper: Remove 3.8.4 ([#851]).
 
 ### Fixed
@@ -54,6 +59,8 @@ All notable changes to this project will be documented in this file.
 [#848]: https://github.com/stackabletech/docker-images/pull/848
 [#851]: https://github.com/stackabletech/docker-images/pull/851
 [#852]: https://github.com/stackabletech/docker-images/pull/852
+[#853]: https://github.com/stackabletech/docker-images/pull/853
+[#866]: https://github.com/stackabletech/docker-images/pull/866
 [#867]: https://github.com/stackabletech/docker-images/pull/867
 
 ## [24.7.0] - 2024-07-24

airflow/Dockerfile

@@ -101,7 +101,7 @@ RUN mkdir -pv ${AIRFLOW_HOME} && \
     mkdir -pv ${AIRFLOW_HOME}/dags && \
     mkdir -pv ${AIRFLOW_HOME}/logs && \
     chown --recursive stackable:stackable ${AIRFLOW_HOME} && \
-    curl --fail -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}"
+    curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}"
 
 COPY airflow/stackable/utils/entrypoint.sh /entrypoint.sh
 COPY airflow/stackable/utils/run-airflow.sh /run-airflow.sh

airflow/versions.py

@@ -3,15 +3,15 @@
         "product": "2.9.2",
         "python": "3.9",
         "git_sync": "v4.2.4",
-        "statsd_exporter": "0.26.1",
+        "statsd_exporter": "0.27.0",
         "tini": "0.19.0",
         "vector": "0.41.1",
     },
     {
         "product": "2.9.3",
         "python": "3.9",
         "git_sync": "v4.2.4",
-        "statsd_exporter": "0.26.1",
+        "statsd_exporter": "0.27.0",
         "tini": "0.19.0",
         "vector": "0.41.1",
     },

druid/Dockerfile

@@ -48,7 +48,7 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=1000,target=/stackable/.m2/reposi
     --mount=type=cache,id=npm-${PRODUCT},uid=1000,target=/stackable/.npm \
     --mount=type=cache,id=cache-${PRODUCT},uid=1000,target=/stackable/.cache \
     <<EOF
-curl --fail -L "https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-src.tar.gz" | tar -xzC .
+curl "https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-src.tar.gz" | tar -xzC .
 cd apache-druid-${PRODUCT}-src
 ./patches/apply_patches.sh ${PRODUCT}
 
@@ -71,7 +71,7 @@ fi
 # testdata in kuttl tests and the getting started guide.
 
 # Install OPA authorizer extension.
-curl --fail -L "https://repo.stackable.tech/repository/packages/druid/druid-opa-authorizer-${AUTHORIZER}.tar.gz" | tar -xzC /stackable/apache-druid-${PRODUCT}/extensions
+curl "https://repo.stackable.tech/repository/packages/druid/druid-opa-authorizer-${AUTHORIZER}.tar.gz" | tar -xzC /stackable/apache-druid-${PRODUCT}/extensions
 EOF
 
 FROM stackable/image/java-base AS final

hadoop/Dockerfile

@@ -20,20 +20,20 @@ COPY hadoop/stackable/fuse_dfs_wrapper /stackable/fuse_dfs_wrapper
 # At the same time a new HDFS Operator will still work with older images which do not have the symlink to the versionless jar.
 # After one of our next releases (23.11 or 24.x) we should update the operator to point at the non-versioned symlink (jmx_prometheus_javaagent.jar)
 # And then we can also remove the symlink to 0.16.1 from this Dockerfile.
-RUN curl --fail "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \
+RUN curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \
     chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \
     ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar && \
     ln -s /stackable/jmx/jmx_prometheus_javaagent.jar /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar
 
 RUN ARCH="${TARGETARCH/amd64/x64}" && \
-    curl --fail -L "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz"  | tar -xzC . && \
+    curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz"  | tar -xzC . && \
     ln -s "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" /stackable/async-profiler
 
 # This Protobuf version is the exact version as used in the Hadoop Dockerfile
 # See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
 # (this was hardcoded in the Dockerfile in earlier versions of Hadoop, make sure to look at the exact version in Github)
 WORKDIR /opt/protobuf-src
-RUN curl --fail -L -s -S https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-${PROTOBUF}.tar.gz -o /opt/protobuf.tar.gz && \
+RUN curl https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-${PROTOBUF}.tar.gz -o /opt/protobuf.tar.gz && \
     tar xzf /opt/protobuf.tar.gz --strip-components 1 --no-same-owner && \
     ./configure --prefix=/opt/protobuf && \
     make "-j$(nproc)" && \
@@ -60,7 +60,7 @@ COPY hadoop/stackable/patches /stackable/patches
 # Also skip building the yarn, mapreduce and minicluster modules: this will result in the modules being excluded but not all
 # jar files will be stripped if they are needed elsewhere e.g. share/hadoop/yarn will not be part of the build, but yarn jars
 # will still exist in share/hadoop/tools as they would be needed by the resource estimator tool. Such jars are removed in a later step.
-RUN curl --fail -L "https://repo.stackable.tech/repository/packages/hadoop/hadoop-${PRODUCT}-src.tar.gz" | tar -xzC . && \
+RUN curl "https://repo.stackable.tech/repository/packages/hadoop/hadoop-${PRODUCT}-src.tar.gz" | tar -xzC . && \
     patches/apply_patches.sh ${PRODUCT} && \
     cd hadoop-${PRODUCT}-src && \
     mvn --no-transfer-progress clean package -Pdist,native -pl '!hadoop-tools/hadoop-pipes,!hadoop-yarn-project,!hadoop-mapreduce-project,!hadoop-minicluster' -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true && \
@@ -99,7 +99,7 @@ WORKDIR /stackable
 # labels to build a rackID from.
 # Starting with hdfs-utils version 0.3.0 the topology provider is not a standalone jar anymore and included in hdfs-utils.
 
-RUN curl --fail -L "https://github.com/stackabletech/hdfs-utils/archive/refs/tags/v${HDFS_UTILS}.tar.gz" | tar -xzC . && \
+RUN curl "https://github.com/stackabletech/hdfs-utils/archive/refs/tags/v${HDFS_UTILS}.tar.gz" | tar -xzC . && \
     cd hdfs-utils-${HDFS_UTILS} && \
     mvn --no-transfer-progress clean package -P hadoop-${PRODUCT} -DskipTests -Dmaven.javadoc.skip=true && \
     mkdir -p /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib && \

hbase/Dockerfile

@@ -42,7 +42,7 @@ RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=1000,target=/stackable/.m2/
 ###
 ### HBase
 ###
-curl --fail -L "https://repo.stackable.tech/repository/packages/hbase/hbase-${PRODUCT}-src.tar.gz" | tar -xzC .
+curl "https://repo.stackable.tech/repository/packages/hbase/hbase-${PRODUCT}-src.tar.gz" | tar -xzC .
 mv hbase-${PRODUCT} hbase-${PRODUCT}-src
 
 chmod +x patches/apply_patches.sh
@@ -63,7 +63,7 @@ ln -s "/stackable/hbase-${PRODUCT}" /stackable/hbase
 ### JMX Prometheus Exporter/Agent
 ###
 if [[ -n "${JMX_EXPORTER}" ]] ; then
-  curl --fail -L "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar"
+  curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar"
   chmod +x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar"
   ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar
 fi
@@ -73,7 +73,7 @@ fi
 ###
 cd /stackable
 export ARCH="${TARGETARCH/amd64/x64}"
-curl --fail -L "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz"  | tar -xzC .
+curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz"  | tar -xzC .
 ln -s "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" /stackable/async-profiler
 
 # We're removing these to make the intermediate layer smaller
@@ -144,7 +144,7 @@ WORKDIR /stackable
 # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base
 RUN --mount=type=cache,id=maven-hbase-operator-tools,uid=1000,target=/stackable/.m2/repository <<EOF
 
-curl --fail -L "https://repo.stackable.tech/repository/packages/hbase-operator-tools/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}-src.tar.gz" | tar -xzC .
+curl "https://repo.stackable.tech/repository/packages/hbase-operator-tools/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}-src.tar.gz" | tar -xzC .
 mv hbase-operator-tools-${HBASE_OPERATOR_TOOLS} hbase-operator-tools-${HBASE_OPERATOR_TOOLS}-src
 chmod +x patches/apply_patches.sh
 patches/apply_patches.sh hbase-operator-tools/${HBASE_OPERATOR_TOOLS} hbase-operator-tools-${HBASE_OPERATOR_TOOLS}-src
@@ -231,7 +231,7 @@ WORKDIR /stackable
 
 RUN --mount=type=cache,id=maven-phoenix,uid=1000,target=/stackable/.m2/repository <<EOF
 cd /stackable
-curl --fail -L "https://repo.stackable.tech/repository/packages/phoenix/phoenix-${PHOENIX}-src.tar.gz" | tar -xzC .
+curl "https://repo.stackable.tech/repository/packages/phoenix/phoenix-${PHOENIX}-src.tar.gz" | tar -xzC .
 mv phoenix-${PHOENIX} phoenix-${PHOENIX}-src
 
 chmod +x patches/apply_patches.sh
@@ -317,6 +317,14 @@ COPY --chown=stackable:stackable --from=phoenix-builder /stackable/phoenix /stac
 COPY --chown=stackable:stackable --from=hadoop-s3-builder /stackable/bin/export-snapshot-to-s3 /stackable/bin/export-snapshot-to-s3
 COPY --chown=stackable:stackable --from=hadoop-s3-builder /stackable/hadoop/share/hadoop/tools/lib/ /stackable/hadoop/share/hadoop/tools/lib/
 
+# Copy the dependencies from Hadoop which are required for the Azure Data Lake
+# Storage (ADLS) to /stackable/hbase-${PRODUCT}/lib which is on the classpath.
+# hadoop-azure-${HADOOP}.jar contains the AzureBlobFileSystem which is required
+# by hadoop-common-${HADOOP}.jar if the scheme of a file system is "abfs://".
+COPY --chown=stackable:stackable --from=hadoop-builder \
+    /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar \
+    /stackable/hbase-${PRODUCT}/lib/
+
 COPY --chown=stackable:stackable --from=opa-authorizer-builder /stackable/hbase-opa-authorizer/target/hbase-opa-authorizer*.jar /stackable/hbase-${PRODUCT}/lib
 
 RUN <<EOF

hello-world/Dockerfile

@@ -20,6 +20,6 @@ RUN microdnf update && \
 USER stackable
 WORKDIR /stackable
 
-RUN curl --fail -L https://repo.stackable.tech/repository/packages/hello-world/hello-world-${PRODUCT}.jar -o hello-world.jar
+RUN curl "https://repo.stackable.tech/repository/packages/hello-world/hello-world-${PRODUCT}.jar" -o hello-world.jar
 
 CMD ["java", "-jar", "hello-world.jar"]

hive/Dockerfile

@@ -25,7 +25,7 @@ WORKDIR /stackable
 # Cache mounts are owned by root by default
 # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base
 RUN --mount=type=cache,id=maven-hive,uid=1000,target=/stackable/.m2/repository <<EOF
-curl --fail -L "https://repo.stackable.tech/repository/packages/hive/apache-hive-${PRODUCT}-src.tar.gz" | tar -xzC .
+curl "https://repo.stackable.tech/repository/packages/hive/apache-hive-${PRODUCT}-src.tar.gz" | tar -xzC .
 
 patches/apply_patches.sh ${PRODUCT}
 
@@ -53,7 +53,7 @@ fi
 cp /stackable/bin/start-metastore /stackable/apache-hive-metastore-${PRODUCT}-bin/bin
 rm -rf /stackable/apache-hive-${PRODUCT}-src
 
-curl --fail -L "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar"
+curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar"
 ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar
 
 # We're removing these to make the intermediate layer smaller

java-devel/Dockerfile

@@ -47,6 +47,8 @@ RUN microdnf update && \
 
 ENV JAVA_HOME=/usr/lib/jvm/jre-${PRODUCT}
 
+COPY --chown=stackable:0 java-devel/stackable/settings.xml /stackable/.m2/settings.xml
+
 # Mitigation for CVE-2021-44228 (Log4Shell)
 # This variable is supported as of Log4j version 2.10 and
 # disables the vulnerable feature