Merge branch 'main' into chore/spark-1080

Author: razvan

CHANGELOG.md

@@ -39,6 +39,7 @@ All notable changes to this project will be documented in this file.
   `check-permissions-ownership.sh` provided in stackable-base image ([#1043]).
 - java: Add JDK 24 ([#1097]).
 - ci: Add golang image to mirror workflow ([#1103]).
+- hbase: add 2.6.2 and upgrade dependencies ([#1101])
 
 ### Changed
 
@@ -84,6 +85,7 @@ All notable changes to this project will be documented in this file.
 - hadoop: Remove `3.3.4` and `3.4.0` ([#1099]).
 - opa: Remove `0.67.1` ([#1103]).
 - opa: Remove legacy bundle-builder from container build ([#1103]).
+- hbase: Remove 2.4.18 ([#1101])
 - druid: Remove `30.0.0` ([#1110]).
 - nifi: Remove `2.2.0` ([#1114]).
 
@@ -114,6 +116,7 @@ All notable changes to this project will be documented in this file.
 [#1097]: https://github.com/stackabletech/docker-images/pull/1097
 [#1098]: https://github.com/stackabletech/docker-images/pull/1098
 [#1099]: https://github.com/stackabletech/docker-images/pull/1099
+[#1101]: https://github.com/stackabletech/docker-images/pull/1101
 [#1102]: https://github.com/stackabletech/docker-images/pull/1102
 [#1103]: https://github.com/stackabletech/docker-images/pull/1103
 [#1106]: https://github.com/stackabletech/docker-images/pull/1106

hbase/Dockerfile

@@ -6,12 +6,10 @@ FROM stackable/image/hadoop AS hadoop-builder
 FROM stackable/image/java-devel AS hbase-builder
 
 ARG PRODUCT
-ARG HBASE_THIRDPARTY
 ARG HBASE_OPERATOR_TOOLS
 ARG ASYNC_PROFILER
 ARG PHOENIX
 ARG HBASE_PROFILE
-ARG JMX_EXPORTER
 ARG HADOOP
 ARG TARGETARCH
 ARG TARGETOS
@@ -28,7 +26,6 @@ WORKDIR /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/patchable.toml /stackable/src/hbase/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/${PRODUCT} /stackable/src/hbase/stackable/patches/${PRODUCT}
-COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx
 
 # Cache mounts are owned by root by default
 # We need to explicitly give the uid to use
@@ -49,8 +46,25 @@ cd "$(/stackable/patchable --images-repo-root=src checkout hbase ${PRODUCT})"
 
 # The release scripts of HBase also run the build twice (three times in fact, once again to build the site which we skip here).
 # I chose to replicate that exact behavior for consistency so please don't merge the two mvn runs into one unless you really know what you're doing!
-mvn --batch-mode --no-transfer-progress -Dhadoop.profile=3.0 -Dhadoop-three.version=${HADOOP} clean install -DskipTests
-mvn --batch-mode --no-transfer-progress -Dhadoop.profile=3.0 -Dhadoop-three.version=${HADOOP} install assembly:single -DskipTests -Dcheckstyle.skip=true -Prelease
+# Cannot skip building tests here because the assembly plugin needs a shell script from the test directory.
+mvn \
+  --batch-mode \
+  --no-transfer-progress \
+  -Dhadoop.profile=3.0 \
+  -Dhadoop-three.version=${HADOOP} \
+  -DskipTests \
+  clean install
+
+mvn \
+  --batch-mode \
+  --no-transfer-progress \
+  -Dhadoop.profile=3.0 \
+  -Dhadoop-three.version=${HADOOP} \
+  -DskipTests \
+  -Dcheckstyle.skip=true \
+  -Prelease \
+  install assembly:single
+
 tar -xzf hbase-assembly/target/hbase-${PRODUCT}-bin.tar.gz -C /stackable/
 mv hbase-assembly/target/bom.json /stackable/hbase-${PRODUCT}/hbase-${PRODUCT}.cdx.json
 
@@ -59,15 +73,6 @@ mv hbase-assembly/target/bom.json /stackable/hbase-${PRODUCT}/hbase-${PRODUCT}.c
 
 ln -s "/stackable/hbase-${PRODUCT}" /stackable/hbase
 
-###
-### JMX Prometheus Exporter/Agent
-###
-if [[ -n "${JMX_EXPORTER}" ]] ; then
-  curl --fail "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar"
-  chmod +x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar"
-  ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar
-fi
-
 ###
 ### Async Profiler
 ###
@@ -93,14 +98,17 @@ EOF
 
 FROM stackable/image/java-devel AS opa-authorizer-builder
 
+ARG PRODUCT
 ARG OPA_AUTHORIZER
 ARG DELETE_CACHES
 ARG STACKABLE_USER_UID
 
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-RUN --mount=type=cache,id=maven-opa,uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
+# Use the product version as cache id "maven-opa-${PRODUCT}" to avoid the "rm -rf" problem described above.
+# Using the OPA_AUTHORIZER arg is not unique if the same version is used for multiple products.
+RUN --mount=type=cache,id=maven-opa-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 ###
 ### OPA Authorizer (only for 2.6 upwards)
 ###
@@ -109,9 +117,10 @@ if [[ -n "$OPA_AUTHORIZER" ]]; then
     mvn \
       --batch-mode \
       --no-transfer-progress \
-      package \
       -DskipTests \
-      -fhbase-opa-authorizer
+      -Dmaven.test.skip=true \
+      -fhbase-opa-authorizer \
+      package
 else
   # Create a dummy jar to avoid errors when copying it the final image
   mkdir -p hbase-opa-authorizer/target
@@ -155,16 +164,13 @@ WORKDIR /stackable
 RUN --mount=type=cache,id=maven-hbase-operator-tools-${HBASE_OPERATOR_TOOLS},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 cd "$(/stackable/patchable --images-repo-root=src checkout hbase-operator-tools ${HBASE_OPERATOR_TOOLS})"
 
-# Remove the Git repo because `git-commit-id-maven-plugin` tries to get the latest commit if a Git repo is present,
-# which fails due to a problem with worktrees, see https://github.com/git-commit-id/git-commit-id-maven-plugin/issues/215
-rm .git
-
 mvn \
   --batch-mode \
   --no-transfer-progress \
   -Dhbase.version=${PRODUCT} \
   -Dhbase-thirdparty.version=${HBASE_THIRDPARTY} \
   -DskipTests \
+  -Dmaven.test.skip=true \
   package assembly:single
 
 # We need the "*" here as the directory won't be the same as the final tar file for SNAPSHOTs which we currently have to use for 2.6
@@ -208,6 +214,11 @@ USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
 COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \
+  # The artifact name of the AWS bundle has changed between Haddop 3.3.6 and 3.4.1
+  # from aws-java-sdk-bundle-*.jar to bundle-*.jar.
+  # See: https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html
+  # So we try to copy both and if one of them doesn't exist buildx will just ignore it :)
+  /stackable/hadoop/share/hadoop/tools/lib/bundle-*.jar \
   /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-*.jar \
   /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar \
   /stackable/hadoop/share/hadoop/tools/lib/
@@ -248,7 +259,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PHOENIX}
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-RUN --mount=type=cache,id=maven-phoenix-${PHOENIX},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
+RUN --mount=type=cache,id=maven-phoenix-${PHOENIX}-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 cd "$(/stackable/patchable --images-repo-root=src checkout phoenix ${PHOENIX})"
 
 # The Maven command can be found inside of the scripts in the create-release folder (release-util.sh as of Phoenix 5.2.0)
@@ -260,6 +271,7 @@ mvn \
   -Dhbase.profile=${HBASE_PROFILE} \
   -Dhadoop.version=${HADOOP} \
   -DskipTests \
+  -Dmaven.test.skip=true \
   -Dcheckstyle.skip=true \
   clean \
   package
@@ -324,7 +336,6 @@ LABEL io.k8s.display-name="${NAME}"
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${PRODUCT} /stackable/hbase-${PRODUCT}/
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/async-profiler /stackable/async-profiler/
-COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/jmx /stackable/jmx/
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools-builder /stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS} /stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}/
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools-builder /stackable/bin/hbck2 /stackable/bin/hbck2
@@ -378,7 +389,6 @@ chmod g=u "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar"
 # fix missing permissions
 chmod g=u /stackable/async-profiler
 chmod g=u /stackable/bin
-chmod g=u /stackable/jmx
 chmod g=u /stackable/phoenix
 # the whole directory tree /stackable/hadoop/share/hadoop/tools/lib/ must be adapted
 find /stackable/hadoop -type d -exec chmod g=u {} +

hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/0002-Exclude-hbase-testing-utils-dependency-from-the-buil.patch

@@ -0,0 +1,56 @@
+From f9e952767faf3032c0d4e729e51b5901fcf1a0e1 Mon Sep 17 00:00:00 2001
+From: Razvan-Daniel Mihai <[email protected]>
+Date: Tue, 6 May 2025 17:58:34 +0200
+Subject: Exclude hbase-testing-utils dependency from the build.
+
+---
+ hbase-hbck2/pom.xml | 11 +++++++++--
+ hbase-tools/pom.xml |  5 +++--
+ 2 files changed, 12 insertions(+), 4 deletions(-)
+
+diff --git a/hbase-hbck2/pom.xml b/hbase-hbck2/pom.xml
+index 1d57225..fd0018c 100644
+--- a/hbase-hbck2/pom.xml
++++ b/hbase-hbck2/pom.xml
+@@ -114,12 +114,19 @@
+         </exclusion>
+       </exclusions>
+     </dependency>
+-    <dependency>
++    <!-- Starting with Hbase 2.6.2, this brings in a dependency that is not publicly available:
++        org.apache.directory.jdbm:apacheds-jdbm1:bundle:2.0.0-M2
++
++      Since we do not run the tests, we exclude this dependency.
++
++      To build successfully we also need to use -Dmaven.test.skip=true to skip building the tests.
++    -->
++    <!-- dependency>
+       <groupId>org.apache.hbase</groupId>
+       <artifactId>hbase-testing-util</artifactId>
+       <version>${hbase.version}</version>
+       <scope>test</scope>
+-    </dependency>
++    </dependency -->
+     <dependency>
+       <groupId>org.mockito</groupId>
+       <artifactId>mockito-core</artifactId>
+diff --git a/hbase-tools/pom.xml b/hbase-tools/pom.xml
+index 55ef075..eeebe14 100644
+--- a/hbase-tools/pom.xml
++++ b/hbase-tools/pom.xml
+@@ -103,12 +103,13 @@
+         </exclusion>
+       </exclusions>
+     </dependency>
+-    <dependency>
++    <!-- See the hbase-hbck2/pom.xml for an explanation of why this is commented out -->
++    <!-- dependency>
+       <groupId>org.apache.hbase</groupId>
+       <artifactId>hbase-testing-util</artifactId>
+       <version>${hbase.version}</version>
+       <scope>test</scope>
+-    </dependency>
++    </dependency -->
+     <dependency>
+       <groupId>org.mockito</groupId>
+       <artifactId>mockito-core</artifactId>

hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/0003-Configure-git-commit-id-plugin-to-use-native-git.patch

@@ -0,0 +1,29 @@
+From 187db522edc569def2aea3f9453ecb1a674592fe Mon Sep 17 00:00:00 2001
+From: Razvan-Daniel Mihai <[email protected]>
+Date: Wed, 7 May 2025 10:26:37 +0200
+Subject: Configure git-commit-id-plugin to use native git
+
+This fixes the problem that jgit cannot handle worktrees and the build
+fails.
+
+See
+https://github.com/git-commit-id/git-commit-id-maven-plugin/issues/215
+---
+ hbase-hbck2/pom.xml | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/hbase-hbck2/pom.xml b/hbase-hbck2/pom.xml
+index fd0018c..f00bd18 100644
+--- a/hbase-hbck2/pom.xml
++++ b/hbase-hbck2/pom.xml
+@@ -215,8 +215,9 @@
+       <plugin>
+         <groupId>pl.project13.maven</groupId>
+         <artifactId>git-commit-id-plugin</artifactId>
+-        <version>2.2.5</version>
++        <version>4.9.10</version>
+         <configuration>
++          <useNativeGit>true</useNativeGit>
+           <dotGitDirectory>${project.basedir}/.git</dotGitDirectory>
+           <commitIdGenerationMode>flat</commitIdGenerationMode>
+           <failOnNoGitDirectory>false</failOnNoGitDirectory>