fix: hbase operator tools / trino 476 / copy libs fix

Author: dervoeti

druid/Dockerfile

@@ -65,7 +65,7 @@ NEW_VERSION="${PRODUCT}-stackable${RELEASE}"
 mvn versions:set -DnewVersion=$NEW_VERSION
 
 # Make Maven aware of custom Stackable libraries
-mv /stackable/patched-libs/maven /stackable/.m2/repository
+cp -r /stackable/patched-libs/maven/* /stackable/.m2/repository
 
 # Create snapshot of the source code including custom patches
 tar -czf /stackable/druid-${NEW_VERSION}-src.tar.gz .

hbase/hbase-operator-tools/Dockerfile

@@ -36,37 +36,41 @@ RUN --mount=type=cache,id=maven-hbase-operator-tools-${PRODUCT},uid=${STACKABLE_
 cd "$(/stackable/patchable --images-repo-root=src checkout hbase/hbase-operator-tools ${HBASE_OPERATOR_TOOLS_VERSION})"
 
 # Make Maven aware of custom Stackable libraries
-mv /stackable/patched-libs/maven /stackable/.m2/repository
+cp -r /stackable/patched-libs/maven/* /stackable/.m2/repository
 
 ORIGINAL_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
 NEW_VERSION="${HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE}"
+FULL_HBASE_OPERATOR_TOOLS_VERSION="${PRODUCT}-stackable${RELEASE}" # This includes the HBase version and the Stackable release suffix
+PATCHED_HBASE_VERSION="${HBASE_HBASE}-stackable${RELEASE}"
 
 mvn versions:set -DnewVersion=$NEW_VERSION
 
 # Create snapshot of the source code including custom patches
-tar -czf /stackable/hbase-operator-tools-${NEW_VERSION}-src.tar.gz .
+tar -czf /stackable/hbase-operator-tools-${FULL_HBASE_OPERATOR_TOOLS_VERSION}-src.tar.gz .
 
 mvn \
   --batch-mode \
   --no-transfer-progress \
-  -Dhbase.version=${HBASE_HBASE}-stackable${RELEASE} \
+  -Dhbase.version=${PATCHED_HBASE_VERSION} \
   -Dhbase-thirdparty.version=${HBASE_THIRDPARTY} \
   -DskipTests \
   -Dmaven.test.skip=true \
   package assembly:single
 
 # We need the "*" here as the directory won't be the same as the final tar file for SNAPSHOTs which we currently have to use for 2.6
 # And we're stripping the top level directory while extracting because it may be called different than the folder name when it's a SNAPSHOT
-mkdir /stackable/hbase-operator-tools-${NEW_VERSION}
+mkdir /stackable/hbase-operator-tools-${FULL_HBASE_OPERATOR_TOOLS_VERSION}
 tar -xz \
   -f hbase-operator-tools-assembly/target/hbase-operator-tools-*-bin.tar.gz \
-  -C /stackable/hbase-operator-tools-${NEW_VERSION}/ \
+  -C /stackable/hbase-operator-tools-${FULL_HBASE_OPERATOR_TOOLS_VERSION}/ \
   --strip-components=1
 sed -i "s/${NEW_VERSION}/${ORIGINAL_VERSION}/g" hbase-operator-tools-assembly/target/bom.json
-mv hbase-operator-tools-assembly/target/bom.json /stackable/hbase-operator-tools-${NEW_VERSION}/hbase-operator-tools-${NEW_VERSION}.cdx.json
+mv hbase-operator-tools-assembly/target/bom.json /stackable/hbase-operator-tools-${FULL_HBASE_OPERATOR_TOOLS_VERSION}/hbase-operator-tools-${FULL_HBASE_OPERATOR_TOOLS_VERSION}.cdx.json
 (cd .. && rm -r ${HBASE_OPERATOR_TOOLS_VERSION})
 
-envsubst '${HBASE_HBASE}:${HBASE_OPERATOR_TOOLS_VERSION}' < /stackable/bin/hbck2.env > /stackable/bin/hbck2
+PATCHED_HBASE_OPERATOR_TOOLS_VERSION=${NEW_VERSION}
+export PATCHED_HBASE_VERSION FULL_HBASE_OPERATOR_TOOLS_VERSION PATCHED_HBASE_OPERATOR_TOOLS_VERSION
+envsubst '${PATCHED_HBASE_VERSION}:${FULL_HBASE_OPERATOR_TOOLS_VERSION}:${PATCHED_HBASE_OPERATOR_TOOLS_VERSION}' < /stackable/bin/hbck2.env > /stackable/bin/hbck2
 chmod +x /stackable/bin/hbck2
 rm /stackable/bin/hbck2.env
 

hbase/hbase/Dockerfile

@@ -38,7 +38,7 @@ RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=${STACKABLE_USER_UID},targe
 cd "$(/stackable/patchable --images-repo-root=src checkout hbase/hbase ${PRODUCT})"
 
 # Make Maven aware of custom Stackable libraries
-mv /stackable/patched-libs/maven /stackable/.m2/repository
+cp -r /stackable/patched-libs/maven/* /stackable/.m2/repository
 
 ORIGINAL_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
 NEW_VERSION="${PRODUCT}-stackable${RELEASE}"

hbase/hbase/stackable/bin/hbck2.env

@@ -1,5 +1,5 @@
 #!/bin/sh
 
-exec /stackable/hbase-${PRODUCT}/bin/hbase hbck \
-  -j /stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}/hbase-hbck2/hbase-hbck2-${HBASE_OPERATOR_TOOLS}.jar \
+exec /stackable/hbase-${PATCHED_HBASE_VERSION}/bin/hbase hbck \
+  -j /stackable/hbase-operator-tools-${FULL_HBASE_OPERATOR_TOOLS_VERSION}/hbase-hbck2/hbase-hbck2-${PATCHED_HBASE_OPERATOR_TOOLS_VERSION}.jar \
   "$@"

hbase/phoenix/Dockerfile

@@ -28,7 +28,7 @@ WORKDIR /stackable
 RUN --mount=type=cache,id=maven-phoenix-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 
 # Make Maven aware of custom Stackable libraries
-mv /stackable/patched-libs/maven /stackable/.m2/repository
+cp -r /stackable/patched-libs/maven/* /stackable/.m2/repository
 
 cd "$(/stackable/patchable --images-repo-root=src checkout phoenix ${PHOENIX_VERSION})"
 
@@ -38,7 +38,7 @@ NEW_VERSION="${PHOENIX_VERSION}-stackable${RELEASE}"
 mvn versions:set -DnewVersion=$NEW_VERSION
 
 # Create snapshot of the source code including custom patches
-tar -czf /stackable/phoenix-${NEW_VERSION}-src.tar.gz .
+tar -czf /stackable/phoenix-${PRODUCT}-stackable${RELEASE}-src.tar.gz .
 
 # The Maven command can be found inside of the scripts in the create-release folder (release-util.sh as of Phoenix 5.2.0)
 # https://github.com/apache/phoenix/tree/5.2.0/dev/create-release
@@ -55,13 +55,14 @@ mvn \
 
 # We need the "*" here as the directory won't be the same as the final tar file for SNAPSHOTs which we currently have to use for 2.6
 # And we're stripping the top level directory while extracting because it may be called different than the folder name when it's a SNAPSHOT
-mkdir /stackable/phoenix-${HBASE_PROFILE}-${NEW_VERSION}-bin
-tar -xz -f phoenix-assembly/target/phoenix-hbase-*-bin.tar.gz -C /stackable/phoenix-${HBASE_PROFILE}-${NEW_VERSION}-bin/ --strip-components=1
-mv phoenix-assembly/target/bom.json /stackable/phoenix-${HBASE_PROFILE}-${NEW_VERSION}-bin/phoenix-${HBASE_PROFILE}-${NEW_VERSION}.cdx.json
+mkdir /stackable/phoenix-${HBASE_PROFILE}-${PRODUCT}-stackable${RELEASE}-bin
+tar -xz -f phoenix-assembly/target/phoenix-hbase-*-bin.tar.gz -C /stackable/phoenix-${HBASE_PROFILE}-${PRODUCT}-stackable${RELEASE}-bin/ --strip-components=1
+sed -i "s/${NEW_VERSION}/${ORIGINAL_VERSION}/g" phoenix-assembly/target/bom.json
+mv phoenix-assembly/target/bom.json /stackable/phoenix-${HBASE_PROFILE}-${PRODUCT}-stackable${RELEASE}-bin/phoenix-${HBASE_PROFILE}-${PRODUCT}-stackable${RELEASE}.cdx.json
 
 # Remove sources
 (cd .. && rm -r ${PHOENIX_VERSION})
-ln -s "/stackable/phoenix-${HBASE_PROFILE}-${NEW_VERSION}-bin" /stackable/phoenix
+ln -s "/stackable/phoenix-${HBASE_PROFILE}-${PRODUCT}-stackable${RELEASE}-bin" /stackable/phoenix
 
 # We're removing these to make the intermediate layer smaller
 # This can be necessary even though it's only a builder image because the GitHub Action Runners only have very limited space available

hbase/versions.py

@@ -4,7 +4,7 @@
     {
         "product": "2.6.1",
         "hbase/hbase": "2.6.1",
-        "hbase/hbase-operator-tools": "1.3.0-fd5a5fb",
+        "hbase/hbase-operator-tools": "1.3.0-fd5a5fb-hbase2.6.1",
         "hbase/phoenix": "5.2.1-hbase2.6.1",
         "hbase/hbase-opa-authorizer": "0.1.0",  # only for HBase 2.6.1
         "hadoop": "3.3.6",
@@ -16,8 +16,8 @@
     {
         "product": "2.6.2",
         "hbase/hbase": "2.6.2",
-        "hbase/hbase-operator-tools": "1.3.0-fd5a5fb",
-        "hbase/phoenix": "5.2.1-hbase2.6.1",
+        "hbase/hbase-operator-tools": "1.3.0-fd5a5fb-hbase2.6.2",
+        "hbase/phoenix": "5.2.1-hbase2.6.2",
         "hbase/hbase-opa-authorizer": "0.1.0",  # only for HBase 2.6.1
         "hadoop": "3.4.1",
         "java-base": "11",

hive/Dockerfile

@@ -43,7 +43,7 @@ BUILD_SRC_DIR="$(/stackable/patchable --images-repo-root=src checkout hive ${PRO
 cd "$BUILD_SRC_DIR"
 
 # Make Maven aware of custom Stackable libraries
-mv /stackable/patched-libs/maven /stackable/.m2/repository
+cp -r /stackable/patched-libs/maven/* /stackable/.m2/repository
 
 ORIGINAL_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
 NEW_VERSION="${PRODUCT}-stackable${RELEASE}"

trino/storage-connector/Dockerfile

@@ -18,11 +18,11 @@ USER ${STACKABLE_USER_UID}
 
 # adding a hadolint ignore for SC2215, due to https://github.com/hadolint/hadolint/issues/980
 # hadolint ignore=SC2215
-RUN --mount=type=cache,id=maven-${PRODUCT},target=/stackable/.m2/repository <<EOF
+RUN --mount=type=cache,id=maven-${PRODUCT},target=/stackable/.m2/repository,uid=${STACKABLE_USER_UID} <<EOF
 cd "$(/stackable/patchable --images-repo-root=src checkout trino-storage-connector ${PRODUCT})"
 
 # Make Maven aware of custom Stackable libraries
-cp -r /stackable/patched-libs/maven /stackable/.m2/repository
+cp -r /stackable/patched-libs/maven/* /stackable/.m2/repository
 
 ORIGINAL_VERSION="${PRODUCT}"
 NEW_VERSION="${PRODUCT}-stackable${RELEASE}"

trino/storage-connector/versions.py

@@ -11,6 +11,7 @@
     },
     {
         "product": "476",
+        "trino/trino": "476",
         "java-devel": "24",
     },
 ]

trino/trino/Dockerfile

@@ -1,6 +1,5 @@
 # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
 # check=error=true
-
 FROM stackable/image/java-devel AS trino-builder
 
 ARG PRODUCT
@@ -39,9 +38,22 @@ git config user.name "Fake commiter"
 git commit --allow-empty --message "Fake commit, so that we can create a tag"
 git tag ${NEW_VERSION}
 
+SKIP_PROJECTS="!docs"
+if [ "$PRODUCT" = "470" ] || [ "$PRODUCT" = "451" ]; then
+  SKIP_PROJECTS="$SKIP_PROJECTS,!core/trino-server-rpm"
+fi
+
 # We need to use ./mvnw instead of mvn to get a recent maven version (which is required to build Trino)
-# We have to set dep.presto-jdbc-under-test explicitly, otherwise trino-test-jdbc-compatibility-old-driver depends on the original Trino dependencies
-./mvnw --batch-mode --no-transfer-progress install -DskipTests -Ddep.presto-jdbc-under-test=${NEW_VERSION} --projects="!docs,!core/trino-server-rpm"
+./mvnw \
+  --batch-mode \
+  --no-transfer-progress \
+  install \
+  `# -Dmaven.test.skip # Unable to skip test compilation without an unused dependency error for software.amazon.awssdk:identity-spi` \
+  -DskipTests `# Skip test execution` \
+  -Dcheckstyle.skip `# Skip checkstyle checks. We dont care if the code is properly formatted, it just wastes time` \
+  -Dmaven.javadoc.skip=true `# Dont generate javadoc` \
+  -Ddep.presto-jdbc-under-test=${NEW_VERSION} \
+  --projects="$SKIP_PROJECTS"
 
 mkdir -p /stackable/patched-libs/maven/io
 cp -r /root/.m2/repository/io/trino /stackable/patched-libs/maven/io