feat: include source code of components / migrate some components to patchable

Author: dervoeti

druid/Dockerfile

@@ -54,6 +54,9 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/sta
 cd "$(cat /tmp/DRUID_SOURCE_DIR)" || exit 1
 rm /tmp/DRUID_SOURCE_DIR
 
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/druid-${PRODUCT}-src.tar.gz .
+
 mvn \
   --batch-mode \
   --no-transfer-progress \
@@ -120,6 +123,7 @@ LABEL io.k8s.display-name="${NAME}"
 
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=druid-builder /stackable/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=druid-builder /stackable/druid-${PRODUCT}-src.tar.gz /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/bin /stackable/bin
 COPY --chown=${STACKABLE_USER_UID}:0 druid/licenses /licenses
@@ -141,7 +145,7 @@ chown -h ${STACKABLE_USER_UID}:0 /stackable/druid/bin/run-druid
 
 # fix missing permissions
 chmod -R g=u /stackable/bin
-chmod g=u /stackable/apache-druid-${PRODUCT}
+chmod g=u /stackable/apache-druid-${PRODUCT} /stackable/druid-${PRODUCT}-src.tar.gz
 EOF
 
 # ----------------------------------------

hadoop/Dockerfile

@@ -11,32 +11,40 @@ ARG TARGETARCH
 ARG TARGETOS
 ARG STACKABLE_USER_UID
 
+WORKDIR /stackable
+
+COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable.toml /stackable/src/shared/protobuf/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF}
+
+RUN <<EOF
+rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+microdnf update
+# boost is a build dependency starting in Hadoop 3.4.0 if compiling native code
+# automake and are required to build protobuf
+microdnf install boost1.78-devel automake libtool
+microdnf clean all
+rm -rf /var/cache/yum
+EOF
+
 # This Protobuf version is the exact version as used in the Hadoop Dockerfile
 # See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
 # (this was hardcoded in the Dockerfile in earlier versions of Hadoop, make sure to look at the exact version in Github)
-WORKDIR /opt/protobuf-src
 RUN <<EOF
-curl https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-${PROTOBUF}.tar.gz -o /opt/protobuf.tar.gz
-    tar xzf /opt/protobuf.tar.gz --strip-components 1 --no-same-owner
+    cd "$(/stackable/patchable --images-repo-root=src checkout shared/protobuf ${PROTOBUF})"
+
+    # Create snapshot of the source code including custom patches
+    tar -czf /stackable/protobuf-${PROTOBUF}-src.tar.gz .
+
+    ./autogen.sh
     ./configure --prefix=/opt/protobuf
     make "-j$(nproc)"
     make install
-    rm -rf /opt/protobuf-src
+    (cd .. && rm -r ${PROTOBUF})
 EOF
 
 ENV PROTOBUF_HOME=/opt/protobuf
 ENV PATH="${PATH}:/opt/protobuf/bin"
 
-RUN <<EOF
-rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
-microdnf update
-# boost is a build dependency starting in Hadoop 3.4.0 if compiling native code
-microdnf install boost1.78-devel
-microdnf clean all
-rm -rf /var/cache/yum
-EOF
-
-WORKDIR /stackable
 RUN <<EOF
 # async-profiler
 ARCH="${TARGETARCH/amd64/x64}"
@@ -63,6 +71,9 @@ COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx
 RUN <<EOF
 cd "$(/stackable/patchable --images-repo-root=src checkout hadoop ${PRODUCT})"
 
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/hadoop-${PRODUCT}-src.tar.gz .
+
 mvn \
     --batch-mode \
     --no-transfer-progress \
@@ -125,14 +136,20 @@ EOF
 ENV JAVA_HOME="/usr/lib/jvm/temurin-17-jdk"
 
 USER ${STACKABLE_USER_UID}
-WORKDIR /build
+WORKDIR /stackable
+
+COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/patchable.toml /stackable/src/hadoop/hdfs-utils/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} /stackable/src/hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS}
+
 # The Stackable HDFS utils contain an OPA authorizer, group mapper & topology provider.
 # The topology provider provides rack awareness functionality for HDFS by allowing users to specify Kubernetes
 # labels to build a rackID from.
 # Starting with hdfs-utils version 0.3.0 the topology provider is not a standalone jar anymore and included in hdfs-utils.
 RUN <<EOF
-curl "https://github.com/stackabletech/hdfs-utils/archive/refs/tags/v${HDFS_UTILS}.tar.gz" | tar -xzC .
-cd hdfs-utils-${HDFS_UTILS}
+cd "$(/stackable/patchable --images-repo-root=src checkout hadoop/hdfs-utils ${HDFS_UTILS})"
+
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/hdfs-utils-${HDFS_UTILS}-src.tar.gz .
 
 mvn \
     --batch-mode \
@@ -147,7 +164,7 @@ cp target/hdfs-utils-$HDFS_UTILS.jar /stackable/hdfs-utils-${HDFS_UTILS}.jar
 rm -rf hdfs-utils-main
 
 # Set correct groups
-chmod g=u /stackable/hdfs-utils-${HDFS_UTILS}.jar
+chmod g=u /stackable/hdfs-utils-${HDFS_UTILS}.jar /stackable/hdfs-utils-${HDFS_UTILS}-src.tar.gz
 EOF
 
 FROM stackable/image/java-base AS final
@@ -168,6 +185,7 @@ LABEL \
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable /stackable
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS}-src.tar.gz /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 hadoop/licenses /licenses
 

hadoop/hdfs-utils/stackable/patches/0.4.0/patchable.toml

@@ -0,0 +1 @@
+base = "cf24929fad3da02e530151d5ac66802cd5fa4109"

hadoop/hdfs-utils/stackable/patches/0.4.1/patchable.toml

@@ -0,0 +1 @@
+base = "fb9bd978a3c855984e93f1a6407c57d62fa34733"

hadoop/hdfs-utils/stackable/patches/patchable.toml

@@ -0,0 +1 @@
+upstream = "https://github.com/stackabletech/hdfs-utils.git"

hbase/Dockerfile

@@ -49,6 +49,9 @@ RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=${STACKABLE_USER_UID},targe
 ###
 cd "$(/stackable/patchable --images-repo-root=src checkout hbase ${PRODUCT})"
 
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/hbase-${PRODUCT}-src.tar.gz .
+
 # The release scripts of HBase also run the build twice (three times in fact, once again to build the site which we skip here).
 # I chose to replicate that exact behavior for consistency so please don't merge the two mvn runs into one unless you really know what you're doing!
 # Cannot skip building tests here because the assembly plugin needs a shell script from the test directory.
@@ -146,6 +149,8 @@ ARG RELEASE
 ARG HADOOP
 ARG HBASE_PROFILE
 ARG HBASE_HBASE_OPERATOR_TOOLS
+ARG HBASE_HBASE_OPA_AUTHORIZER
+ARG HBASE_PHOENIX
 ARG STACKABLE_USER_UID
 
 ARG NAME="Apache HBase"
@@ -171,13 +176,16 @@ LABEL io.k8s.description="${DESCRIPTION}"
 LABEL io.k8s.display-name="${NAME}"
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${PRODUCT} /stackable/hbase-${PRODUCT}/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${PRODUCT}-src.tar.gz /stackable
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/async-profiler /stackable/async-profiler/
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS} /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-src.tar.gz /stackable
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbck2 /stackable/bin/hbck2
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbase-entrypoint.sh /stackable/hbase-${PRODUCT}/bin/hbase-entrypoint.sh
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix /stackable/phoenix/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix-${HBASE_PHOENIX}-src.tar.gz /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/bin/export-snapshot-to-s3 /stackable/bin/export-snapshot-to-s3
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/hadoop/share/hadoop/tools/lib/ /stackable/hadoop/share/hadoop/tools/lib/
@@ -190,6 +198,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder \
   /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar \
   /stackable/hbase-${PRODUCT}/lib/
 
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer-${HBASE_HBASE_OPA_AUTHORIZER}-src.tar.gz /stackable
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer/target/hbase-opa-authorizer*.jar /stackable/hbase-${PRODUCT}/lib
 
 RUN <<EOF
@@ -226,6 +235,7 @@ chmod g=u "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar"
 chmod g=u /stackable/async-profiler
 chmod g=u /stackable/bin
 chmod g=u /stackable/phoenix
+chmod g=u /stackable/*-src.tar.gz
 # the whole directory tree /stackable/hadoop/share/hadoop/tools/lib/ must be adapted
 find /stackable/hadoop -type d -exec chmod g=u {} +
 EOF

hbase/hbase-opa-authorizer/Dockerfile

@@ -7,25 +7,34 @@ ARG STACKABLE_USER_UID
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/patchable.toml /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT} /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT}
+
 RUN --mount=type=cache,id=maven-opa,uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 ###
 ### OPA Authorizer (only for 2.6 upwards)
 ###
 if [[ -n "$PRODUCT" ]]; then
-    git clone --depth 1 --branch "$PRODUCT" https://github.com/stackabletech/hbase-opa-authorizer.git
+    cd "$(/stackable/patchable --images-repo-root=src checkout hbase/hbase-opa-authorizer ${PRODUCT})"
+
+    # Create snapshot of the source code including custom patches
+    tar -czf /stackable/hbase-opa-authorizer-${PRODUCT}-src.tar.gz .
     mvn \
       --batch-mode \
       --no-transfer-progress \
       -DskipTests \
       -Dmaven.test.skip=true \
-      -fhbase-opa-authorizer \
       package
 else
-  # Create a dummy jar to avoid errors when copying it the final image
-  mkdir -p hbase-opa-authorizer/target
-  touch hbase-opa-authorizer/target/hbase-opa-authorizer.jar
+  # Create a dummy jar to avoid errors when copying it to the final image
+  mkdir -p target
+  touch target/hbase-opa-authorizer.jar
+  touch /stackable/hbase-opa-authorizer-${PRODUCT}-src.tar.gz
 fi
 
+mkdir /stackable/hbase-opa-authorizer
+mv target /stackable/hbase-opa-authorizer
+
 if [ "${DELETE_CACHES}" = "true" ] ; then
   rm -rf /stackable/.m2/repository/*
 fi

hbase/hbase-opa-authorizer/stackable/patches/0.1.0/patchable.toml

@@ -0,0 +1 @@
+base = "c7eb27ca6a162bbfdb98262ba69e40d109f1fdd4"

hbase/hbase-opa-authorizer/stackable/patches/patchable.toml

@@ -0,0 +1 @@
+upstream = "https://github.com/stackabletech/hbase-opa-authorizer.git"

hbase/hbase-operator-tools/Dockerfile

@@ -14,8 +14,8 @@ ARG DELETE_CACHES="true"
 # so that they are not expanded. Disabling ShellCheck rules in a Dockerfile
 # does not work, so please ignore the according warning (SC2016).
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbck2.env /stackable/bin/
-COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/patchable.toml /stackable/src/hbase-operator-tools/stackable/patches/patchable.toml
-COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${PRODUCT} /stackable/src/hbase-operator-tools/stackable/patches/${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/patchable.toml /stackable/src/hbase/hbase-operator-tools/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${PRODUCT} /stackable/src/hbase/hbase-operator-tools/stackable/patches/${PRODUCT}
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbase-entrypoint.sh /stackable/bin/
 
 USER ${STACKABLE_USER_UID}
@@ -24,7 +24,10 @@ WORKDIR /stackable
 # Cache mounts are owned by root by default
 # We need to explicitly give the uid to use
 RUN --mount=type=cache,id=maven-hbase-operator-tools-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
-cd "$(/stackable/patchable --images-repo-root=src checkout hbase-operator-tools ${PRODUCT})"
+cd "$(/stackable/patchable --images-repo-root=src checkout hbase/hbase-operator-tools ${PRODUCT})"
+
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/hbase-operator-tools-$PRODUCT-src.tar.gz .
 
 mvn \
   --batch-mode \