Merge branch 'main' into feat/airflow-3-opa

Author: razvan

.scripts/upload_new_kcat_version.sh

@@ -46,6 +46,7 @@ All notable changes to this project will be documented in this file.
 - omid: bump version to 1.1.3 ([#1105])
 - hbase: add 2.6.2 and upgrade dependencies ([#1101])
 - kafka: Add `4.0.0` ([#1117])
+- Include `.tar.gz` snapshots of the product source code in container images ([#1126])
 
 ### Changed
 
@@ -145,6 +146,7 @@ All notable changes to this project will be documented in this file.
 [#1122]: https://github.com/stackabletech/docker-images/pull/1122
 [#1124]: https://github.com/stackabletech/docker-images/pull/1124
 [#1125]: https://github.com/stackabletech/docker-images/pull/1125
+[#1126]: https://github.com/stackabletech/docker-images/pull/1126
 [#1128]: https://github.com/stackabletech/docker-images/pull/1128
 
 ## [25.3.0] - 2025-03-21

CHANGELOG.md

@@ -54,6 +54,9 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/sta
 cd "$(cat /tmp/DRUID_SOURCE_DIR)" || exit 1
 rm /tmp/DRUID_SOURCE_DIR
 
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/druid-${PRODUCT}-src.tar.gz .
+
 mvn \
   --batch-mode \
   --no-transfer-progress \
@@ -120,6 +123,7 @@ LABEL io.k8s.display-name="${NAME}"
 
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=druid-builder /stackable/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=druid-builder /stackable/druid-${PRODUCT}-src.tar.gz /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/bin /stackable/bin
 COPY --chown=${STACKABLE_USER_UID}:0 druid/licenses /licenses

druid/Dockerfile

@@ -11,32 +11,40 @@ ARG TARGETARCH
 ARG TARGETOS
 ARG STACKABLE_USER_UID
 
+WORKDIR /stackable
+
+COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable.toml /stackable/src/shared/protobuf/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF}
+
+RUN <<EOF
+rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+microdnf update
+# boost is a build dependency starting in Hadoop 3.4.0 if compiling native code
+# automake and libtool are required to build protobuf
+microdnf install boost1.78-devel automake libtool
+microdnf clean all
+rm -rf /var/cache/yum
+EOF
+
 # This Protobuf version is the exact version as used in the Hadoop Dockerfile
 # See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
 # (this was hardcoded in the Dockerfile in earlier versions of Hadoop, make sure to look at the exact version in Github)
-WORKDIR /opt/protobuf-src
 RUN <<EOF
-curl https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-${PROTOBUF}.tar.gz -o /opt/protobuf.tar.gz
-    tar xzf /opt/protobuf.tar.gz --strip-components 1 --no-same-owner
+    cd "$(/stackable/patchable --images-repo-root=src checkout shared/protobuf ${PROTOBUF})"
+
+    # Create snapshot of the source code including custom patches
+    tar -czf /stackable/protobuf-${PROTOBUF}-src.tar.gz .
+
+    ./autogen.sh
     ./configure --prefix=/opt/protobuf
     make "-j$(nproc)"
     make install
-    rm -rf /opt/protobuf-src
+    (cd .. && rm -r ${PROTOBUF})
 EOF
 
 ENV PROTOBUF_HOME=/opt/protobuf
 ENV PATH="${PATH}:/opt/protobuf/bin"
 
-RUN <<EOF
-rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
-microdnf update
-# boost is a build dependency starting in Hadoop 3.4.0 if compiling native code
-microdnf install boost1.78-devel
-microdnf clean all
-rm -rf /var/cache/yum
-EOF
-
-WORKDIR /stackable
 RUN <<EOF
 # async-profiler
 ARCH="${TARGETARCH/amd64/x64}"
@@ -63,6 +71,9 @@ COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx
 RUN <<EOF
 cd "$(/stackable/patchable --images-repo-root=src checkout hadoop ${PRODUCT})"
 
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/hadoop-${PRODUCT}-src.tar.gz .
+
 mvn \
     --batch-mode \
     --no-transfer-progress \
@@ -125,14 +136,20 @@ EOF
 ENV JAVA_HOME="/usr/lib/jvm/temurin-17-jdk"
 
 USER ${STACKABLE_USER_UID}
-WORKDIR /build
+WORKDIR /stackable
+
+COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/patchable.toml /stackable/src/hadoop/hdfs-utils/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} /stackable/src/hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS}
+
 # The Stackable HDFS utils contain an OPA authorizer, group mapper & topology provider.
 # The topology provider provides rack awareness functionality for HDFS by allowing users to specify Kubernetes
 # labels to build a rackID from.
 # Starting with hdfs-utils version 0.3.0 the topology provider is not a standalone jar anymore and included in hdfs-utils.
 RUN <<EOF
-curl "https://github.com/stackabletech/hdfs-utils/archive/refs/tags/v${HDFS_UTILS}.tar.gz" | tar -xzC .
-cd hdfs-utils-${HDFS_UTILS}
+cd "$(/stackable/patchable --images-repo-root=src checkout hadoop/hdfs-utils ${HDFS_UTILS})"
+
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/hdfs-utils-${HDFS_UTILS}-src.tar.gz .
 
 mvn \
     --batch-mode \
@@ -147,7 +164,7 @@ cp target/hdfs-utils-$HDFS_UTILS.jar /stackable/hdfs-utils-${HDFS_UTILS}.jar
 rm -rf hdfs-utils-main
 
 # Set correct groups
-chmod g=u /stackable/hdfs-utils-${HDFS_UTILS}.jar
+chmod g=u /stackable/hdfs-utils-${HDFS_UTILS}.jar /stackable/hdfs-utils-${HDFS_UTILS}-src.tar.gz
 EOF
 
 FROM stackable/image/java-base AS final
@@ -168,6 +185,7 @@ LABEL \
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable /stackable
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS}-src.tar.gz /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 hadoop/licenses /licenses
 

druid/upload_new_druid_version.sh

@@ -0,0 +1 @@
+base = "cf24929fad3da02e530151d5ac66802cd5fa4109"

hadoop/Dockerfile

@@ -0,0 +1 @@
+base = "fb9bd978a3c855984e93f1a6407c57d62fa34733"

hadoop/hdfs-utils/stackable/patches/0.4.0/patchable.toml

@@ -0,0 +1 @@
+upstream = "https://github.com/stackabletech/hdfs-utils.git"