Merge remote-tracking branch 'origin/main' into feat/opensearch

Author: labrenbe

.github/workflows/mirror.yaml

@@ -57,7 +57,7 @@ jobs:
           echo "IMAGE_REPOSITORY=$(.scripts/get_repo_name.sh)" | tee -a "$GITHUB_ENV"
 
       - name: Publish Container Image on oci.stackable.tech
-        uses: stackabletech/actions/publish-image@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2
+        uses: stackabletech/actions/publish-image@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$sdp+github-action-build
@@ -84,7 +84,7 @@ jobs:
           echo "IMAGE_REPOSITORY=$(.scripts/get_repo_name.sh)" | tee -a "$GITHUB_ENV"
 
       - name: Publish and Sign Image Index Manifest to oci.stackable.tech
-        uses: stackabletech/actions/publish-index-manifest@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2
+        uses: stackabletech/actions/publish-index-manifest@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$sdp+github-action-build

.github/workflows/pr_pre-commit.yaml

@@ -16,7 +16,7 @@ jobs:
         with:
           persist-credentials: false
           fetch-depth: 0
-      - uses: stackabletech/actions/run-pre-commit@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2
+      - uses: stackabletech/actions/run-pre-commit@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           hadolint: ${{ env.HADOLINT_VERSION }}

.github/workflows/reusable_build_image.yaml

@@ -27,7 +27,7 @@ jobs:
         with:
           persist-credentials: false
       - id: shard
-        uses: stackabletech/actions/shard@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2
+        uses: stackabletech/actions/shard@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3
         with:
           product-name: ${{ inputs.product-name }}
     outputs:
@@ -53,18 +53,18 @@ jobs:
           persist-credentials: false
 
       - name: Free Disk Space
-        uses: stackabletech/actions/free-disk-space@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2
+        uses: stackabletech/actions/free-disk-space@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3
 
       - name: Build Product Image
         id: build
-        uses: stackabletech/actions/build-product-image@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2
+        uses: stackabletech/actions/build-product-image@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3
         with:
           product-name: ${{ inputs.product-name }}
           product-version: ${{ matrix.versions }}
           sdp-version: ${{ inputs.sdp-version }}
 
       - name: Publish Container Image on oci.stackable.tech
-        uses: stackabletech/actions/publish-image@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2
+        uses: stackabletech/actions/publish-image@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$${{ inputs.registry-namespace }}+github-action-build
@@ -90,7 +90,7 @@ jobs:
           persist-credentials: false
 
       - name: Publish and Sign Image Index Manifest to oci.stackable.tech
-        uses: stackabletech/actions/publish-index-manifest@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2
+        uses: stackabletech/actions/publish-index-manifest@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$${{ inputs.registry-namespace }}+github-action-build

CHANGELOG.md

@@ -22,6 +22,7 @@ All notable changes to this project will be documented in this file.
 - spark-connect-client: A new image for Spark connect tests and demos ([#1034])
 - kafka: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1041]).
+- kafka: build kafka-opa-plugin from source ([#1177]).
 - nifi: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1027]).
 - nifi: Add [nifi-iceberg-bundle] for NiFi `2.4.0` ([#1060], [#1106]).
@@ -90,6 +91,7 @@ All notable changes to this project will be documented in this file.
 - opa: Enable custom versions ([#1170]).
 - use custom product versions for Hadoop, HBase, Phoenix, hbase-operator-tools, Druid, Hive and Spark ([#1173]).
 - hbase: Bump dependencies to the latest patch level for HBase `2.6.1` and `2.6.2` ([#1185]).
+- Changed default user & group IDs from 1000/1000 to 782252253/574654813 ([#1164])
 
 ### Fixed
 
@@ -112,6 +114,8 @@ All notable changes to this project will be documented in this file.
 - ubi9-rust-builder: Use pinned `rustup` version ([#1121]).
 - hive: Patch for postgres CVE-2024-1597 ([#1100]).
 - bump image-tools (for `bake`) and nixpkgs (for `nodejs_20`, used by pre-commit) ([#1100]).
+- bump image-tools (for `bake`) to fix `RELEASE` arg ([#1188]).
+- nifi: automatically determine NiFi version create reporting task script ([#1189]).
 
 ### Removed
 
@@ -131,6 +135,7 @@ All notable changes to this project will be documented in this file.
 - nifi: Remove `2.2.0` ([#1114]).
 - kafka: Remove `3.7.1` and `3.8.0` ([#1117]).
 - spark-connect-client: Remove `3.5.5` ([#1142]).
+- nifi: Enable custom versions ([#1172]).
 - kafka: Enable custom versions ([#1171]).
 - omid: Enable custom versions ([#1174]).
 
@@ -195,17 +200,21 @@ All notable changes to this project will be documented in this file.
 [#1152]: https://github.com/stackabletech/docker-images/pull/1152
 [#1156]: https://github.com/stackabletech/docker-images/pull/1156
 [#1163]: https://github.com/stackabletech/docker-images/pull/1163
+[#1164]: https://github.com/stackabletech/docker-images/pull/1164
 [#1165]: https://github.com/stackabletech/docker-images/pull/1165
 [#1168]: https://github.com/stackabletech/docker-images/pull/1168
 [#1169]: https://github.com/stackabletech/docker-images/pull/1169
 [#1170]: https://github.com/stackabletech/docker-images/pull/1170
 [#1171]: https://github.com/stackabletech/docker-images/pull/1171
 [#1173]: https://github.com/stackabletech/docker-images/pull/1173
 [#1174]: https://github.com/stackabletech/docker-images/pull/1174
+[#1177]: https://github.com/stackabletech/docker-images/pull/1177
 [#1179]: https://github.com/stackabletech/docker-images/pull/1179
 [#1180]: https://github.com/stackabletech/docker-images/pull/1180
 [#1184]: https://github.com/stackabletech/docker-images/pull/1184
 [#1185]: https://github.com/stackabletech/docker-images/pull/1185
+[#1188]: https://github.com/stackabletech/docker-images/pull/1188
+[#1189]: https://github.com/stackabletech/docker-images/pull/1189
 
 ## [25.3.0] - 2025-03-21
 

conf.py

@@ -40,6 +40,7 @@
 trino_storage_connector = importlib.import_module("trino.storage-connector.versions")
 kafka_testing_tools = importlib.import_module("kafka-testing-tools.versions")
 kcat = importlib.import_module("kafka.kcat.versions")
+kafka_opa_plugin = importlib.import_module("kafka.kafka-opa-plugin.versions")
 testing_tools = importlib.import_module("testing-tools.versions")
 zookeeper = importlib.import_module("zookeeper.versions")
 tools = importlib.import_module("tools.versions")
@@ -77,6 +78,7 @@
     {"name": "trino/storage-connector", "versions": trino_storage_connector.versions},
     {"name": "kafka-testing-tools", "versions": kafka_testing_tools.versions},
     {"name": "kafka/kcat", "versions": kcat.versions},
+    {"name": "kafka/kafka-opa-plugin", "versions": kafka_opa_plugin.versions},
     {"name": "testing-tools", "versions": testing_tools.versions},
     {"name": "zookeeper", "versions": zookeeper.versions},
     {"name": "tools", "versions": tools.versions},
@@ -112,7 +114,7 @@
 
 args = {
     "STACKABLE_USER_NAME": "stackable",
-    "STACKABLE_USER_UID": "1000",
-    "STACKABLE_USER_GID": "1000",
+    "STACKABLE_USER_UID": "782252253",  # This is a random high id to not conflict with any existing user
+    "STACKABLE_USER_GID": "574654813",  # This is a random high id to not conflict with any existing group
     "DELETE_CACHES": "true",
 }

druid/Dockerfile

@@ -41,7 +41,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackab
 
 COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 /stackable/patched-libs /stackable/patched-libs
 # Cache mounts are owned by root by default
-# We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base
+# We need to explicitly give the uid to use.
 # The cache id has to include the product version that we are building because otherwise
 # docker encounters race conditions when building multiple versions in parallel, as all
 # builder containers will share the same cache and the `rm -rf` commands will fail

java-devel/Dockerfile

@@ -23,40 +23,40 @@ gpgcheck=1
 gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public
 EOF
 
-RUN microdnf update && \
-    microdnf install -y \
-    cmake \
-    cyrus-sasl-devel \
-    # diff is required by maven during the build of hbase \
-    # Cannot run program "diff" (in directory "/stackable/hbase-2.4.12-src/hbase-shaded/hbase-shaded-check-invariants"
-    diffutils \
-    fuse-devel \
-    gcc \
-    gcc-c++ \
-    # The GNU gettext utilities contain the envsubst program which
-    # substitutes the values of environment variables.
-    gettext \
-    # For the apply_patches.sh script
-    git \
-    # Needed by the maven ant run plugin for the "set-hostname-property" step in zookeeper
-    hostname \
-    # Needed for compiling Java projects
-    "temurin-${PRODUCT}-jdk" \
-    krb5-devel \
-    libcurl-devel \
-    make \
-    maven \
-    openssl-devel \
-    # Required to unpack Omid tarball
-    tar \
-    wget \
-    which \
-    xz \
-    zlib-devel \
-    # Required for log4shell.sh
-    unzip zip && \
-    microdnf clean all && \
-    rm -rf /var/cache/yum
+RUN <<EOF
+microdnf update
+microdnf install \
+  cmake \
+  cyrus-sasl-devel \
+  `# diff is required by maven during the build of hbase` \
+  `# Cannot run program "diff" (in directory "/stackable/hbase-2.4.12-src/hbase-shaded/hbase-shaded-check-invariants"` \
+  diffutils \
+  fuse-devel \
+  gcc \
+  gcc-c++ \
+  `# The GNU gettext utilities contain the envsubst program which` \
+  `# substitutes the values of environment variables.` \
+  gettext \
+  `# For the apply_patches.sh script`\
+  git \
+  `# Needed by the maven ant run plugin for the "set-hostname-property" step in zookeeper` \
+  hostname \
+  `# Needed for compiling Java projects` \
+  "temurin-${PRODUCT}-jdk" \
+  krb5-devel \
+  libcurl-devel \
+  make \
+  maven \
+  openssl-devel \
+  `# Required to unpack Omid tarball` \
+  tar \
+  wget \
+  which \
+  xz \
+  zlib-devel
+microdnf clean all
+rm -rf /var/cache/yum
+EOF
 
 ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk"
 

kafka/Dockerfile

@@ -2,13 +2,13 @@
 # check=error=true
 
 FROM stackable/image/kafka/kcat AS kcat
+FROM stackable/image/kafka/kafka-opa-plugin AS kafka-opa-plugin
 
 FROM stackable/image/java-devel AS kafka-builder
 
 ARG PRODUCT
 ARG RELEASE
 ARG SCALA
-ARG OPA_AUTHORIZER
 ARG JMX_EXPORTER
 ARG STACKABLE_USER_UID
 
@@ -40,10 +40,6 @@ cp build/reports/bom.json /stackable/kafka_${SCALA}-${NEW_VERSION}.cdx.json
 rm -rf /stackable/kafka_${SCALA}-${NEW_VERSION}/site-docs/
 (cd .. && rm -rf ${PRODUCT})
 
-# TODO (@NickLarsenNZ): Compile from source: https://github.com/StyraInc/opa-kafka-plugin
-curl https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
-  -o /stackable/kafka_${SCALA}-${NEW_VERSION}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
-
 # JMX exporter
 curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
   -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar
@@ -60,6 +56,7 @@ ARG RELEASE
 ARG PRODUCT
 ARG SCALA
 ARG KAFKA_KCAT
+ARG KAFKA_KAFKA_OPA_PLUGIN
 ARG STACKABLE_USER_UID
 
 LABEL \
@@ -78,6 +75,8 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stack
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/bin/kcat-${KAFKA_KCAT}
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}-src.tar.gz /stackable
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-opa-plugin /stackable/src/kafka/kafka-opa-plugin/patchable-work/worktree/${KAFKA_KAFKA_OPA_PLUGIN}/build/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN}-all.jar /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN}-all.jar
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-opa-plugin /stackable/kafka-opa-plugin-${KAFKA_KAFKA_OPA_PLUGIN}-src.tar.gz /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 kafka/licenses /licenses
 
@@ -107,6 +106,7 @@ chown -h ${STACKABLE_USER_UID}:0 /stackable/kafka
 chmod g=u /stackable/bin
 chmod g=u /stackable/jmx
 chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}
+chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN}-all.jar
 chmod g=u /stackable/*-src.tar.gz
 EOF
 

kafka/kafka-opa-plugin/Dockerfile

@@ -0,0 +1,22 @@
+# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
+# check=error=true
+
+FROM stackable/image/java-devel
+
+ARG PRODUCT
+ARG STACKABLE_USER_UID
+
+USER ${STACKABLE_USER_UID}
+WORKDIR /stackable
+
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/patchable.toml /stackable/src/kafka/kafka-opa-plugin/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/${PRODUCT} /stackable/src/kafka/kafka-opa-plugin/stackable/patches/${PRODUCT}
+
+RUN <<EOF
+cd "$(/stackable/patchable --images-repo-root=src checkout kafka/kafka-opa-plugin ${PRODUCT})"
+
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/kafka-opa-plugin-${PRODUCT}-src.tar.gz .
+
+./gradlew clean shadowJar
+EOF

kafka/kafka-opa-plugin/stackable/patches/1.5.1/patchable.toml

@@ -0,0 +1,2 @@
+mirror = "https://github.com/stackabletech/opa-kafka-plugin"
+base = "d2c7851cb66dde7903eb4f0d5fab40f1a3d434a4"