Merge branch 'main' into chore/spark-1080

Author: razvan

.github/workflows/mirror.yaml

@@ -14,6 +14,7 @@ on:
           - registry.k8s.io/sig-storage/csi-node-driver-registrar
           - registry.k8s.io/sig-storage/csi-provisioner
           - registry.k8s.io/git-sync/git-sync
+          - registry-1.docker.io/library/golang
       image-index-manifest-tag:
         description: |
           The image index manifest tag, like 1.0.14 or v1.0.14

CHANGELOG.md

@@ -23,20 +23,25 @@ All notable changes to this project will be documented in this file.
   `check-permissions-ownership.sh` provided in stackable-base image ([#1027]).
 - opa: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1038]).
+- opa: Add `1.4.2` ([#1103]).
 - spark-k8s: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1055]).
 - superset: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1053]).
+- superset: Add version `4.1.2` ([#1102]).
 - trino: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1025]).
 - zookeeper: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1043]).
+- nifi: Add [nifi-iceberg-bundle](https://github.com/stackabletech/nifi-iceberg-bundle) for NiFi `2.2.0` ([#1060], [#1106]).
 - java: Add JDK 24 ([#1097]).
+- ci: Add golang image to mirror workflow ([#1103]).
 
 ### Changed
 
 - ubi-rust-builder: Bump Rust toolchain to 1.85.0, cargo-cyclonedx to 0.5.7, and cargo-auditable to 0.6.6 ([#1050]).
 - ubi9-rust-builder: Bump base image and update protoc to `30.2` ([#1091]).
+- stackable-devel: Bump ubi9 base image ([#1103]).
 - spark-k8s: Include spark-connect jars, replace OpenJDK with Temurin JDK, cleanup ([#1034]).
 - spark-connect-client: Image is now completely based on spark-k8s and includes JupyterLab and other demo dependencies ([#1071]).
 - jmx_exporter: Bump products to use `1.2.0` ([#1090]).
@@ -56,6 +61,7 @@ All notable changes to this project will be documented in this file.
 - Add `--locked` flag to `cargo install` commands for reproducible builds ([#1044]).
 - nifi: reduce docker image size by removing the recursive chown/chmods in the final image ([#1027]).
 - opa: reduce docker image size by removing the recursive chown/chmods in the final image ([#1038]).
+- opa: Manually install Go 1.23.9 ([#1103]).
 - spark-k8s: reduce docker image size by removing the recursive chown/chmods in the final image ([#1042]).
 - trino: reduce docker image size by removing the recursive chown/chmods in the final image ([#1025]).
 - zookeeper: reduce docker image size by removing the recursive chown/chmods in the final image ([#1043]).
@@ -68,6 +74,8 @@ All notable changes to this project will be documented in this file.
 - zookeeper: Remove 3.9.2 ([#1093]).
 - Remove ubi8-rust-builder image ([#1091]).
 - spark: remove 3.5.2 ([#1094])
+- opa: Remove `0.67.1` ([#1103]).
+- opa: Remove legacy bundle-builder from container build ([#1103]).
 
 [#1025]: https://github.com/stackabletech/docker-images/pull/1025
 [#1027]: https://github.com/stackabletech/docker-images/pull/1027
@@ -87,12 +95,16 @@ All notable changes to this project will be documented in this file.
 [#1054]: https://github.com/stackabletech/docker-images/pull/1054
 [#1055]: https://github.com/stackabletech/docker-images/pull/1055
 [#1056]: https://github.com/stackabletech/docker-images/pull/1056
+[#1060]: https://github.com/stackabletech/docker-images/pull/1060
 [#1090]: https://github.com/stackabletech/docker-images/pull/1090
 [#1091]: https://github.com/stackabletech/docker-images/pull/1091
 [#1093]: https://github.com/stackabletech/docker-images/pull/1093
 [#1094]: https://github.com/stackabletech/docker-images/pull/1094
 [#1097]: https://github.com/stackabletech/docker-images/pull/1097
 [#1098]: https://github.com/stackabletech/docker-images/pull/1098
+[#1102]: https://github.com/stackabletech/docker-images/pull/1102
+[#1103]: https://github.com/stackabletech/docker-images/pull/1103
+[#1106]: https://github.com/stackabletech/docker-images/pull/1106
 
 ## [25.3.0] - 2025-03-21
 

java-base/versions.py

@@ -25,6 +25,6 @@
     },
     {
         "product": "24",
-        "vector": "0.43.1",
+        "vector": "0.46.1",
     },
 ]

nifi/Dockerfile

@@ -63,6 +63,47 @@ rm -rf /stackable/nifi-${PRODUCT}/docs
 chmod -R g=u /stackable
 EOF
 
+FROM stackable/image/java-devel AS nifi-iceberg-bundle-builder
+
+ARG NIFI_ICEBERG_BUNDLE
+ARG PRODUCT
+ARG STACKABLE_USER_UID
+
+USER ${STACKABLE_USER_UID}
+WORKDIR /build
+
+RUN <<EOF
+mkdir -p /stackable
+
+# NiFI 1.x natively supports Iceberg, no need to build an iceberg-bundle for it
+if [[ "${PRODUCT}" != 1.* ]] ; then
+    curl "https://github.com/stackabletech/nifi-iceberg-bundle/archive/refs/tags/${NIFI_ICEBERG_BUNDLE}.tar.gz" | tar -xzC .
+    cd nifi-iceberg-bundle-${NIFI_ICEBERG_BUNDLE} || exit
+
+    mvn \
+        --batch-mode \
+        --no-transfer-progress\
+        clean package \
+        -D nifi.version=${PRODUCT} \
+        -Dmaven.javadoc.skip=true \
+        -Denforcer.skip=true
+    # We need "-Denforcer.skip=true", as the Maven version is too old
+
+    cp ./nifi-iceberg-services-api-nar/target/nifi-iceberg-services-api-nar-${NIFI_ICEBERG_BUNDLE}.nar /stackable
+    cp ./nifi-iceberg-services-nar/target/nifi-iceberg-services-nar-${NIFI_ICEBERG_BUNDLE}.nar /stackable
+    cp ./nifi-iceberg-processors-nar/target/nifi-iceberg-processors-nar-${NIFI_ICEBERG_BUNDLE}.nar /stackable
+    cp ./target/bom.json /stackable/nifi-iceberg-bundle.sbom.json
+
+    cd ..
+    # Save disk space, even for intermediate images
+    rm -rf nifi-iceberg-bundle-${NIFI_ICEBERG_BUNDLE}
+
+# Set correct groups
+chmod g=u /stackable/*.nar
+chmod g=u /stackable/*.sbom.json
+fi
+EOF
+
 FROM stackable/image/java-base AS final
 
 ARG PRODUCT
@@ -79,6 +120,8 @@ LABEL name="Apache NiFi" \
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/nifi-${PRODUCT} /stackable/nifi-${PRODUCT}/
 COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/stackable-bcrypt.jar /stackable/stackable-bcrypt.jar
+COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-iceberg-bundle-builder /stackable/*.nar /stackable/nifi-${PRODUCT}/lib/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-iceberg-bundle-builder /stackable/*.sbom.json /stackable/nifi-${PRODUCT}/lib/
 
 COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/bin /stackable/bin
 COPY --chown=${STACKABLE_USER_UID}:0 nifi/licenses /licenses

nifi/versions.py

@@ -13,5 +13,6 @@
         "product": "2.2.0",
         "java-base": "21",
         "java-devel": "21",
+        "nifi_iceberg_bundle": "0.0.3",
     },
 ]

opa/Dockerfile

@@ -1,39 +1,9 @@
 # syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5
-# check=error=true
+# check=error=true;skip=InvalidDefaultArgInFrom
 
-FROM stackable/image/stackable-base AS opa-bundle-builder
+ARG GOLANG
 
-ARG BUNDLE_BUILDER_VERSION
-
-# Update image and install everything needed for Rustup & Rust
-RUN <<EOF
-microdnf update
-microdnf install \
-  cmake \
-  gcc \
-  gcc-c++ \
-  git \
-  make \
-  openssl-devel \
-  pkg-config \
-  systemd-devel \
-  unzip
-rm -rf /var/cache/yum
-EOF
-
-WORKDIR /
-
-# WARNING (@NickLarsenNZ): We should pin the rustup version
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-RUN git clone --depth 1 --branch ${BUNDLE_BUILDER_VERSION} https://github.com/stackabletech/opa-bundle-builder
-RUN <<EOF
-cd ./opa-bundle-builder
-. "$HOME/.cargo/env"
-rustup toolchain install
-cargo --quiet build --release
-# set correct groups
-chmod -R g=u /opa-bundle-builder/target/release/
-EOF
+FROM oci.stackable.tech/sdp/library/golang:${GOLANG} AS golang-image
 
 FROM stackable/image/stackable-base AS multilog-builder
 
@@ -81,19 +51,21 @@ ENV GOOS=$TARGETOS
 
 # gzip, tar - used to unpack the OPA source
 # git - needed by the cyclonedx-gomod tool to determine the version of OPA
-# golang - used to build OPA
 RUN <<EOF
 microdnf update
 microdnf install \
   git \
-  golang \
   gzip \
   tar
 microdnf clean all
 EOF
 
 COPY --chown=${STACKABLE_USER_UID}:0 opa/stackable/bin /stackable/opa/bin
 
+# Manually install Go since the dnf package is sometimes not recent enough
+COPY --from=golang-image /usr/local/go/ /usr/local/go/
+ENV PATH="/usr/local/go/bin:${PATH}"
+
 RUN <<EOF
 # We use version 1.7.0, since a newer version of cyclonedx-gomod is not compatible with the version of Golang (>= 1.23.1)
 go install github.com/CycloneDX/cyclonedx-gomod/cmd/[email protected]
@@ -137,7 +109,6 @@ LABEL name="Open Policy Agent" \
 COPY --chown=${STACKABLE_USER_UID}:0 opa/licenses /licenses
 
 COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /stackable/opa /stackable/opa
-COPY --from=opa-bundle-builder --chown=${STACKABLE_USER_UID}:0 /opa-bundle-builder/target/release/stackable-opa-bundle-builder /stackable/opa-bundle-builder
 COPY --from=multilog-builder --chown=${STACKABLE_USER_UID}:0 /daemontools/admin/daemontools/command/multilog /stackable/multilog
 
 RUN <<EOF

opa/versions.py

@@ -1,14 +1,14 @@
 versions = [
     {
-        "product": "1.0.1",
+        "product": "1.4.2",
         "vector": "0.46.1",
-        "bundle_builder_version": "1.1.2",
+        "golang": "1.23.9",
         "stackable-base": "1.0.0",
     },
     {
-        "product": "0.67.1",
+        "product": "1.0.1",
         "vector": "0.46.1",
-        "bundle_builder_version": "1.1.2",
+        "golang": "1.23.9",
         "stackable-base": "1.0.0",
     },
 ]

stackable-devel/Dockerfile

@@ -8,7 +8,7 @@
 # Find the latest version at https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5?container-tabs=gti
 # IMPORTANT: Make sure to use the "Manifest List Digest" that references the images for multiple architectures
 # rather than just the "Image Digest" that references the image for the selected architecture.
-FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:c0e70387664f30cd9cf2795b547e4a9a51002c44a4a86aa9335ab030134bf392
+FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:e1c4703364c5cb58f5462575dc90345bcd934ddc45e6c32f9c162f2b5617681c
 
 # intentionally unused
 ARG PRODUCT