feat: move statsd_exporter, kcat and trino-storage-connector / separate nifi iceberg builder

Author: dervoeti

airflow/Dockerfile

@@ -7,8 +7,6 @@ ARG GIT_SYNC
 # which should contain a image location (e.g. registry.k8s.io/git-sync/git-sync:v3.6.8)
 FROM oci.stackable.tech/sdp/git-sync/git-sync:${GIT_SYNC} AS gitsync-image
 
-FROM stackable/image/statsd_exporter AS statsd_exporter-builder
-
 FROM python:3.12-bookworm AS opa-auth-manager-builder
 
 COPY airflow/opa-auth-manager/ /tmp/opa-auth-manager
@@ -25,7 +23,7 @@ EOF
 FROM stackable/image/vector AS airflow-build-image
 
 ARG PRODUCT
-ARG STATSD_EXPORTER
+ARG SHARED_STATSD_EXPORTER
 ARG PYTHON
 ARG TARGETARCH
 ARG STACKABLE_USER_UID
@@ -94,8 +92,8 @@ else
 end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT}.cdx.json
 EOF
 
-COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
-COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${STATSD_EXPORTER}.cdx.json
+COPY --from=stackable/image/shared/statsd-exporter /statsd_exporter/statsd_exporter /stackable/statsd_exporter
+COPY --from=stackable/image/shared/statsd-exporter /statsd_exporter/statsd_exporter-${SHARED_STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${SHARED_STATSD_EXPORTER}.cdx.json
 COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync
 
 RUN <<EOF

airflow/versions.py

@@ -5,7 +5,7 @@
         "git_sync": "v4.4.0",
         "s3fs": "2024.9.0",
         "cyclonedx_bom": "6.0.0",
-        "statsd_exporter": "0.28.0",
+        "shared/statsd-exporter": "0.28.0",
         "tini": "0.19.0",
         "vector": "0.46.1",
         "uv": "0.7.3",
@@ -16,7 +16,7 @@
         "git_sync": "v4.4.0",
         "s3fs": "2024.9.0",
         "cyclonedx_bom": "6.0.0",
-        "statsd_exporter": "0.28.0",
+        "shared/statsd-exporter": "0.28.0",
         "tini": "0.19.0",
         "vector": "0.46.1",
         "uv": "0.7.3",
@@ -27,7 +27,7 @@
         "git_sync": "v4.4.0",
         "s3fs": "2024.9.0",
         "cyclonedx_bom": "6.0.0",
-        "statsd_exporter": "0.28.0",
+        "shared/statsd-exporter": "0.28.0",
         "tini": "0.19.0",
         "vector": "0.46.1",
         "uv": "0.7.3",

conf.py

@@ -22,6 +22,7 @@
 krb5 = importlib.import_module("krb5.versions")
 vector = importlib.import_module("vector.versions")
 nifi = importlib.import_module("nifi.versions")
+nifi_iceberg_bundle = importlib.import_module("nifi.iceberg-bundle.versions")
 omid = importlib.import_module("omid.versions")
 opa = importlib.import_module("opa.versions")
 spark_k8s = importlib.import_module("spark-k8s.versions")
@@ -30,13 +31,13 @@
 superset = importlib.import_module("superset.versions")
 trino_cli = importlib.import_module("trino-cli.versions")
 trino = importlib.import_module("trino.versions")
-trino_storage_connector = importlib.import_module("trino-storage-connector.versions")
+trino_storage_connector = importlib.import_module("trino.storage-connector.versions")
 kafka_testing_tools = importlib.import_module("kafka-testing-tools.versions")
-kcat = importlib.import_module("kcat.versions")
+kcat = importlib.import_module("kafka.kcat.versions")
 testing_tools = importlib.import_module("testing-tools.versions")
 zookeeper = importlib.import_module("zookeeper.versions")
 tools = importlib.import_module("tools.versions")
-statsd_exporter = importlib.import_module("statsd_exporter.versions")
+statsd_exporter = importlib.import_module("shared.statsd-exporter.versions")
 spark_connect_client = importlib.import_module("spark-connect-client.versions")
 
 products = [
@@ -52,6 +53,7 @@
     {"name": "krb5", "versions": krb5.versions},
     {"name": "vector", "versions": vector.versions},
     {"name": "nifi", "versions": nifi.versions},
+    {"name": "nifi/iceberg-bundle", "versions": nifi_iceberg_bundle.versions},
     {"name": "omid", "versions": omid.versions},
     {"name": "opa", "versions": opa.versions},
     {"name": "spark-k8s", "versions": spark_k8s.versions},
@@ -60,13 +62,13 @@
     {"name": "superset", "versions": superset.versions},
     {"name": "trino-cli", "versions": trino_cli.versions},
     {"name": "trino", "versions": trino.versions},
-    {"name": "trino-storage-connector", "versions": trino_storage_connector.versions},
+    {"name": "trino/storage-connector", "versions": trino_storage_connector.versions},
     {"name": "kafka-testing-tools", "versions": kafka_testing_tools.versions},
-    {"name": "kcat", "versions": kcat.versions},
+    {"name": "kafka/kcat", "versions": kcat.versions},
     {"name": "testing-tools", "versions": testing_tools.versions},
     {"name": "zookeeper", "versions": zookeeper.versions},
     {"name": "tools", "versions": tools.versions},
-    {"name": "statsd_exporter", "versions": statsd_exporter.versions},
+    {"name": "shared/statsd-exporter", "versions": statsd_exporter.versions},
     {"name": "spark-connect-client", "versions": spark_connect_client.versions},
 ]
 

kafka-testing-tools/Dockerfile

@@ -1,12 +1,10 @@
 # syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5
 # check=error=true
 
-FROM stackable/image/kcat AS kcat
-
 FROM stackable/image/stackable-base AS final
 
 ARG PRODUCT
-ARG KCAT
+ARG KAFKA_KCAT
 ARG RELEASE
 ARG STACKABLE_USER_UID
 
@@ -30,9 +28,9 @@ RUN microdnf install  \
     && rm -rf /var/cache/yum
 
 # Store kcat version with binary name and add softlink
-COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT}
-RUN ln -s /stackable/kcat-${KCAT} /stackable/kcat
-COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 --from=stackable/image/kafka/kcat /stackable/kcat-${KAFKA_KCAT}/kcat /stackable/kcat-${KAFKA_KCAT}
+RUN ln -s /stackable/kcat-${KAFKA_KCAT} /stackable/kcat
+COPY --chown=${STACKABLE_USER_UID}:0 --from=stackable/image/kafka/kcat /licenses /licenses
 
 COPY --chown=${STACKABLE_USER_UID}:0 kafka-testing-tools/licenses /licenses
 

kafka-testing-tools/versions.py

@@ -1,7 +1,7 @@
 versions = [
     {
         "product": "1.0.0",
-        "kcat": "1.7.0",
+        "kafka/kcat": "1.7.0",
         "java-base": "11",
         "stackable-base": "1.0.0",
     }

kafka/Dockerfile

@@ -1,8 +1,6 @@
 # syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5
 # check=error=true
 
-FROM stackable/image/kcat AS kcat
-
 FROM stackable/image/java-devel AS kafka-builder
 
 ARG PRODUCT
@@ -50,7 +48,7 @@ FROM stackable/image/java-base AS final
 ARG RELEASE
 ARG PRODUCT
 ARG SCALA
-ARG KCAT
+ARG KAFKA_KCAT
 ARG STACKABLE_USER_UID
 
 LABEL \
@@ -65,8 +63,8 @@ LABEL \
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}/kafka_${SCALA}-${PRODUCT}.cdx.json
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stackable/jmx/
-COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT}
-COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 --from=stackable/image/kafka/kcat /stackable/kcat-${KAFKA_KCAT}/kcat /stackable/bin/kcat-${KAFKA_KCAT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=stackable/image/kafka/kcat /licenses /licenses
 
 COPY --chown=${STACKABLE_USER_UID}:0 kafka/licenses /licenses
 
@@ -84,7 +82,7 @@ chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt
 chmod g=u /stackable/package_manifest.txt
 rm -rf /var/cache/yum
 
-ln -s /stackable/bin/kcat-${KCAT} /stackable/bin/kcat
+ln -s /stackable/bin/kcat-${KAFKA_KCAT} /stackable/bin/kcat
 chown -h ${STACKABLE_USER_UID}:0 /stackable/bin/kcat
 # kcat was located in /stackable/kcat - legacy
 ln -s /stackable/bin/kcat /stackable/kcat