Merge branch 'main' into chore/update-listener-classes

Author: sbernauer

.github/workflows/dev_nifi.yaml

@@ -25,5 +25,5 @@ jobs:
       image-name: nifi
       # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
       # the demo is reproducable for the release and it will be automatically replaced for the release branch.
-      image-version: 2.4.0-postgresql
+      image-version: 2.6.0-postgresql
       containerfile-path: demos/signal-processing/Dockerfile-nifi

demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.json

@@ -26,7 +26,7 @@ spec:
               echo 'Waiting for all nifi instances to be ready'
               kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=nifi,app.kubernetes.io/instance=nifi
         - name: wait-for-kafka-topics
-          image: oci.stackable.tech/sdp/kafka:3.9.1-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/kafka:4.1.0-stackable0.0.0-dev
           command:
             - bash
             - -euo

demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml

@@ -1,4 +1,4 @@
-FROM oci.stackable.tech/sdp/nifi:2.2.0-stackable0.0.0-dev
+FROM oci.stackable.tech/sdp/nifi:2.6.0-stackable0.0.0-dev
 
 # This is the postgresql JDBC driver from https://jdbc.postgresql.org/download/
 # There appear to be no signatures to validate against 😬
@@ -8,6 +8,6 @@ FROM oci.stackable.tech/sdp/nifi:2.2.0-stackable0.0.0-dev
 # curl --fail -u "your_username" --upload-file "postgresql-$VERSION.jar" 'https://repo.stackable.tech/repository/misc/jdbc/'
 # rm "postgresql-$VERSION.jar"
 
-# IMPORTANT (@NickLarsenNZ): Changing this version requires a change in the NiFi template (DownloadAndWriteToDB.xml)
+# IMPORTANT (@NickLarsenNZ): Changing this version requires a change in the NiFi template (DownloadAndWriteToDB.json)
 ARG DRIVER_VERSION="42.7.5"
 RUN curl --fail -o "/stackable/nifi/postgresql-$DRIVER_VERSION.jar" "https://repo.stackable.tech/repository/misc/jdbc/postgresql-$DRIVER_VERSION.jar"

demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.json

@@ -24,7 +24,7 @@
         "bundle": {
           "group": "org.apache.nifi",
           "artifact": "nifi-standard-nar",
-          "version": "2.2.0"
+          "version": "2.6.0"
         },
         "properties": {
           "Proxy Host": null,
@@ -337,7 +337,7 @@
         "bundle": {
           "group": "org.apache.nifi",
           "artifact": "nifi-standard-nar",
-          "version": "2.2.0"
+          "version": "2.6.0"
         },
         "properties": {
           "put-db-record-allow-multiple-statements": "false",
@@ -538,7 +538,7 @@
         "bundle": {
           "group": "org.apache.nifi",
           "artifact": "nifi-standard-nar",
-          "version": "2.2.0"
+          "version": "2.6.0"
         },
         "properties": {
           "Support Fragmented Transactions": "false",
@@ -729,7 +729,7 @@
         "bundle": {
           "group": "org.apache.nifi",
           "artifact": "nifi-dbcp-service-nar",
-          "version": "2.2.0"
+          "version": "2.6.0"
         },
         "properties": {
           "dbcp-min-idle-conns": "0",
@@ -873,7 +873,7 @@
             "bundle": {
               "group": "org.apache.nifi",
               "artifact": "nifi-standard-services-api-nar",
-              "version": "2.2.0"
+              "version": "2.6.0"
             }
           }
         ],
@@ -891,7 +891,7 @@
         "bundle": {
           "group": "org.apache.nifi",
           "artifact": "nifi-record-serialization-services-nar",
-          "version": "2.2.0"
+          "version": "2.6.0"
         },
         "properties": {
           "ignore-csv-header": "false",
@@ -1064,7 +1064,7 @@
             "bundle": {
               "group": "org.apache.nifi",
               "artifact": "nifi-standard-services-api-nar",
-              "version": "2.2.0"
+              "version": "2.6.0"
             }
           }
         ],

demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.json

@@ -26,16 +26,6 @@ To run this demo, your system needs at least:
 
 If you use MacOS or Windows and use Docker to run Kubernetes, set the RAM to at least 4 GB in _Preferences > Resources_.
 
-==== Linux
-
-OpenSearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts
-are likely too low - usually 65530, which may result in out-of-memory exceptions. So, the Linux setting
-`vm.max_map_count` on the host machine where the containers are running must be set to at least 262144.
-
-This is automatically set by default in this demo (via the `setSysctlMaxMapCount` Stack parameter).
-
-OpenSearch has more information about this setting in their https://opensearch.org/docs/2.12/install-and-configure/install-opensearch/index/#important-settings[documentation].
-
 == Overview
 
 This demo will
@@ -63,15 +53,16 @@ To list the installed Stackable services run the following command:
 [source,console]
 ----
 $ stackablectl stacklet list
-┌───────────────────────┬───────────────────────┬───────────┬─────────────────────────────────────────────────┬─────────────────────────────────┐
-│ PRODUCT               ┆ NAME                  ┆ NAMESPACE ┆ ENDPOINTS                                       ┆ CONDITIONS                      │
-╞═══════════════════════╪═══════════════════════╪═══════════╪═════════════════════════════════════════════════╪═════════════════════════════════╡
-│ zookeeper             ┆ simple-zk             ┆ default   ┆ server-zk                                       ┆ Available, Reconciling, Running │
-│                       ┆                       ┆           ┆ simple-zk-server.default.svc.cluster.local:2282 ┆                                 │
-├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
-│ opensearch-dashboards ┆ opensearch-dashboards ┆ default   ┆ http       http://172.18.0.2:31734              ┆                                 │
-│                       ┆                       ┆           ┆ metrics    172.18.0.2:32120                     ┆                                 │
-└───────────────────────┴───────────────────────┴───────────┴─────────────────────────────────────────────────┴─────────────────────────────────┘
+┌───────────────────────┬───────────────────────┬───────────┬────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐
+│ PRODUCT               ┆ NAME                  ┆ NAMESPACE ┆ ENDPOINTS                                                                          ┆ CONDITIONS                      │
+╞═══════════════════════╪═══════════════════════╪═══════════╪════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡
+│ opensearch            ┆ opensearch            ┆ default   ┆ nodes-default-http  http://opensearch-nodes-default.default.svc.cluster.local:9200 ┆ Available, Reconciling, Running │
+├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ zookeeper             ┆ simple-zk             ┆ default   ┆ server-zk           simple-zk-server.default.svc.cluster.local:2282                ┆ Available, Reconciling, Running │
+├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ opensearch-dashboards ┆ opensearch-dashboards ┆ default   ┆ http                http://172.18.0.2:30595                                        ┆                                 │
+│                       ┆                       ┆           ┆ metrics             172.18.0.2:31767                                               ┆                                 │
+└───────────────────────┴───────────────────────┴───────────┴────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘
 ----
 
 include::partial$instance-hint.adoc[]

demos/signal-processing/Dockerfile-nifi

@@ -86,103 +86,6 @@ $ stackablectl stacklet list
 
 include::partial$instance-hint.adoc[]
 
-== Inspect the data in Kafka
-
-Kafka is an event streaming platform to stream the data in near real-time.
-All the messages put in and read from Kafka are structured in dedicated queues called topics.
-The test data will be put into a topic called earthquakes.
-The records are produced (written) by the test data generator and consumed (read) by Druid afterwards in the same order they were created.
-
-As Kafka has no web interface, you must use a Kafka client like {kcat}[kcat].
-Kafka uses mutual TLS, so clients wanting to connect to Kafka must present a valid TLS certificate.
-The easiest way to obtain this is to shell into the `kafka-broker-default-0` Pod, as we will do in the following section for demonstration purposes.
-For a production setup, you should spin up a dedicated Pod provisioned with a certificate acting as a Kafka client instead of shell-ing into the Kafka Pod.
-
-=== List the available Topics
-
-You can execute a command on the Kafka broker to list the available topics as follows:
-
-// In the following commands the kcat-prober container instead of the kafka container is used to send requests to Kafka.
-// This is necessary because kcat cannot use key- and truststore files with empty passwords, which are mounted here to the kafka container.
-// However, the kcat-prober container has TLS certificates mounted, which can be used by kcat to connect to Kafka.
-[source,console]
-----
-$ kubectl exec -it kafka-broker-default-0 -c kcat-prober -- /bin/bash -c "/stackable/kcat -b localhost:9093 -X security.protocol=SSL -X ssl.key.location=/stackable/tls-kcat/tls.key -X ssl.certificate.location=/stackable/tls-kcat/tls.crt -X ssl.ca.location=/stackable/tls-kcat/ca.crt -L"
-Metadata for all topics (from broker -1: ssl://localhost:9093/bootstrap):
- 1 brokers:
-  broker 1001 at 172.19.0.4:32321 (controller)
- 1 topics:
-  topic "earthquakes" with 8 partitions:
-    partition 0, leader 1001, replicas: 1001, isrs: 1001
-    partition 1, leader 1001, replicas: 1001, isrs: 1001
-    partition 2, leader 1001, replicas: 1001, isrs: 1001
-    partition 3, leader 1001, replicas: 1001, isrs: 1001
-    partition 4, leader 1001, replicas: 1001, isrs: 1001
-    partition 5, leader 1001, replicas: 1001, isrs: 1001
-    partition 6, leader 1001, replicas: 1001, isrs: 1001
-    partition 7, leader 1001, replicas: 1001, isrs: 1001
-----
-
-You can see that Kafka consists of one broker, and the topic `earthquakes` with eight partitions has been created. To
-see some records sent to Kafka, run the following command. You can change the number of records to print via the `-c`
-parameter.
-
-[source,console]
-----
-$ kubectl exec -it kafka-broker-default-0 -c kcat-prober -- /bin/bash -c "/stackable/kcat -b localhost:9093 -X security.protocol=SSL -X ssl.key.location=/stackable/tls-kcat/tls.key -X ssl.certificate.location=/stackable/tls-kcat/tls.crt -X ssl.ca.location=/stackable/tls-kcat/ca.crt -C -t earthquakes -c 1"
-----
-
-Below is an example of the output of one record:
-
-[source,json]
-----
- {
-   "time":"1950-02-07T10:37:29.240Z",
-   "latitude":45.949,
-   "longitude":151.59,
-   "depth":35.0,
-   "mag":5.94,
-   "magType":"mw",
-   "nst":null,
-   "gap":null,
-   "dmin":null,
-   "rms":null,
-   "net":"iscgem",
-   "id":"iscgem895202",
-   "updated":"2022-04-26T18:23:38.377Z",
-   "place":"Kuril Islands",
-   "type":"earthquake",
-   "horizontalError":null,
-   "depthError":12.6,
-   "magError":0.55,
-   "magNst":null,
-   "status":"reviewed",
-   "locationSource":"iscgem",
-   "magSource":"iscgem"
-}
-----
-
-If you are interested in how many records have been produced to the Kafka topic so far, use the following command.
-It will print the last record produced to the topic partition, formatted with the pattern specified in the `-f` parameter.
-The given pattern will print some metadata of the record.
-
-[source,console]
-----
-$ kubectl exec -it kafka-broker-default-0 -c kcat-prober -- /bin/bash -c "/stackable/kcat -b localhost:9093 -X security.protocol=SSL -X ssl.key.location=/stackable/tls-kcat/tls.key -X ssl.certificate.location=/stackable/tls-kcat/tls.crt -X ssl.ca.location=/stackable/tls-kcat/ca.crt -C -t earthquakes -o -8 -c 8 -f 'Topic %t / Partition %p / Offset: %o / Timestamp: %T\n'"
-Topic earthquakes / Partition 0 / Offset: 378859 / Timestamp: 1752584024936
-Topic earthquakes / Partition 0 / Offset: 378860 / Timestamp: 1752584024936
-Topic earthquakes / Partition 0 / Offset: 378861 / Timestamp: 1752584024936
-Topic earthquakes / Partition 0 / Offset: 378862 / Timestamp: 1752584024936
-Topic earthquakes / Partition 0 / Offset: 378863 / Timestamp: 1752584024936
-Topic earthquakes / Partition 0 / Offset: 378864 / Timestamp: 1752584024936
-Topic earthquakes / Partition 0 / Offset: 378865 / Timestamp: 1752584024936
-Topic earthquakes / Partition 0 / Offset: 378866 / Timestamp: 1752584024936
-----
-
-If you calculate `379,000` records * `8` partitions, you end up with ~ 3,032,000 records.
-The output also shows that the last measurement record was produced at the timestamp `1752584024936`, which translates to `Tuesday, 15 July 2025 14:53:44.936 GMT+02:00`
-(using e.g. the command `date -d @1752584024`).
-
 == NiFi
 
 NiFi is used to fetch earthquake data from the internet and ingest it into Kafka.