diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0497f782..1ec44e88 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ All notable changes to this project will be documented in this file.
- Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept.
- Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation.
- Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`.
+- Add test for Apache Iceberg integration ([#785]).
### Changed
@@ -39,6 +40,7 @@ All notable changes to this project will be documented in this file.
[#774]: https://github.com/stackabletech/nifi-operator/pull/774
[#776]: https://github.com/stackabletech/nifi-operator/pull/776
[#782]: https://github.com/stackabletech/nifi-operator/pull/782
+[#785]: https://github.com/stackabletech/nifi-operator/pull/785
[#787]: https://github.com/stackabletech/nifi-operator/pull/787
[#789]: https://github.com/stackabletech/nifi-operator/pull/789
diff --git a/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc b/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc
index a026a5b6..13048014 100644
--- a/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc
+++ b/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc
@@ -2,13 +2,27 @@
:description: Write to Apache Iceberg tables in NiFi using the PutIceberg processor. Supports integration with S3 and Hive Metastore for scalable data handling.
:iceberg: https://iceberg.apache.org/
-WARNING: In NiFi `2.0.0` Iceberg support https://issues.apache.org/jira/browse/NIFI-13938[has been removed].
-
{iceberg}[Apache Iceberg] is a high-performance format for huge analytic tables.
Iceberg brings the reliability and simplicity of SQL tables to big data, while making it possible for engines like Spark, Trino, Flink, Presto, Hive and Impala to safely work with the same tables, at the same time.
NiFi supports a `PutIceberg` processor to add rows to an existing Iceberg table https://issues.apache.org/jira/browse/NIFI-10442[starting from version 1.19.0].
-As of NiFi version `1.23.1` only `PutIceberg` is supported, you need to create and compact your tables with other tools such as Trino or Spark (both included in the Stackable Data Platform).
+As of NiFi version `2.4.0` only `PutIceberg` is supported, you need to create and compact your tables with other tools such as Trino or Spark (both included in the Stackable Data Platform).
+
+== NiFi 2
+
+In NiFi `2.0.0` Iceberg support https://issues.apache.org/jira/browse/NIFI-13938[has been removed] from upstream NiFi.
+
+We forked the `nifi-iceberg-bundle` and made it available at https://github.com/stackabletech/nifi-iceberg-bundle.
+Starting with SDP 25.7, we have added the necessary bundle to NiFi by default, you don't need to explicitly add Iceberg support to the Stackable NiFi.
+
+Please read on https://github.com/stackabletech/nifi-iceberg-bundle[its documentation] on how to ingest data into Iceberg tables.
+You don't need any special configs on the `NiFiCluster` in case you are using S3 and no Kerberos.
+
+HDFS and Kerberos are also supported, please have a look at the https://github.com/stackabletech/nifi-operator/tree/main/tests/templates/kuttl/iceberg[Iceberg integration test] for that.
+
+== NiFi 1
+
+Starting with `1.19.0`, NiFi supports writing to Iceberg tables.
The following example shows an example NiFi setup using the Iceberg integration.
diff --git a/tests/release.yaml b/tests/release.yaml
index b6ecf959..04ee6498 100644
--- a/tests/release.yaml
+++ b/tests/release.yaml
@@ -12,7 +12,15 @@ releases:
operatorVersion: 0.0.0-dev
listener:
operatorVersion: 0.0.0-dev
+ opa:
+ operatorVersion: 0.0.0-dev
zookeeper:
operatorVersion: 0.0.0-dev
+ hdfs:
+ operatorVersion: 0.0.0-dev
+ hive:
+ operatorVersion: 0.0.0-dev
+ trino:
+ operatorVersion: 0.0.0-dev
nifi:
operatorVersion: 0.0.0-dev
diff --git a/tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2 b/tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2
new file mode 100644
index 00000000..67185acf
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2
@@ -0,0 +1,9 @@
+{% if test_scenario['values']['openshift'] == 'true' %}
+# see https://github.com/stackabletech/issues/issues/566
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+ - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
+ timeout: 120
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/00-rbac.yaml.j2 b/tests/templates/kuttl/iceberg/00-rbac.yaml.j2
new file mode 100644
index 00000000..7ee61d23
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/00-rbac.yaml.j2
@@ -0,0 +1,29 @@
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: test-role
+rules:
+{% if test_scenario['values']['openshift'] == "true" %}
+ - apiGroups: ["security.openshift.io"]
+ resources: ["securitycontextconstraints"]
+ resourceNames: ["privileged"]
+ verbs: ["use"]
+{% endif %}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: test-sa
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: test-rb
+subjects:
+ - kind: ServiceAccount
+ name: test-sa
+roleRef:
+ kind: Role
+ name: test-role
+ apiGroup: rbac.authorization.k8s.io
diff --git a/tests/templates/kuttl/iceberg/01-create-s3-connection.yaml b/tests/templates/kuttl/iceberg/01-create-s3-connection.yaml
new file mode 100644
index 00000000..26f46c12
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/01-create-s3-connection.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+ - script: envsubst '$NAMESPACE' < 01_s3-connection.yaml | kubectl apply -n $NAMESPACE -f -
diff --git a/tests/templates/kuttl/iceberg/01_s3-connection.yaml b/tests/templates/kuttl/iceberg/01_s3-connection.yaml
new file mode 100644
index 00000000..56c30d36
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/01_s3-connection.yaml
@@ -0,0 +1,36 @@
+---
+apiVersion: s3.stackable.tech/v1alpha1
+kind: S3Connection
+metadata:
+ name: minio
+spec:
+ host: "minio.${NAMESPACE}.svc.cluster.local"
+ port: 9000
+ accessStyle: Path
+ credentials:
+ secretClass: s3-credentials-class
+ tls:
+ verification:
+ server:
+ caCert:
+ secretClass: tls
+---
+apiVersion: secrets.stackable.tech/v1alpha1
+kind: SecretClass
+metadata:
+ name: s3-credentials-class
+spec:
+ backend:
+ k8sSearch:
+ searchNamespace:
+ pod: {}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: minio-credentials
+ labels:
+ secrets.stackable.tech/class: s3-credentials-class
+stringData:
+ accessKey: admin
+ secretKey: adminadmin
diff --git a/tests/templates/kuttl/iceberg/02-assert.yaml.j2 b/tests/templates/kuttl/iceberg/02-assert.yaml.j2
new file mode 100644
index 00000000..9d648d0b
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/02-assert.yaml.j2
@@ -0,0 +1,14 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 300
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: krb5-kdc
+status:
+ readyReplicas: 1
+ replicas: 1
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2 b/tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2
new file mode 100644
index 00000000..15b1e04c
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2
@@ -0,0 +1,146 @@
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: krb5-kdc
+spec:
+ selector:
+ matchLabels:
+ app: krb5-kdc
+ template:
+ metadata:
+ labels:
+ app: krb5-kdc
+ spec:
+ serviceAccountName: test-sa
+ initContainers:
+ - name: init
+ image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev
+ args:
+ - sh
+ - -euo
+ - pipefail
+ - -c
+ - |
+ test -e /var/kerberos/krb5kdc/principal || kdb5_util create -s -P asdf
+ kadmin.local get_principal -terse root/admin || kadmin.local add_principal -pw asdf root/admin
+ # stackable-secret-operator principal must match the keytab specified in the SecretClass
+ kadmin.local get_principal -terse stackable-secret-operator || kadmin.local add_principal -e aes256-cts-hmac-sha384-192:normal -pw asdf stackable-secret-operator
+ env:
+ - name: KRB5_CONFIG
+ value: /stackable/config/krb5.conf
+ volumeMounts:
+ - mountPath: /stackable/config
+ name: config
+ - mountPath: /var/kerberos/krb5kdc
+ name: data
+ containers:
+ - name: kdc
+ image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev
+ args:
+ - krb5kdc
+ - -n
+ env:
+ - name: KRB5_CONFIG
+ value: /stackable/config/krb5.conf
+ volumeMounts:
+ - mountPath: /stackable/config
+ name: config
+ - mountPath: /var/kerberos/krb5kdc
+ name: data
+# Root permissions required on Openshift to access internal ports
+{% if test_scenario['values']['openshift'] == "true" %}
+ securityContext:
+ runAsUser: 0
+{% endif %}
+ - name: kadmind
+ image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev
+ args:
+ - kadmind
+ - -nofork
+ env:
+ - name: KRB5_CONFIG
+ value: /stackable/config/krb5.conf
+ volumeMounts:
+ - mountPath: /stackable/config
+ name: config
+ - mountPath: /var/kerberos/krb5kdc
+ name: data
+# Root permissions required on Openshift to access internal ports
+{% if test_scenario['values']['openshift'] == "true" %}
+ securityContext:
+ runAsUser: 0
+{% endif %}
+ - name: client
+ image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev
+ tty: true
+ stdin: true
+ env:
+ - name: KRB5_CONFIG
+ value: /stackable/config/krb5.conf
+ volumeMounts:
+ - mountPath: /stackable/config
+ name: config
+ volumes:
+ - name: config
+ configMap:
+ name: krb5-kdc
+ volumeClaimTemplates:
+ - metadata:
+ name: data
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: krb5-kdc
+spec:
+ selector:
+ app: krb5-kdc
+ ports:
+ - name: kadmin
+ port: 749
+ - name: kdc
+ port: 88
+ - name: kdc-udp
+ port: 88
+ protocol: UDP
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: krb5-kdc
+data:
+ krb5.conf: |
+ [logging]
+ default = STDERR
+ kdc = STDERR
+ admin_server = STDERR
+ # default = FILE:/var/log/krb5libs.log
+ # kdc = FILE:/var/log/krb5kdc.log
+ # admin_server = FILE:/vaggr/log/kadmind.log
+ [libdefaults]
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = {{ test_scenario['values']['kerberos-realm'] }}
+ spake_preauth_groups = edwards25519
+ [realms]
+ {{ test_scenario['values']['kerberos-realm'] }} = {
+ acl_file = /stackable/config/kadm5.acl
+ disable_encrypted_timestamp = false
+ }
+ [domain_realm]
+ .cluster.local = {{ test_scenario['values']['kerberos-realm'] }}
+ cluster.local = {{ test_scenario['values']['kerberos-realm'] }}
+ kadm5.acl: |
+ root/admin *e
+ stackable-secret-operator *e
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2
new file mode 100644
index 00000000..d3061645
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2
@@ -0,0 +1,8 @@
+---
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+ # We need to replace $NAMESPACE (by KUTTL)
+ - script: envsubst '$NAMESPACE' < 03_kerberos-secretclass.yaml | kubectl apply -n $NAMESPACE -f -
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2
new file mode 100644
index 00000000..e5033462
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2
@@ -0,0 +1,33 @@
+apiVersion: secrets.stackable.tech/v1alpha1
+kind: SecretClass
+metadata:
+ name: kerberos-$NAMESPACE
+spec:
+ backend:
+ kerberosKeytab:
+ realmName: {{ test_scenario['values']['kerberos-realm'] }}
+ kdc: krb5-kdc.$NAMESPACE.svc.cluster.local
+ admin:
+ mit:
+ kadminServer: krb5-kdc.$NAMESPACE.svc.cluster.local
+ adminKeytabSecret:
+ namespace: $NAMESPACE
+ name: secret-operator-keytab
+ adminPrincipal: stackable-secret-operator
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: secret-operator-keytab
+data:
+ # To create keytab. When promted enter password asdf
+ # cat | ktutil << 'EOF'
+ # list
+ # add_entry -password -p stackable-secret-operator@CLUSTER.LOCAL -k 1 -e aes256-cts-hmac-sha384-192
+ # wkt /tmp/keytab
+ # EOF
+{% if test_scenario['values']['kerberos-realm'] == 'CLUSTER.LOCAL' %}
+ keytab: BQIAAABdAAEADUNMVVNURVIuTE9DQUwAGXN0YWNrYWJsZS1zZWNyZXQtb3BlcmF0b3IAAAABZAYWIgEAFAAgm8MCZ8B//XF1tH92GciD6/usWUNAmBTZnZQxLua2TkgAAAAB
+{% elif test_scenario['values']['kerberos-realm'] == 'PROD.MYCORP' %}
+ keytab: BQIAAABbAAEAC1BST0QuTVlDT1JQABlzdGFja2FibGUtc2VjcmV0LW9wZXJhdG9yAAAAAWQZa0EBABQAIC/EnFNejq/K5lX6tX+B3/tkI13TCzkPB7d2ggCIEzE8AAAAAQ==
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/10-assert.yaml.j2 b/tests/templates/kuttl/iceberg/10-assert.yaml.j2
new file mode 100644
index 00000000..50b1d4c3
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/10-assert.yaml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: vector-aggregator-discovery
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2
new file mode 100644
index 00000000..2d6a0df5
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2
@@ -0,0 +1,9 @@
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: vector-aggregator-discovery
+data:
+ ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/20-assert.yaml b/tests/templates/kuttl/iceberg/20-assert.yaml
new file mode 100644
index 00000000..e1829b77
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/20-assert.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: minio
+status:
+ readyReplicas: 1
+ replicas: 1
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: minio-post-job
+status:
+ succeeded: 1
diff --git a/tests/templates/kuttl/iceberg/20-install-minio.yaml b/tests/templates/kuttl/iceberg/20-install-minio.yaml
new file mode 100644
index 00000000..bbef7238
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/20-install-minio.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+ - script: kubectl -n $NAMESPACE apply -f https://raw.githubusercontent.com/stackabletech/demos/refs/heads/release-25.3/stacks/_templates/minio-tls/rendered-chart.yaml
diff --git a/tests/templates/kuttl/iceberg/21-assert.yaml b/tests/templates/kuttl/iceberg/21-assert.yaml
new file mode 100644
index 00000000..1ac12423
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/21-assert.yaml
@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: postgresql
+status:
+ readyReplicas: 1
+ replicas: 1
diff --git a/tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml b/tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml
new file mode 100644
index 00000000..fa5698e0
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml
@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 300
+commands:
+ - script: >-
+ helm upgrade postgresql
+ --install
+ --version=12.5.6
+ --namespace $NAMESPACE
+ -f 21_helm-bitnami-postgresql-values.yaml
+ --repo https://charts.bitnami.com/bitnami postgresql
diff --git a/tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2
new file mode 100644
index 00000000..2526c3df
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2
@@ -0,0 +1,29 @@
+---
+volumePermissions:
+ enabled: false
+ securityContext:
+ runAsUser: auto
+
+primary:
+ extendedConfiguration: |
+ password_encryption=md5
+ podSecurityContext:
+{% if test_scenario['values']['openshift'] == 'true' %}
+ enabled: false
+{% else %}
+ enabled: true
+{% endif %}
+ containerSecurityContext:
+ enabled: false
+ resources:
+ requests:
+ memory: "512Mi"
+ cpu: "512m"
+ limits:
+ memory: "512Mi"
+ cpu: "1"
+
+auth:
+ username: hive
+ password: hive
+ database: hive
diff --git a/tests/templates/kuttl/iceberg/30-assert.yaml b/tests/templates/kuttl/iceberg/30-assert.yaml
new file mode 100644
index 00000000..49ba7437
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/30-assert.yaml
@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: zookeeper-server-default
+status:
+ readyReplicas: 1
+ replicas: 1
diff --git a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2
new file mode 100644
index 00000000..58440246
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2
@@ -0,0 +1,21 @@
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperCluster
+metadata:
+ name: zookeeper
+spec:
+ image:
+ productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}"
+ pullPolicy: IfNotPresent
+ clusterConfig:
+ listenerClass: {{ test_scenario['values']['listener-class'] }}
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+ vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+ servers:
+ config:
+ logging:
+ enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+ roleGroups:
+ default:
+ replicas: 1
diff --git a/tests/templates/kuttl/iceberg/31-opa.yaml.j2 b/tests/templates/kuttl/iceberg/31-opa.yaml.j2
new file mode 100644
index 00000000..179bfbc2
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/31-opa.yaml.j2
@@ -0,0 +1,17 @@
+---
+apiVersion: opa.stackable.tech/v1alpha1
+kind: OpaCluster
+metadata:
+ name: opa
+spec:
+ image:
+{% if test_scenario['values']['opa-l'].find(",") > 0 %}
+ custom: "{{ test_scenario['values']['opa-l'].split(',')[1] }}"
+ productVersion: "{{ test_scenario['values']['opa-l'].split(',')[0] }}"
+{% else %}
+ productVersion: "{{ test_scenario['values']['opa-l'] }}"
+{% endif %}
+ servers:
+ roleGroups:
+ default:
+ replicas: 1
diff --git a/tests/templates/kuttl/iceberg/32-assert.yaml b/tests/templates/kuttl/iceberg/32-assert.yaml
new file mode 100644
index 00000000..126b5639
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/32-assert.yaml
@@ -0,0 +1,28 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 900
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: hdfs-namenode-default
+status:
+ readyReplicas: 2
+ replicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: hdfs-journalnode-default
+status:
+ readyReplicas: 1
+ replicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: hdfs-datanode-default
+status:
+ readyReplicas: 1
+ replicas: 1
diff --git a/tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2 b/tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2
new file mode 100644
index 00000000..9bbe6c50
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2
@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+ # We need to replace $NAMESPACE (by KUTTL)
+ - script: envsubst '$NAMESPACE' < 32_hdfs.yaml | kubectl apply -n $NAMESPACE -f -
diff --git a/tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 b/tests/templates/kuttl/iceberg/32_hdfs.yaml.j2
new file mode 100644
index 00000000..7a23c747
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/32_hdfs.yaml.j2
@@ -0,0 +1,78 @@
+---
+apiVersion: hdfs.stackable.tech/v1alpha1
+kind: HdfsCluster
+metadata:
+ name: hdfs
+spec:
+ image:
+{% if test_scenario['values']['hdfs-l'].find(",") > 0 %}
+ custom: "{{ test_scenario['values']['hdfs-l'].split(',')[1] }}"
+ productVersion: "{{ test_scenario['values']['hdfs-l'].split(',')[0] }}"
+{% else %}
+ productVersion: "{{ test_scenario['values']['hdfs-l'] }}"
+{% endif %}
+ pullPolicy: IfNotPresent
+ clusterConfig:
+ dfsReplication: 1
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+ authentication:
+ tlsSecretClass: tls
+ kerberos:
+ secretClass: kerberos-$NAMESPACE
+ authorization:
+ opa:
+ configMapName: opa
+ package: hdfs
+{% endif %}
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+ vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+ zookeeperConfigMapName: hdfs-znode
+ nameNodes:
+ config:
+ logging:
+ enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+ configOverrides: &configOverrides
+ core-site.xml:
+ # The idea is that the user "hive" can't do anything in hdfs,
+ # *but* it can impersonate other users (such as trino),
+ # that have the needed permissions
+ hadoop.proxyuser.hive.users: "*"
+ hadoop.proxyuser.hive.hosts: "*"
+ roleGroups:
+ default:
+ replicas: 2
+ dataNodes:
+ config:
+ logging:
+ enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+ roleGroups:
+ default:
+ replicas: 1
+ journalNodes:
+ config:
+ logging:
+ enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+ roleGroups:
+ default:
+ replicas: 1
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+ name: hdfs-znode
+spec:
+ clusterRef:
+ name: zookeeper
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: hdfs-regorules
+ labels:
+ opa.stackable.tech/bundle: "true"
+data:
+ hdfs.rego: |
+ package hdfs
+
+ default allow := true
diff --git a/tests/templates/kuttl/iceberg/33-assert.yaml b/tests/templates/kuttl/iceberg/33-assert.yaml
new file mode 100644
index 00000000..50c27fd9
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/33-assert.yaml
@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 900
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: hive-metastore-default
+status:
+ readyReplicas: 1
+ replicas: 1
diff --git a/tests/templates/kuttl/iceberg/33-install-hive.yaml.j2 b/tests/templates/kuttl/iceberg/33-install-hive.yaml.j2
new file mode 100644
index 00000000..126fe2ab
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/33-install-hive.yaml.j2
@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+ # We need to replace $NAMESPACE (by KUTTL)
+ - script: envsubst '$NAMESPACE' < 33_hive.yaml | kubectl apply -n $NAMESPACE -f -
diff --git a/tests/templates/kuttl/iceberg/33_hive.yaml.j2 b/tests/templates/kuttl/iceberg/33_hive.yaml.j2
new file mode 100644
index 00000000..ad533522
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/33_hive.yaml.j2
@@ -0,0 +1,47 @@
+---
+apiVersion: hive.stackable.tech/v1alpha1
+kind: HiveCluster
+metadata:
+ name: hive
+spec:
+ image:
+{% if test_scenario['values']['hive-l'].find(",") > 0 %}
+ custom: "{{ test_scenario['values']['hive-l'].split(',')[1] }}"
+ productVersion: "{{ test_scenario['values']['hive-l'].split(',')[0] }}"
+{% else %}
+ productVersion: "{{ test_scenario['values']['hive-l'] }}"
+{% endif %}
+ pullPolicy: IfNotPresent
+ clusterConfig:
+ database:
+ connString: jdbc:postgresql://postgresql:5432/hive
+ credentialsSecret: postgres-credentials
+ dbType: postgres
+ hdfs:
+ configMap: hdfs
+ s3:
+ reference: minio
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+ authentication:
+ kerberos:
+ secretClass: kerberos-$NAMESPACE
+{% endif %}
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+ vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+ metastore:
+ config:
+ logging:
+ enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+ roleGroups:
+ default:
+ replicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: postgres-credentials
+type: Opaque
+stringData:
+ username: hive
+ password: hive
diff --git a/tests/templates/kuttl/iceberg/34-assert.yaml b/tests/templates/kuttl/iceberg/34-assert.yaml
new file mode 100644
index 00000000..b9f1bbe1
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/34-assert.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 720
+commands:
+ - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s
diff --git a/tests/templates/kuttl/iceberg/34-install-trino.yaml.j2 b/tests/templates/kuttl/iceberg/34-install-trino.yaml.j2
new file mode 100644
index 00000000..3bfdc0d2
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/34-install-trino.yaml.j2
@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+ # We need to replace $NAMESPACE (by KUTTL)
+ - script: envsubst '$NAMESPACE' < 34_trino.yaml | kubectl apply -n $NAMESPACE -f -
diff --git a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 b/tests/templates/kuttl/iceberg/34_trino.yaml.j2
new file mode 100644
index 00000000..e5c1e80c
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/34_trino.yaml.j2
@@ -0,0 +1,108 @@
+---
+apiVersion: trino.stackable.tech/v1alpha1
+kind: TrinoCluster
+metadata:
+ name: trino
+spec:
+ image:
+{% if test_scenario['values']['trino-l'].find(",") > 0 %}
+ custom: "{{ test_scenario['values']['trino-l'].split(',')[1] }}"
+ productVersion: "{{ test_scenario['values']['trino-l'].split(',')[0] }}"
+{% else %}
+ productVersion: "{{ test_scenario['values']['trino-l'] }}"
+{% endif %}
+ pullPolicy: IfNotPresent
+ clusterConfig:
+ catalogLabelSelector:
+ matchLabels:
+ trino: trino
+ listenerClass: external-unstable
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+ vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+ coordinators:
+ config:
+ logging:
+ enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+ podOverrides: &podOverrides
+ spec:
+ containers:
+ - name: trino
+ env:
+ - name: KERBEROS_REALM
+ value: {{ test_scenario['values']['kerberos-realm'] }}
+ volumeMounts:
+ - name: kerberos
+ mountPath: /stackable/kerberos
+ # Normally we would use a different location and set `-Djava.security.krb5.conf=/example/path/krb5.conf`,
+ # but we can not influence the JVM arguments (edit: we can now).
+ - name: kerberos
+ mountPath: /etc/krb5.conf
+ subPath: krb5.conf
+ volumes:
+ - name: kerberos
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ annotations:
+ secrets.stackable.tech/class: kerberos-$NAMESPACE
+ secrets.stackable.tech/kerberos.service.names: trino
+ secrets.stackable.tech/scope: service=trino
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: "1"
+ storageClassName: secrets.stackable.tech
+{% endif %}
+ roleGroups:
+ default:
+ replicas: 1
+ workers:
+ config:
+ gracefulShutdownTimeout: 60s # Let the test run faster
+ logging:
+ enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+ podOverrides: *podOverrides
+{% endif %}
+ roleGroups:
+ default:
+ replicas: 1
+ config: {}
+---
+apiVersion: trino.stackable.tech/v1alpha1
+kind: TrinoCatalog
+metadata:
+ name: iceberg
+ labels:
+ trino: trino
+spec:
+ connector:
+ iceberg:
+ metastore:
+ configMap: hive
+ s3:
+ reference: minio
+ hdfs:
+ configMap: hdfs
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+ configOverrides:
+ # HDFS configuration
+ hive.hdfs.authentication.type: KERBEROS
+ hive.hdfs.trino.principal: trino/trino.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }}
+ hive.hdfs.trino.keytab: /stackable/kerberos/keytab
+ hive.hdfs.impersonation.enabled: "false"
+ hive.hdfs.wire-encryption.enabled: "true"
+ # HMS configuration
+ hive.metastore.authentication.type: KERBEROS
+ hive.metastore.client.principal: trino/trino.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }}
+ hive.metastore.client.keytab: /stackable/kerberos/keytab
+ hive.metastore.service.principal: hive/hive.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }}
+ hive.metastore.thrift.impersonation.enabled: "false"
+ # By default, Hive views are executed with the RUN AS DEFINER security mode. Set the hive.hive-views.run-as-invoker catalog configuration property to true to use RUN AS INVOKER semantics.
+ # However, this does *not* work for Iceberg catalogs :/ (I asked on the Trino slack: https://trinodb.slack.com/archives/CJ6UC075E/p1711449384648869)
+ # hive.hive-views.run-as-invoker: "true"
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/40-assert.yaml b/tests/templates/kuttl/iceberg/40-assert.yaml
new file mode 100644
index 00000000..485373c8
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/40-assert.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: create-iceberg-tables
+status:
+ succeeded: 1
diff --git a/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 b/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2
new file mode 100644
index 00000000..7958931a
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2
@@ -0,0 +1,24 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: create-iceberg-tables
+spec:
+ template:
+ spec:
+ containers:
+ - name: create-iceberg-tables
+ image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-l'] }}-stackable0.0.0-dev"
+ command:
+ - bash
+ - -euo
+ - pipefail
+ - -c
+ - |
+ cat << 'EOF' | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin
+ CREATE SCHEMA IF NOT EXISTS iceberg.s3 WITH (location = 's3a://demo/lakehouse/s3');
+ CREATE TABLE IF NOT EXISTS iceberg.s3.greetings (hello varchar);
+ CREATE SCHEMA IF NOT EXISTS iceberg.hdfs WITH (location = 'hdfs:/lakehouse/hdfs');
+ CREATE TABLE IF NOT EXISTS iceberg.hdfs.greetings (hello varchar);
+ EOF
+ restartPolicy: OnFailure
diff --git a/tests/templates/kuttl/iceberg/50-assert.yaml b/tests/templates/kuttl/iceberg/50-assert.yaml
new file mode 100644
index 00000000..35aae31d
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/50-assert.yaml
@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 1200
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: nifi-node-default
+status:
+ readyReplicas: 1
+ replicas: 1
diff --git a/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2
new file mode 100644
index 00000000..006c39e3
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2
@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+ # We need to replace $NAMESPACE (by KUTTL)
+ - script: envsubst '$NAMESPACE' < 50_nifi.yaml | kubectl apply -n $NAMESPACE -f -
diff --git a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2
new file mode 100644
index 00000000..ac1d6ae3
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2
@@ -0,0 +1,154 @@
+---
+apiVersion: nifi.stackable.tech/v1alpha1
+kind: NifiCluster
+metadata:
+ name: nifi
+spec:
+ image:
+{% if test_scenario['values']['nifi-iceberg'].find(",") > 0 %}
+ custom: "{{ test_scenario['values']['nifi-iceberg'].split(',')[1] }}"
+ productVersion: "{{ test_scenario['values']['nifi-iceberg'].split(',')[0] }}"
+{% else %}
+ custom: null
+ productVersion: "{{ test_scenario['values']['nifi-iceberg'] }}"
+{% endif %}
+ pullPolicy: IfNotPresent
+ clusterConfig:
+ zookeeperConfigMapName: nifi-znode
+ listenerClass: external-unstable
+ authentication:
+ - authenticationClass: nifi-users
+ sensitiveProperties:
+ keySecret: nifi-sensitive-property-key
+ extraVolumes:
+ - name: hdfs-config
+ configMap:
+ name: hdfs
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+ - name: hive-config
+ configMap:
+ name: nifi-hive-config
+ - name: kerberos
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ annotations:
+ secrets.stackable.tech/class: kerberos-$NAMESPACE
+ secrets.stackable.tech/kerberos.service.names: nifi
+ secrets.stackable.tech/scope: service=nifi
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: "1"
+ storageClassName: secrets.stackable.tech
+{% endif %}
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+ vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+ nodes:
+ config:
+ logging:
+ enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+ configOverrides:
+ nifi.properties:
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+ nifi.kerberos.krb5.file: /stackable/userdata/kerberos/krb5.conf
+{% endif %}
+
+ # Quicker startup, and we only have a single node
+ nifi.cluster.flow.election.max.wait.time: 5 secs
+ jvmArgumentOverrides:
+ add:
+ # Needed for NiFi to trust the minio cert
+ - -Djavax.net.ssl.trustStore=/stackable/keystore/truststore.p12
+ - -Djavax.net.ssl.trustStorePassword=secret
+ - -Djavax.net.ssl.trustStoreType=PKCS12
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+ podOverrides: &podOverrides
+ spec:
+ containers:
+ - name: nifi
+ env:
+ - name: KERBEROS_REALM
+ value: {{ test_scenario['values']['kerberos-realm'] }}
+ volumeMounts:
+ - name: kerberos
+ mountPath: /stackable/kerberos
+ # Normally we would use a different location and set `-Djava.security.krb5.conf=/example/path/krb5.conf`,
+ # but we can not influence the JVM arguments (edit: we can now)
+ - name: kerberos
+ mountPath: /etc/krb5.conf
+ subPath: krb5.conf
+ volumes:
+ - name: kerberos
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ annotations:
+ secrets.stackable.tech/class: kerberos-$NAMESPACE
+ secrets.stackable.tech/kerberos.service.names: nifi
+ secrets.stackable.tech/scope: service=nifi
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: "1"
+ storageClassName: secrets.stackable.tech
+{% endif %}
+ roleGroups:
+ default:
+ replicas: 1
+---
+apiVersion: authentication.stackable.tech/v1alpha1
+kind: AuthenticationClass
+metadata:
+ name: nifi-users
+spec:
+ provider:
+ static:
+ userCredentialsSecret:
+ name: nifi-users
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: nifi-users
+stringData:
+ admin: adminadmin
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: nifi-sensitive-property-key
+stringData:
+ nifiSensitivePropsKey: mYsUp3rS3cr3tk3y
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+ name: nifi-znode
+spec:
+ clusterRef:
+ name: zookeeper
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: nifi-hive-config
+data:
+ hive-site.xml: |
+
+
+ hive.metastore.sasl.enabled
+ true
+
+
+ hive.metastore.kerberos.principal
+ hive/hive.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }}
+
+
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2 b/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2
new file mode 100644
index 00000000..f6373a77
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
+ - script: cat 60_nifi-flow-with-kerberos.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin
+{% else %}
+ - script: cat 60_nifi-flow-without-kerberos.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin
+
+{% endif %}
diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json
new file mode 100644
index 00000000..a5f773f1
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json
@@ -0,0 +1,1144 @@
+{
+ "externalControllerServices": {},
+ "flowContents": {
+ "comments": "",
+ "componentType": "PROCESS_GROUP",
+ "connections": [
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "0ccd3baf-e0f5-3935-b660-456082596e69",
+ "instanceIdentifier": "c6381ef4-05d1-323a-9392-dac83fba33b3",
+ "name": "Funnel",
+ "type": "FUNNEL"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "a3bbe2a3-2115-3ede-8759-d32c3b7c899e",
+ "instanceIdentifier": "6872c492-329c-36fe-85b5-9593161ccbeb",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "failure"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "abd8dd79-b5f5-368a-9562-633fd93c784b",
+ "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c",
+ "name": "PutIceberg into S3",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1",
+ "instanceIdentifier": "dd4c3b4d-7ad6-3ed3-b869-24a06f65a3e5",
+ "name": "Funnel",
+ "type": "FUNNEL"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "73b06526-5744-3aad-b148-995a910f28c5",
+ "instanceIdentifier": "c17f8b9d-ab2d-3797-82e2-deb7905f90ed",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "failure"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "27ba6117-133a-38e6-0000-0000390d1716",
+ "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439",
+ "name": "PutIceberg into HDFS",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "27ba6117-133a-38e6-0000-0000390d1716",
+ "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439",
+ "name": "PutIceberg into HDFS",
+ "type": "PROCESSOR"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "bc79d195-d4b2-3531-846e-77318e9f08c0",
+ "instanceIdentifier": "c197d602-e207-3d65-b7f7-cfc966a09089",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "success"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "27782850-84c8-37e6-a437-a74b2308e57e",
+ "instanceIdentifier": "a6286bb0-4644-39c7-b37b-eed047db10f3",
+ "name": "GenerateFlowFile",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4",
+ "instanceIdentifier": "5c168bce-0d60-372b-94f4-0690b880ce6c",
+ "name": "Funnel",
+ "type": "FUNNEL"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "79b3e396-3be1-3175-8c52-4f93347b1f9d",
+ "instanceIdentifier": "334a9821-8b6f-302c-a34d-51faa6bb36f1",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "success"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "27ba6117-133a-38e6-0000-0000390d1716",
+ "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439",
+ "name": "PutIceberg into HDFS",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "abd8dd79-b5f5-368a-9562-633fd93c784b",
+ "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c",
+ "name": "PutIceberg into S3",
+ "type": "PROCESSOR"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "11cf418f-4873-3913-b573-a2b07322b81d",
+ "instanceIdentifier": "01f6d8a0-cd07-36e0-b417-c757a727915b",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "success"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "27782850-84c8-37e6-a437-a74b2308e57e",
+ "instanceIdentifier": "a6286bb0-4644-39c7-b37b-eed047db10f3",
+ "name": "GenerateFlowFile",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "5a024e6e-7818-3e73-ffff-ffff964a69c4",
+ "instanceIdentifier": "77e72db2-fe65-31b7-a3b3-f9769051c8ac",
+ "name": "Funnel",
+ "type": "FUNNEL"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "6a11518e-a129-3faa-8bf2-71c2ad5d8d6f",
+ "instanceIdentifier": "5558acd0-942d-34ed-8507-dca4bd42307b",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "success"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "id": "abd8dd79-b5f5-368a-9562-633fd93c784b",
+ "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c",
+ "name": "PutIceberg into S3",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ }
+ ],
+ "controllerServices": [
+ {
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-aws-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "comments": "",
+ "componentType": "CONTROLLER_SERVICE",
+ "controllerServiceApis": [
+ {
+ "bundle": {
+ "artifact": "nifi-aws-service-api-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService"
+ },
+ {
+ "bundle": {
+ "artifact": "nifi-aws-service-api-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService"
+ }
+ ],
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "d9e8d00a-c387-3064-add2-c6060f158ae7",
+ "instanceIdentifier": "8bf71c50-5f81-3241-b983-28199b65d3d4",
+ "name": "AWSCredentialsProviderControllerService",
+ "properties": {
+ "anonymous-credentials": "false",
+ "Assume Role ARN": null,
+ "Assume Role Session Name": null,
+ "assume-role-external-id": null,
+ "assume-role-proxy-configuration-service": null,
+ "assume-role-ssl-context-service": null,
+ "assume-role-sts-endpoint": null,
+ "assume-role-sts-region": "us-west-2",
+ "assume-role-sts-signer-override": "Default Signature",
+ "Credentials File": null,
+ "custom-signer-class-name": null,
+ "custom-signer-module-location": null,
+ "default-credentials": "false",
+ "profile-name": null,
+ "Session Time": "3600",
+ "Access Key": "admin",
+ "Secret Key": "adminadmin"
+ },
+ "propertyDescriptors": {
+ "Access Key": {
+ "displayName": "Access Key ID",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Access Key",
+ "sensitive": true
+ },
+ "anonymous-credentials": {
+ "displayName": "Use Anonymous Credentials",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "anonymous-credentials",
+ "sensitive": false
+ },
+ "Assume Role ARN": {
+ "displayName": "Assume Role ARN",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Assume Role ARN",
+ "sensitive": false
+ },
+ "Assume Role Session Name": {
+ "displayName": "Assume Role Session Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Assume Role Session Name",
+ "sensitive": false
+ },
+ "assume-role-external-id": {
+ "displayName": "Assume Role External ID",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "assume-role-external-id",
+ "sensitive": false
+ },
+ "assume-role-proxy-configuration-service": {
+ "displayName": "Assume Role Proxy Configuration Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "assume-role-proxy-configuration-service",
+ "sensitive": false
+ },
+ "assume-role-ssl-context-service": {
+ "displayName": "Assume Role SSL Context Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "assume-role-ssl-context-service",
+ "sensitive": false
+ },
+ "assume-role-sts-endpoint": {
+ "displayName": "Assume Role STS Endpoint Override",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "assume-role-sts-endpoint",
+ "sensitive": false
+ },
+ "assume-role-sts-region": {
+ "displayName": "Assume Role STS Region",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "assume-role-sts-region",
+ "sensitive": false
+ },
+ "assume-role-sts-signer-override": {
+ "displayName": "Assume Role STS Signer Override",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "assume-role-sts-signer-override",
+ "sensitive": false
+ },
+ "Credentials File": {
+ "displayName": "Credentials File",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Credentials File",
+ "resourceDefinition": {
+ "cardinality": "SINGLE",
+ "resourceTypes": [
+ "FILE"
+ ]
+ },
+ "sensitive": false
+ },
+ "custom-signer-class-name": {
+ "displayName": "Custom Signer Class Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "custom-signer-class-name",
+ "sensitive": false
+ },
+ "custom-signer-module-location": {
+ "displayName": "Custom Signer Module Location",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "custom-signer-module-location",
+ "resourceDefinition": {
+ "cardinality": "MULTIPLE",
+ "resourceTypes": [
+ "DIRECTORY",
+ "FILE"
+ ]
+ },
+ "sensitive": false
+ },
+ "default-credentials": {
+ "displayName": "Use Default Credentials",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "default-credentials",
+ "sensitive": false
+ },
+ "profile-name": {
+ "displayName": "Profile Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "profile-name",
+ "sensitive": false
+ },
+ "Secret Key": {
+ "displayName": "Secret Access Key",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Secret Key",
+ "sensitive": true
+ },
+ "Session Time": {
+ "displayName": "Assume Role Session Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Session Time",
+ "sensitive": false
+ }
+ },
+ "scheduledState": "DISABLED",
+ "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService"
+ },
+ {
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-record-serialization-services-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "comments": "",
+ "componentType": "CONTROLLER_SERVICE",
+ "controllerServiceApis": [
+ {
+ "bundle": {
+ "artifact": "nifi-standard-services-api-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "type": "org.apache.nifi.serialization.RecordReaderFactory"
+ }
+ ],
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "af6f000d-1afb-39a5-89be-2ff9176b53fe",
+ "instanceIdentifier": "b9cdca6d-40e9-3f45-bd6e-448e9c772985",
+ "name": "JsonTreeReader",
+ "properties": {
+ "Allow Comments": "false",
+ "Date Format": null,
+ "Max String Length": "20 MB",
+ "schema-access-strategy": "infer-schema",
+ "schema-application-strategy": "SELECTED_PART",
+ "schema-branch": null,
+ "schema-inference-cache": null,
+ "schema-name": "${schema.name}",
+ "schema-reference-reader": null,
+ "schema-registry": null,
+ "schema-text": "${avro.schema}",
+ "schema-version": null,
+ "starting-field-name": null,
+ "starting-field-strategy": "ROOT_NODE",
+ "Time Format": null,
+ "Timestamp Format": null
+ },
+ "propertyDescriptors": {
+ "Allow Comments": {
+ "displayName": "Allow Comments",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Allow Comments",
+ "sensitive": false
+ },
+ "Date Format": {
+ "displayName": "Date Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Date Format",
+ "sensitive": false
+ },
+ "Max String Length": {
+ "displayName": "Max String Length",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Max String Length",
+ "sensitive": false
+ },
+ "schema-access-strategy": {
+ "displayName": "Schema Access Strategy",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-access-strategy",
+ "sensitive": false
+ },
+ "schema-application-strategy": {
+ "displayName": "Schema Application Strategy",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-application-strategy",
+ "sensitive": false
+ },
+ "schema-branch": {
+ "displayName": "Schema Branch",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-branch",
+ "sensitive": false
+ },
+ "schema-inference-cache": {
+ "displayName": "Schema Inference Cache",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "schema-inference-cache",
+ "sensitive": false
+ },
+ "schema-name": {
+ "displayName": "Schema Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-name",
+ "sensitive": false
+ },
+ "schema-reference-reader": {
+ "displayName": "Schema Reference Reader",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "schema-reference-reader",
+ "sensitive": false
+ },
+ "schema-registry": {
+ "displayName": "Schema Registry",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "schema-registry",
+ "sensitive": false
+ },
+ "schema-text": {
+ "displayName": "Schema Text",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-text",
+ "sensitive": false
+ },
+ "schema-version": {
+ "displayName": "Schema Version",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-version",
+ "sensitive": false
+ },
+ "starting-field-name": {
+ "displayName": "Starting Field Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "starting-field-name",
+ "sensitive": false
+ },
+ "starting-field-strategy": {
+ "displayName": "Starting Field Strategy",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "starting-field-strategy",
+ "sensitive": false
+ },
+ "Time Format": {
+ "displayName": "Time Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Time Format",
+ "sensitive": false
+ },
+ "Timestamp Format": {
+ "displayName": "Timestamp Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Timestamp Format",
+ "sensitive": false
+ }
+ },
+ "scheduledState": "DISABLED",
+ "type": "org.apache.nifi.json.JsonTreeReader"
+ },
+ {
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-kerberos-user-service-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "comments": "",
+ "componentType": "CONTROLLER_SERVICE",
+ "controllerServiceApis": [
+ {
+ "bundle": {
+ "artifact": "nifi-standard-services-api-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "type": "org.apache.nifi.kerberos.SelfContainedKerberosUserService"
+ },
+ {
+ "bundle": {
+ "artifact": "nifi-standard-services-api-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "type": "org.apache.nifi.kerberos.KerberosUserService"
+ }
+ ],
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "79d1509c-17ab-3c2a-a431-51324f50e829",
+ "instanceIdentifier": "a5bbd59e-0196-1000-0000-0000560be512",
+ "name": "KerberosKeytabUserService",
+ "properties": {
+ "Kerberos Keytab": "/stackable/userdata/kerberos/keytab",
+ "Kerberos Principal": "nifi/nifi.${NAMESPACE}.svc.cluster.local@PROD.MYCORP"
+ },
+ "propertyDescriptors": {
+ "Kerberos Keytab": {
+ "displayName": "Kerberos Keytab",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Kerberos Keytab",
+ "resourceDefinition": {
+ "cardinality": "SINGLE",
+ "resourceTypes": [
+ "FILE"
+ ]
+ },
+ "sensitive": false
+ },
+ "Kerberos Principal": {
+ "displayName": "Kerberos Principal",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Kerberos Principal",
+ "sensitive": false
+ }
+ },
+ "scheduledState": "DISABLED",
+ "type": "org.apache.nifi.kerberos.KerberosKeytabUserService"
+ },
+ {
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-iceberg-services-nar",
+ "group": "tech.stackable.nifi",
+ "version": "0.0.3"
+ },
+ "comments": "",
+ "componentType": "CONTROLLER_SERVICE",
+ "controllerServiceApis": [
+ {
+ "bundle": {
+ "artifact": "nifi-iceberg-services-api-nar",
+ "group": "tech.stackable.nifi",
+ "version": "0.0.3"
+ },
+ "type": "tech.stackable.nifi.services.iceberg.IcebergCatalogService"
+ }
+ ],
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "64202e39-4c8e-3ba5-9096-35c75272f5a4",
+ "instanceIdentifier": "e2c4259a-d4e0-3be9-af1c-c361796e6a0f",
+ "name": "IcebergHiveCatalogService",
+ "properties": {
+ "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7",
+ "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml,/stackable/userdata/hive-config/hive-site.xml",
+ "hive-metastore-uri": "thrift://hive:9083",
+ "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000",
+ "s3-path-style-access": "true",
+ "warehouse-location": "s3a://demo/lakehouse"
+ },
+ "propertyDescriptors": {
+ "AWS Credentials Provider service": {
+ "displayName": "AWS Credentials Provider Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "AWS Credentials Provider service",
+ "sensitive": false
+ },
+ "hadoop-config-resources": {
+ "displayName": "Hadoop Configuration Resources",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "hadoop-config-resources",
+ "resourceDefinition": {
+ "cardinality": "MULTIPLE",
+ "resourceTypes": [
+ "FILE"
+ ]
+ },
+ "sensitive": false
+ },
+ "hive-metastore-uri": {
+ "displayName": "Hive Metastore URI",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "hive-metastore-uri",
+ "sensitive": false
+ },
+ "s3-endpoint": {
+ "displayName": "S3 endpoint",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "s3-endpoint",
+ "sensitive": false
+ },
+ "s3-path-style-access": {
+ "displayName": "S3 path style access",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "s3-path-style-access",
+ "sensitive": false
+ },
+ "warehouse-location": {
+ "displayName": "Default Warehouse Location",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "warehouse-location",
+ "sensitive": false
+ }
+ },
+ "scheduledState": "DISABLED",
+ "type": "tech.stackable.nifi.services.iceberg.IcebergHiveCatalogService"
+ }
+ ],
+ "defaultBackPressureDataSizeThreshold": "1 GB",
+ "defaultBackPressureObjectThreshold": 10000,
+ "defaultFlowFileExpiration": "0 sec",
+ "executionEngine": "INHERITED",
+ "flowFileConcurrency": "UNBOUNDED",
+ "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE",
+ "funnels": [
+ {
+ "componentType": "FUNNEL",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4",
+ "instanceIdentifier": "5c168bce-0d60-372b-94f4-0690b880ce6c",
+ "position": {
+ "x": -880.0,
+ "y": -200.0
+ }
+ },
+ {
+ "componentType": "FUNNEL",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1",
+ "instanceIdentifier": "dd4c3b4d-7ad6-3ed3-b869-24a06f65a3e5",
+ "position": {
+ "x": -1344.0,
+ "y": -432.0
+ }
+ },
+ {
+ "componentType": "FUNNEL",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "5a024e6e-7818-3e73-ffff-ffff964a69c4",
+ "instanceIdentifier": "77e72db2-fe65-31b7-a3b3-f9769051c8ac",
+ "position": {
+ "x": -328.0,
+ "y": -200.0
+ }
+ },
+ {
+ "componentType": "FUNNEL",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "0ccd3baf-e0f5-3935-b660-456082596e69",
+ "instanceIdentifier": "c6381ef4-05d1-323a-9392-dac83fba33b3",
+ "position": {
+ "x": 160.0,
+ "y": -432.0
+ }
+ }
+ ],
+ "identifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "inputPorts": [],
+ "instanceIdentifier": "a5a6f0ac-0196-1000-ffff-fffffaba5637",
+ "labels": [],
+ "maxConcurrentTasks": 1,
+ "name": "Iceberg Test",
+ "outputPorts": [],
+ "position": {
+ "x": 100.0,
+ "y": 10.0
+ },
+ "processGroups": [],
+ "processors": [
+ {
+ "autoTerminatedRelationships": [],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-iceberg-processors-nar",
+ "group": "tech.stackable.nifi",
+ "version": "0.0.3"
+ },
+ "comments": "",
+ "componentType": "PROCESSOR",
+ "concurrentlySchedulableTaskCount": 1,
+ "executionNode": "ALL",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "27ba6117-133a-38e6-0000-0000390d1716",
+ "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439",
+ "maxBackoffPeriod": "10 mins",
+ "name": "PutIceberg into HDFS",
+ "penaltyDuration": "30 sec",
+ "position": {
+ "x": -1032.0,
+ "y": -472.0
+ },
+ "properties": {
+ "catalog-namespace": "hdfs",
+ "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4",
+ "file-format": "ORC",
+ "kerberos-user-service": "79d1509c-17ab-3c2a-a431-51324f50e829",
+ "maximum-commit-duration": "30 sec",
+ "maximum-commit-wait-time": "2 sec",
+ "maximum-file-size": null,
+ "minimum-commit-wait-time": "100 ms",
+ "number-of-commit-retries": "10",
+ "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe",
+ "table-name": "greetings",
+ "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN"
+ },
+ "propertyDescriptors": {
+ "catalog-namespace": {
+ "displayName": "Catalog Namespace",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "catalog-namespace",
+ "sensitive": false
+ },
+ "catalog-service": {
+ "displayName": "Catalog Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "catalog-service",
+ "sensitive": false
+ },
+ "file-format": {
+ "displayName": "File Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "file-format",
+ "sensitive": false
+ },
+ "kerberos-user-service": {
+ "displayName": "Kerberos User Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "kerberos-user-service",
+ "sensitive": false
+ },
+ "maximum-commit-duration": {
+ "displayName": "Maximum Commit Duration",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-commit-duration",
+ "sensitive": false
+ },
+ "maximum-commit-wait-time": {
+ "displayName": "Maximum Commit Wait Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-commit-wait-time",
+ "sensitive": false
+ },
+ "maximum-file-size": {
+ "displayName": "Maximum File Size",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-file-size",
+ "sensitive": false
+ },
+ "minimum-commit-wait-time": {
+ "displayName": "Minimum Commit Wait Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "minimum-commit-wait-time",
+ "sensitive": false
+ },
+ "number-of-commit-retries": {
+ "displayName": "Number of Commit Retries",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "number-of-commit-retries",
+ "sensitive": false
+ },
+ "record-reader": {
+ "displayName": "Record Reader",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "record-reader",
+ "sensitive": false
+ },
+ "table-name": {
+ "displayName": "Table Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "table-name",
+ "sensitive": false
+ },
+ "unmatched-column-behavior": {
+ "displayName": "Unmatched Column Behavior",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "unmatched-column-behavior",
+ "sensitive": false
+ }
+ },
+ "retriedRelationships": [],
+ "retryCount": 10,
+ "runDurationMillis": 0,
+ "scheduledState": "ENABLED",
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "style": {},
+ "type": "tech.stackable.nifi.processors.iceberg.PutIceberg",
+ "yieldDuration": "1 sec"
+ },
+ {
+ "autoTerminatedRelationships": [],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-iceberg-processors-nar",
+ "group": "tech.stackable.nifi",
+ "version": "0.0.3"
+ },
+ "comments": "",
+ "componentType": "PROCESSOR",
+ "concurrentlySchedulableTaskCount": 1,
+ "executionNode": "ALL",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "abd8dd79-b5f5-368a-9562-633fd93c784b",
+ "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c",
+ "maxBackoffPeriod": "10 mins",
+ "name": "PutIceberg into S3",
+ "penaltyDuration": "30 sec",
+ "position": {
+ "x": -480.0,
+ "y": -472.0
+ },
+ "properties": {
+ "catalog-namespace": "s3",
+ "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4",
+ "file-format": "PARQUET",
+ "kerberos-user-service": "79d1509c-17ab-3c2a-a431-51324f50e829",
+ "maximum-commit-duration": "30 sec",
+ "maximum-commit-wait-time": "2 sec",
+ "maximum-file-size": null,
+ "minimum-commit-wait-time": "100 ms",
+ "number-of-commit-retries": "10",
+ "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe",
+ "table-name": "greetings",
+ "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN"
+ },
+ "propertyDescriptors": {
+ "catalog-namespace": {
+ "displayName": "Catalog Namespace",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "catalog-namespace",
+ "sensitive": false
+ },
+ "catalog-service": {
+ "displayName": "Catalog Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "catalog-service",
+ "sensitive": false
+ },
+ "file-format": {
+ "displayName": "File Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "file-format",
+ "sensitive": false
+ },
+ "kerberos-user-service": {
+ "displayName": "Kerberos User Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "kerberos-user-service",
+ "sensitive": false
+ },
+ "maximum-commit-duration": {
+ "displayName": "Maximum Commit Duration",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-commit-duration",
+ "sensitive": false
+ },
+ "maximum-commit-wait-time": {
+ "displayName": "Maximum Commit Wait Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-commit-wait-time",
+ "sensitive": false
+ },
+ "maximum-file-size": {
+ "displayName": "Maximum File Size",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-file-size",
+ "sensitive": false
+ },
+ "minimum-commit-wait-time": {
+ "displayName": "Minimum Commit Wait Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "minimum-commit-wait-time",
+ "sensitive": false
+ },
+ "number-of-commit-retries": {
+ "displayName": "Number of Commit Retries",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "number-of-commit-retries",
+ "sensitive": false
+ },
+ "record-reader": {
+ "displayName": "Record Reader",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "record-reader",
+ "sensitive": false
+ },
+ "table-name": {
+ "displayName": "Table Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "table-name",
+ "sensitive": false
+ },
+ "unmatched-column-behavior": {
+ "displayName": "Unmatched Column Behavior",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "unmatched-column-behavior",
+ "sensitive": false
+ }
+ },
+ "retriedRelationships": [],
+ "retryCount": 10,
+ "runDurationMillis": 0,
+ "scheduledState": "ENABLED",
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "style": {},
+ "type": "tech.stackable.nifi.processors.iceberg.PutIceberg",
+ "yieldDuration": "1 sec"
+ },
+ {
+ "autoTerminatedRelationships": [],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-standard-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "comments": "",
+ "componentType": "PROCESSOR",
+ "concurrentlySchedulableTaskCount": 1,
+ "executionNode": "ALL",
+ "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563",
+ "identifier": "27782850-84c8-37e6-a437-a74b2308e57e",
+ "instanceIdentifier": "a6286bb0-4644-39c7-b37b-eed047db10f3",
+ "maxBackoffPeriod": "10 mins",
+ "name": "GenerateFlowFile",
+ "penaltyDuration": "30 sec",
+ "position": {
+ "x": -744.0,
+ "y": -704.0
+ },
+ "properties": {
+ "Batch Size": "1",
+ "character-set": "UTF-8",
+ "Data Format": "Text",
+ "File Size": "0B",
+ "generate-ff-custom-text": "{\"hello\": \"world from NiFi :)\"}",
+ "mime-type": null,
+ "Unique FlowFiles": "false"
+ },
+ "propertyDescriptors": {
+ "Batch Size": {
+ "displayName": "Batch Size",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Batch Size",
+ "sensitive": false
+ },
+ "character-set": {
+ "displayName": "Character Set",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "character-set",
+ "sensitive": false
+ },
+ "Data Format": {
+ "displayName": "Data Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Data Format",
+ "sensitive": false
+ },
+ "File Size": {
+ "displayName": "File Size",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "File Size",
+ "sensitive": false
+ },
+ "generate-ff-custom-text": {
+ "displayName": "Custom Text",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "generate-ff-custom-text",
+ "sensitive": false
+ },
+ "mime-type": {
+ "displayName": "Mime Type",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "mime-type",
+ "sensitive": false
+ },
+ "Unique FlowFiles": {
+ "displayName": "Unique FlowFiles",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Unique FlowFiles",
+ "sensitive": false
+ }
+ },
+ "retriedRelationships": [],
+ "retryCount": 10,
+ "runDurationMillis": 0,
+ "scheduledState": "ENABLED",
+ "schedulingPeriod": "1 min",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "style": {},
+ "type": "org.apache.nifi.processors.standard.GenerateFlowFile",
+ "yieldDuration": "1 sec"
+ }
+ ],
+ "remoteProcessGroups": [],
+ "scheduledState": "ENABLED",
+ "statelessFlowTimeout": "1 min"
+ },
+ "flowEncodingVersion": "1.0",
+ "latest": false,
+ "parameterContexts": {},
+ "parameterProviders": {}
+}
diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json
new file mode 100644
index 00000000..c63158b9
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json
@@ -0,0 +1,1068 @@
+{
+ "externalControllerServices": {},
+ "flowContents": {
+ "comments": "",
+ "componentType": "PROCESS_GROUP",
+ "connections": [
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "0ccd3baf-e0f5-3935-b660-456082596e69",
+ "instanceIdentifier": "9013ea63-2dbe-3774-83bf-97110f8adf27",
+ "name": "Funnel",
+ "type": "FUNNEL"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "a3bbe2a3-2115-3ede-8759-d32c3b7c899e",
+ "instanceIdentifier": "1b7d4c80-a27c-32e5-9428-0fce2f8a00bd",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "failure"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "abd8dd79-b5f5-368a-9562-633fd93c784b",
+ "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8",
+ "name": "PutIceberg",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1",
+ "instanceIdentifier": "8fbbad6f-0196-1000-0000-0000468ba4b3",
+ "name": "Funnel",
+ "type": "FUNNEL"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "73b06526-5744-3aad-b148-995a910f28c5",
+ "instanceIdentifier": "8fbbca9c-0196-1000-0000-00002d9b3b50",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "failure"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "27ba6117-133a-38e6-0000-0000390d1716",
+ "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147",
+ "name": "PutIceberg",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4",
+ "instanceIdentifier": "8fbcd068-0196-1000-0000-00003242b691",
+ "name": "Funnel",
+ "type": "FUNNEL"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "79b3e396-3be1-3175-8c52-4f93347b1f9d",
+ "instanceIdentifier": "8fbcdd98-0196-1000-0000-00003cc6f1be",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "success"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "27ba6117-133a-38e6-0000-0000390d1716",
+ "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147",
+ "name": "PutIceberg",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "27ba6117-133a-38e6-0000-0000390d1716",
+ "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147",
+ "name": "PutIceberg",
+ "type": "PROCESSOR"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "bc79d195-d4b2-3531-846e-77318e9f08c0",
+ "instanceIdentifier": "8fbb639e-0196-1000-ffff-ffff9b19bf06",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "success"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "27782850-84c8-37e6-a437-a74b2308e57e",
+ "instanceIdentifier": "a17760a7-4af3-3da3-a31c-24c4cd48e653",
+ "name": "GenerateFlowFile",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "abd8dd79-b5f5-368a-9562-633fd93c784b",
+ "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8",
+ "name": "PutIceberg",
+ "type": "PROCESSOR"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "11cf418f-4873-3913-b573-a2b07322b81d",
+ "instanceIdentifier": "c6e38b01-4f7f-34dd-8be1-22aec59413b6",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "success"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "27782850-84c8-37e6-a437-a74b2308e57e",
+ "instanceIdentifier": "a17760a7-4af3-3da3-a31c-24c4cd48e653",
+ "name": "GenerateFlowFile",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ },
+ {
+ "backPressureDataSizeThreshold": "1 GB",
+ "backPressureObjectThreshold": 10000,
+ "bends": [],
+ "componentType": "CONNECTION",
+ "destination": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "5a024e6e-7818-3e73-ffff-ffff964a69c4",
+ "instanceIdentifier": "6d38af9a-d0e9-3b2e-a776-44da91f4497d",
+ "name": "Funnel",
+ "type": "FUNNEL"
+ },
+ "flowFileExpiration": "0 sec",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "6a11518e-a129-3faa-8bf2-71c2ad5d8d6f",
+ "instanceIdentifier": "195765f2-2d5d-3221-9e72-fb39091ad4d0",
+ "labelIndex": 0,
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "name": "",
+ "partitioningAttribute": "",
+ "prioritizers": [],
+ "selectedRelationships": [
+ "success"
+ ],
+ "source": {
+ "comments": "",
+ "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "id": "abd8dd79-b5f5-368a-9562-633fd93c784b",
+ "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8",
+ "name": "PutIceberg",
+ "type": "PROCESSOR"
+ },
+ "zIndex": 0
+ }
+ ],
+ "controllerServices": [
+ {
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-aws-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "comments": "",
+ "componentType": "CONTROLLER_SERVICE",
+ "controllerServiceApis": [
+ {
+ "bundle": {
+ "artifact": "nifi-aws-service-api-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService"
+ },
+ {
+ "bundle": {
+ "artifact": "nifi-aws-service-api-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService"
+ }
+ ],
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "d9e8d00a-c387-3064-add2-c6060f158ae7",
+ "instanceIdentifier": "a4049610-9d6c-3a38-a411-06fe8e5b7425",
+ "name": "AWSCredentialsProviderControllerService",
+ "properties": {
+ "anonymous-credentials": "false",
+ "Assume Role ARN": null,
+ "Assume Role Session Name": null,
+ "assume-role-external-id": null,
+ "assume-role-proxy-configuration-service": null,
+ "assume-role-ssl-context-service": null,
+ "assume-role-sts-endpoint": null,
+ "assume-role-sts-region": "us-west-2",
+ "assume-role-sts-signer-override": "Default Signature",
+ "Credentials File": null,
+ "custom-signer-class-name": null,
+ "custom-signer-module-location": null,
+ "default-credentials": "false",
+ "profile-name": null,
+ "Session Time": "3600",
+ "Access Key": "admin",
+ "Secret Key": "adminadmin"
+ },
+ "propertyDescriptors": {
+ "Access Key": {
+ "displayName": "Access Key ID",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Access Key",
+ "sensitive": true
+ },
+ "anonymous-credentials": {
+ "displayName": "Use Anonymous Credentials",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "anonymous-credentials",
+ "sensitive": false
+ },
+ "Assume Role ARN": {
+ "displayName": "Assume Role ARN",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Assume Role ARN",
+ "sensitive": false
+ },
+ "Assume Role Session Name": {
+ "displayName": "Assume Role Session Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Assume Role Session Name",
+ "sensitive": false
+ },
+ "assume-role-external-id": {
+ "displayName": "Assume Role External ID",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "assume-role-external-id",
+ "sensitive": false
+ },
+ "assume-role-proxy-configuration-service": {
+ "displayName": "Assume Role Proxy Configuration Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "assume-role-proxy-configuration-service",
+ "sensitive": false
+ },
+ "assume-role-ssl-context-service": {
+ "displayName": "Assume Role SSL Context Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "assume-role-ssl-context-service",
+ "sensitive": false
+ },
+ "assume-role-sts-endpoint": {
+ "displayName": "Assume Role STS Endpoint Override",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "assume-role-sts-endpoint",
+ "sensitive": false
+ },
+ "assume-role-sts-region": {
+ "displayName": "Assume Role STS Region",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "assume-role-sts-region",
+ "sensitive": false
+ },
+ "assume-role-sts-signer-override": {
+ "displayName": "Assume Role STS Signer Override",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "assume-role-sts-signer-override",
+ "sensitive": false
+ },
+ "Credentials File": {
+ "displayName": "Credentials File",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Credentials File",
+ "resourceDefinition": {
+ "cardinality": "SINGLE",
+ "resourceTypes": [
+ "FILE"
+ ]
+ },
+ "sensitive": false
+ },
+ "custom-signer-class-name": {
+ "displayName": "Custom Signer Class Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "custom-signer-class-name",
+ "sensitive": false
+ },
+ "custom-signer-module-location": {
+ "displayName": "Custom Signer Module Location",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "custom-signer-module-location",
+ "resourceDefinition": {
+ "cardinality": "MULTIPLE",
+ "resourceTypes": [
+ "FILE",
+ "DIRECTORY"
+ ]
+ },
+ "sensitive": false
+ },
+ "default-credentials": {
+ "displayName": "Use Default Credentials",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "default-credentials",
+ "sensitive": false
+ },
+ "profile-name": {
+ "displayName": "Profile Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "profile-name",
+ "sensitive": false
+ },
+ "Secret Key": {
+ "displayName": "Secret Access Key",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Secret Key",
+ "sensitive": true
+ },
+ "Session Time": {
+ "displayName": "Assume Role Session Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Session Time",
+ "sensitive": false
+ }
+ },
+ "scheduledState": "DISABLED",
+ "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService"
+ },
+ {
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-record-serialization-services-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "comments": "",
+ "componentType": "CONTROLLER_SERVICE",
+ "controllerServiceApis": [
+ {
+ "bundle": {
+ "artifact": "nifi-standard-services-api-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "type": "org.apache.nifi.serialization.RecordReaderFactory"
+ }
+ ],
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "af6f000d-1afb-39a5-89be-2ff9176b53fe",
+ "instanceIdentifier": "15771472-fd85-37b8-9b4d-f496e31f49c2",
+ "name": "JsonTreeReader",
+ "properties": {
+ "Allow Comments": "false",
+ "Date Format": null,
+ "Max String Length": "20 MB",
+ "schema-access-strategy": "infer-schema",
+ "schema-application-strategy": "SELECTED_PART",
+ "schema-branch": null,
+ "schema-inference-cache": null,
+ "schema-name": "${schema.name}",
+ "schema-reference-reader": null,
+ "schema-registry": null,
+ "schema-text": "${avro.schema}",
+ "schema-version": null,
+ "starting-field-name": null,
+ "starting-field-strategy": "ROOT_NODE",
+ "Time Format": null,
+ "Timestamp Format": null
+ },
+ "propertyDescriptors": {
+ "Allow Comments": {
+ "displayName": "Allow Comments",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Allow Comments",
+ "sensitive": false
+ },
+ "Date Format": {
+ "displayName": "Date Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Date Format",
+ "sensitive": false
+ },
+ "Max String Length": {
+ "displayName": "Max String Length",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Max String Length",
+ "sensitive": false
+ },
+ "schema-access-strategy": {
+ "displayName": "Schema Access Strategy",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-access-strategy",
+ "sensitive": false
+ },
+ "schema-application-strategy": {
+ "displayName": "Schema Application Strategy",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-application-strategy",
+ "sensitive": false
+ },
+ "schema-branch": {
+ "displayName": "Schema Branch",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-branch",
+ "sensitive": false
+ },
+ "schema-inference-cache": {
+ "displayName": "Schema Inference Cache",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "schema-inference-cache",
+ "sensitive": false
+ },
+ "schema-name": {
+ "displayName": "Schema Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-name",
+ "sensitive": false
+ },
+ "schema-reference-reader": {
+ "displayName": "Schema Reference Reader",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "schema-reference-reader",
+ "sensitive": false
+ },
+ "schema-registry": {
+ "displayName": "Schema Registry",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "schema-registry",
+ "sensitive": false
+ },
+ "schema-text": {
+ "displayName": "Schema Text",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-text",
+ "sensitive": false
+ },
+ "schema-version": {
+ "displayName": "Schema Version",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "schema-version",
+ "sensitive": false
+ },
+ "starting-field-name": {
+ "displayName": "Starting Field Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "starting-field-name",
+ "sensitive": false
+ },
+ "starting-field-strategy": {
+ "displayName": "Starting Field Strategy",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "starting-field-strategy",
+ "sensitive": false
+ },
+ "Time Format": {
+ "displayName": "Time Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Time Format",
+ "sensitive": false
+ },
+ "Timestamp Format": {
+ "displayName": "Timestamp Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Timestamp Format",
+ "sensitive": false
+ }
+ },
+ "scheduledState": "DISABLED",
+ "type": "org.apache.nifi.json.JsonTreeReader"
+ },
+ {
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-iceberg-services-nar",
+ "group": "tech.stackable.nifi",
+ "version": "0.0.2"
+ },
+ "comments": "",
+ "componentType": "CONTROLLER_SERVICE",
+ "controllerServiceApis": [
+ {
+ "bundle": {
+ "artifact": "nifi-iceberg-services-api-nar",
+ "group": "tech.stackable.nifi",
+ "version": "0.0.2"
+ },
+ "type": "tech.stackable.nifi.services.iceberg.IcebergCatalogService"
+ }
+ ],
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "64202e39-4c8e-3ba5-9096-35c75272f5a4",
+ "instanceIdentifier": "127f71a3-94c7-3c71-b761-36f2a09629a7",
+ "name": "IcebergHiveCatalogService",
+ "properties": {
+ "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7",
+ "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml",
+ "hive-metastore-uri": "thrift://hive:9083",
+ "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000",
+ "s3-path-style-access": "true",
+ "warehouse-location": "s3a://demo/lakehouse"
+ },
+ "propertyDescriptors": {
+ "AWS Credentials Provider service": {
+ "displayName": "AWS Credentials Provider Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "AWS Credentials Provider service",
+ "sensitive": false
+ },
+ "hadoop-config-resources": {
+ "displayName": "Hadoop Configuration Resources",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "hadoop-config-resources",
+ "resourceDefinition": {
+ "cardinality": "MULTIPLE",
+ "resourceTypes": [
+ "FILE"
+ ]
+ },
+ "sensitive": false
+ },
+ "hive-metastore-uri": {
+ "displayName": "Hive Metastore URI",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "hive-metastore-uri",
+ "sensitive": false
+ },
+ "s3-endpoint": {
+ "displayName": "S3 endpoint",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "s3-endpoint",
+ "sensitive": false
+ },
+ "s3-path-style-access": {
+ "displayName": "S3 path style access",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "s3-path-style-access",
+ "sensitive": false
+ },
+ "warehouse-location": {
+ "displayName": "Default Warehouse Location",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "warehouse-location",
+ "sensitive": false
+ }
+ },
+ "scheduledState": "DISABLED",
+ "type": "tech.stackable.nifi.services.iceberg.IcebergHiveCatalogService"
+ }
+ ],
+ "defaultBackPressureDataSizeThreshold": "1 GB",
+ "defaultBackPressureObjectThreshold": 10000,
+ "defaultFlowFileExpiration": "0 sec",
+ "executionEngine": "INHERITED",
+ "flowFileConcurrency": "UNBOUNDED",
+ "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE",
+ "funnels": [
+ {
+ "componentType": "FUNNEL",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4",
+ "instanceIdentifier": "8fbcd068-0196-1000-0000-00003242b691",
+ "position": {
+ "x": -880.0,
+ "y": -200.0
+ }
+ },
+ {
+ "componentType": "FUNNEL",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1",
+ "instanceIdentifier": "8fbbad6f-0196-1000-0000-0000468ba4b3",
+ "position": {
+ "x": -1344.0,
+ "y": -432.0
+ }
+ },
+ {
+ "componentType": "FUNNEL",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "5a024e6e-7818-3e73-ffff-ffff964a69c4",
+ "instanceIdentifier": "6d38af9a-d0e9-3b2e-a776-44da91f4497d",
+ "position": {
+ "x": -328.0,
+ "y": -200.0
+ }
+ },
+ {
+ "componentType": "FUNNEL",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "0ccd3baf-e0f5-3935-b660-456082596e69",
+ "instanceIdentifier": "9013ea63-2dbe-3774-83bf-97110f8adf27",
+ "position": {
+ "x": 160.0,
+ "y": -432.0
+ }
+ }
+ ],
+ "identifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "inputPorts": [],
+ "instanceIdentifier": "8fb66dc6-0196-1000-ffff-ffff98cfc281",
+ "labels": [],
+ "maxConcurrentTasks": 1,
+ "name": "Iceberg Test",
+ "outputPorts": [],
+ "position": {
+ "x": 100.0,
+ "y": 10.0
+ },
+ "processGroups": [],
+ "processors": [
+ {
+ "autoTerminatedRelationships": [],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-iceberg-processors-nar",
+ "group": "tech.stackable.nifi",
+ "version": "0.0.2"
+ },
+ "comments": "",
+ "componentType": "PROCESSOR",
+ "concurrentlySchedulableTaskCount": 1,
+ "executionNode": "ALL",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "27ba6117-133a-38e6-0000-0000390d1716",
+ "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147",
+ "maxBackoffPeriod": "10 mins",
+ "name": "PutIceberg",
+ "penaltyDuration": "30 sec",
+ "position": {
+ "x": -1032.0,
+ "y": -472.0
+ },
+ "properties": {
+ "catalog-namespace": "hdfs",
+ "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4",
+ "file-format": "ORC",
+ "maximum-commit-duration": "30 sec",
+ "maximum-commit-wait-time": "2 sec",
+ "maximum-file-size": null,
+ "minimum-commit-wait-time": "100 ms",
+ "number-of-commit-retries": "10",
+ "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe",
+ "table-name": "greetings",
+ "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN"
+ },
+ "propertyDescriptors": {
+ "catalog-namespace": {
+ "displayName": "Catalog Namespace",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "catalog-namespace",
+ "sensitive": false
+ },
+ "catalog-service": {
+ "displayName": "Catalog Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "catalog-service",
+ "sensitive": false
+ },
+ "file-format": {
+ "displayName": "File Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "file-format",
+ "sensitive": false
+ },
+ "maximum-commit-duration": {
+ "displayName": "Maximum Commit Duration",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-commit-duration",
+ "sensitive": false
+ },
+ "maximum-commit-wait-time": {
+ "displayName": "Maximum Commit Wait Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-commit-wait-time",
+ "sensitive": false
+ },
+ "maximum-file-size": {
+ "displayName": "Maximum File Size",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-file-size",
+ "sensitive": false
+ },
+ "minimum-commit-wait-time": {
+ "displayName": "Minimum Commit Wait Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "minimum-commit-wait-time",
+ "sensitive": false
+ },
+ "number-of-commit-retries": {
+ "displayName": "Number of Commit Retries",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "number-of-commit-retries",
+ "sensitive": false
+ },
+ "record-reader": {
+ "displayName": "Record Reader",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "record-reader",
+ "sensitive": false
+ },
+ "table-name": {
+ "displayName": "Table Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "table-name",
+ "sensitive": false
+ },
+ "unmatched-column-behavior": {
+ "displayName": "Unmatched Column Behavior",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "unmatched-column-behavior",
+ "sensitive": false
+ }
+ },
+ "retriedRelationships": [],
+ "retryCount": 10,
+ "runDurationMillis": 0,
+ "scheduledState": "ENABLED",
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "style": {},
+ "type": "tech.stackable.nifi.processors.iceberg.PutIceberg",
+ "yieldDuration": "1 sec"
+ },
+ {
+ "autoTerminatedRelationships": [],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-iceberg-processors-nar",
+ "group": "tech.stackable.nifi",
+ "version": "0.0.2"
+ },
+ "comments": "",
+ "componentType": "PROCESSOR",
+ "concurrentlySchedulableTaskCount": 1,
+ "executionNode": "ALL",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "abd8dd79-b5f5-368a-9562-633fd93c784b",
+ "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8",
+ "maxBackoffPeriod": "10 mins",
+ "name": "PutIceberg",
+ "penaltyDuration": "30 sec",
+ "position": {
+ "x": -480.0,
+ "y": -472.0
+ },
+ "properties": {
+ "catalog-namespace": "s3",
+ "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4",
+ "file-format": "PARQUET",
+ "maximum-commit-duration": "30 sec",
+ "maximum-commit-wait-time": "2 sec",
+ "maximum-file-size": null,
+ "minimum-commit-wait-time": "100 ms",
+ "number-of-commit-retries": "10",
+ "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe",
+ "table-name": "greetings",
+ "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN"
+ },
+ "propertyDescriptors": {
+ "catalog-namespace": {
+ "displayName": "Catalog Namespace",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "catalog-namespace",
+ "sensitive": false
+ },
+ "catalog-service": {
+ "displayName": "Catalog Service",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "catalog-service",
+ "sensitive": false
+ },
+ "file-format": {
+ "displayName": "File Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "file-format",
+ "sensitive": false
+ },
+ "maximum-commit-duration": {
+ "displayName": "Maximum Commit Duration",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-commit-duration",
+ "sensitive": false
+ },
+ "maximum-commit-wait-time": {
+ "displayName": "Maximum Commit Wait Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-commit-wait-time",
+ "sensitive": false
+ },
+ "maximum-file-size": {
+ "displayName": "Maximum File Size",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "maximum-file-size",
+ "sensitive": false
+ },
+ "minimum-commit-wait-time": {
+ "displayName": "Minimum Commit Wait Time",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "minimum-commit-wait-time",
+ "sensitive": false
+ },
+ "number-of-commit-retries": {
+ "displayName": "Number of Commit Retries",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "number-of-commit-retries",
+ "sensitive": false
+ },
+ "record-reader": {
+ "displayName": "Record Reader",
+ "dynamic": false,
+ "identifiesControllerService": true,
+ "name": "record-reader",
+ "sensitive": false
+ },
+ "table-name": {
+ "displayName": "Table Name",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "table-name",
+ "sensitive": false
+ },
+ "unmatched-column-behavior": {
+ "displayName": "Unmatched Column Behavior",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "unmatched-column-behavior",
+ "sensitive": false
+ }
+ },
+ "retriedRelationships": [],
+ "retryCount": 10,
+ "runDurationMillis": 0,
+ "scheduledState": "ENABLED",
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "style": {},
+ "type": "tech.stackable.nifi.processors.iceberg.PutIceberg",
+ "yieldDuration": "1 sec"
+ },
+ {
+ "autoTerminatedRelationships": [],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "bulletinLevel": "WARN",
+ "bundle": {
+ "artifact": "nifi-standard-nar",
+ "group": "org.apache.nifi",
+ "version": "2.2.0"
+ },
+ "comments": "",
+ "componentType": "PROCESSOR",
+ "concurrentlySchedulableTaskCount": 1,
+ "executionNode": "ALL",
+ "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969",
+ "identifier": "27782850-84c8-37e6-a437-a74b2308e57e",
+ "instanceIdentifier": "a17760a7-4af3-3da3-a31c-24c4cd48e653",
+ "maxBackoffPeriod": "10 mins",
+ "name": "GenerateFlowFile",
+ "penaltyDuration": "30 sec",
+ "position": {
+ "x": -744.0,
+ "y": -704.0
+ },
+ "properties": {
+ "Batch Size": "1",
+ "character-set": "UTF-8",
+ "Data Format": "Text",
+ "File Size": "0B",
+ "generate-ff-custom-text": "{\"hello\": \"world from NiFi :)\"}",
+ "mime-type": null,
+ "Unique FlowFiles": "false"
+ },
+ "propertyDescriptors": {
+ "Batch Size": {
+ "displayName": "Batch Size",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Batch Size",
+ "sensitive": false
+ },
+ "character-set": {
+ "displayName": "Character Set",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "character-set",
+ "sensitive": false
+ },
+ "Data Format": {
+ "displayName": "Data Format",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Data Format",
+ "sensitive": false
+ },
+ "File Size": {
+ "displayName": "File Size",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "File Size",
+ "sensitive": false
+ },
+ "generate-ff-custom-text": {
+ "displayName": "Custom Text",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "generate-ff-custom-text",
+ "sensitive": false
+ },
+ "mime-type": {
+ "displayName": "Mime Type",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "mime-type",
+ "sensitive": false
+ },
+ "Unique FlowFiles": {
+ "displayName": "Unique FlowFiles",
+ "dynamic": false,
+ "identifiesControllerService": false,
+ "name": "Unique FlowFiles",
+ "sensitive": false
+ }
+ },
+ "retriedRelationships": [],
+ "retryCount": 10,
+ "runDurationMillis": 0,
+ "scheduledState": "ENABLED",
+ "schedulingPeriod": "1 min",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "style": {},
+ "type": "org.apache.nifi.processors.standard.GenerateFlowFile",
+ "yieldDuration": "1 sec"
+ }
+ ],
+ "remoteProcessGroups": [],
+ "scheduledState": "ENABLED",
+ "statelessFlowTimeout": "1 min"
+ },
+ "flowEncodingVersion": "1.0",
+ "latest": false,
+ "parameterContexts": {},
+ "parameterProviders": {}
+}
diff --git a/tests/templates/kuttl/iceberg/61-assert.yaml b/tests/templates/kuttl/iceberg/61-assert.yaml
new file mode 100644
index 00000000..813504b1
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/61-assert.yaml
@@ -0,0 +1,11 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: provision-nifi-flow
+status:
+ succeeded: 1
diff --git a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml
new file mode 100644
index 00000000..7ab671ce
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml
@@ -0,0 +1,114 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: provision-nifi-flow
+spec:
+ template:
+ spec:
+ containers:
+ - name: provision-nifi-flow
+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+ command:
+ - bash
+ - -euo
+ - pipefail
+ - -c
+ - python -u /tmp/script/script.py
+ volumeMounts:
+ - name: script
+ mountPath: /tmp/script
+ - name: nifi-flow
+ mountPath: /tmp/nifi-flow
+ - name: nifi-users
+ mountPath: /nifi-users
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ volumes:
+ - name: script
+ configMap:
+ name: provision-nifi-flow-script
+ - name: nifi-flow
+ configMap:
+ name: nifi-flow
+ - name: nifi-users
+ secret:
+ secretName: nifi-users
+ restartPolicy: OnFailure
+---
+# Taken from https://github.com/stackabletech/demos/blob/1744be00054eec2827b2d9ef0a90645843cb0075/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml#L52
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: provision-nifi-flow-script
+data:
+ script.py: |
+ from nipyapi.canvas import get_root_pg_id, schedule_process_group, list_all_controllers, schedule_controller
+ from nipyapi.security import service_login
+ import nipyapi
+ import os
+ import requests
+ import urllib3
+
+ # As of 2022-08-29 we cant use "https://nifi:8443" here because The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.
+ ENDPOINT = f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443"
+ USERNAME = "admin"
+ PASSWORD = open("/nifi-users/admin").read()
+
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+ nipyapi.config.nifi_config.host = f"{ENDPOINT}/nifi-api"
+ nipyapi.config.nifi_config.verify_ssl = False
+
+ print(f"Logging in as {USERNAME}")
+ service_login(username=USERNAME, password=PASSWORD)
+ print("Logged in")
+
+ filename = "/tmp/nifi-flow/nifi-flow.json"
+
+ pg_id = get_root_pg_id()
+ print(f"Got root process group id: {pg_id}")
+
+ if not nipyapi.config.nifi_config.api_client:
+ nipyapi.config.nifi_config.api_client = ApiClient()
+
+ header_params = {}
+ header_params['Accept'] = nipyapi.config.nifi_config.api_client.select_header_accept(['application/json'])
+ header_params['Content-Type'] = nipyapi.config.nifi_config.api_client.select_header_content_type(['multipart/form-data'])
+
+ print("Uploading process group")
+ nipyapi.config.nifi_config.api_client.call_api('/process-groups/{pg_id}/process-groups/upload', 'POST',
+ path_params={'pg_id': pg_id},
+ header_params=header_params,
+ _return_http_data_only=True,
+ post_params=[
+ ('id', pg_id),
+ ('groupName', 'Iceberg Test'),
+ ('positionX', 100),
+ ('positionY', 10),
+ ('clientId', nipyapi.nifi.FlowApi().generate_client_id()),
+ ],
+ files={
+ 'file': filename
+ },
+ auth_settings=['tokenAuth'])
+ print("Process group uploaded")
+
+ # As they are started in the wrong order :D we need to retry
+ max_retries = 5
+ for _ in range(max_retries):
+ controllers = list_all_controllers(pg_id)
+ print(f"Found {len(controllers)} controllers")
+ for controller in controllers:
+ if controller.component.state != "ENABLED":
+ try:
+ print(f"Scheduling controller {controller.component.name}")
+ schedule_controller(controller, scheduled=True)
+ print(f"Scheduled controller: {controller.component.name}")
+ except Exception as e:
+ print(f"Failed to schedule controller {controller.component.name}: {e}")
+
+ schedule_process_group(pg_id, scheduled=True)
diff --git a/tests/templates/kuttl/iceberg/70-assert.yaml b/tests/templates/kuttl/iceberg/70-assert.yaml
new file mode 100644
index 00000000..4a7bd272
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/70-assert.yaml
@@ -0,0 +1,11 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: check-iceberg-tables
+status:
+ succeeded: 1
diff --git a/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 b/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2
new file mode 100644
index 00000000..079a2722
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2
@@ -0,0 +1,36 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: check-iceberg-tables
+spec:
+ template:
+ spec:
+ containers:
+ - name: check-iceberg-tables
+ image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-l'] }}-stackable0.0.0-dev"
+ command:
+ - bash
+ - -euo
+ - pipefail
+ - -c
+ - |
+ for SCHEMA in iceberg.s3 iceberg.hdfs; do
+ COUNT=$(cat << EOF | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin
+ SELECT COUNT(*) FROM $SCHEMA.greetings WHERE hello = 'world from NiFi :)';
+ EOF
+ )
+
+ COUNT="${COUNT%\"}" # Remove trailing quote if any
+ COUNT="${COUNT#\"}" # Remove leading quote if any
+ echo "Count is $COUNT"
+
+ # Check if it's a number greater than 0
+ if ! [[ "$COUNT" =~ ^[0-9]+$ ]] || [ "$COUNT" -le 0 ]; then
+ echo "Invalid or zero count: $COUNT"
+ exit 1
+ fi
+
+ echo "Count $COUNT was valid"
+ done
+ restartPolicy: OnFailure
diff --git a/tests/templates/kuttl/iceberg/README.md b/tests/templates/kuttl/iceberg/README.md
new file mode 100644
index 00000000..b856148c
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/README.md
@@ -0,0 +1,34 @@
+The file `60_nifi-flow.json` was exported from the NiFi UI.
+
+*However*, we need to update some stuff, such as adding S3 credentials and templating the namespace of MinIO.
+
+TIP: I used `JSON: Sort Document` in VScode to somewhat have consistent formatting, which makes reading and diffs easier.
+
+Notable the following diff has been made (may not be up to date!):
+
+```diff
+diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow.json b/tests/templates/kuttl/iceberg/60_nifi-flow.json
+index 09783fa..23c679f 100644
+--- a/tests/templates/kuttl/iceberg/60_nifi-flow.json
++++ b/tests/templates/kuttl/iceberg/60_nifi-flow.json
+@@ -160,7 +160,9 @@
+ "custom-signer-module-location": null,
+ "default-credentials": "false",
+ "profile-name": null,
+- "Session Time": "3600"
++ "Session Time": "3600",
++ "Access Key": "admin",
++ "Secret Key": "adminadmin"
+ },
+ "propertyDescriptors": {
+ "Access Key": {
+@@ -483,7 +485,7 @@
+ "properties": {
+ "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7",
+ "hive-metastore-uri": "thrift://hive:9083",
+- "s3-endpoint": "https://minio.kuttl-test-patient-tarpon.svc.cluster.local:9000",
++ "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000",
+ "s3-path-style-access": "true",
+ "warehouse-location": "s3a://demo/lakehouse"
+ },
+```
diff --git a/tests/templates/kuttl/iceberg/interactive-nifi.yaml b/tests/templates/kuttl/iceberg/interactive-nifi.yaml
new file mode 100644
index 00000000..d21fff01
--- /dev/null
+++ b/tests/templates/kuttl/iceberg/interactive-nifi.yaml
@@ -0,0 +1,88 @@
+apiVersion: nifi.stackable.tech/v1alpha1
+kind: NifiCluster
+metadata:
+ name: nifi-interactive
+spec:
+ image:
+ productVersion: 2.2.0
+ clusterConfig:
+ authentication:
+ - authenticationClass: nifi-users
+ listenerClass: external-unstable
+ sensitiveProperties:
+ keySecret: nifi-interactive-sensitive-property-key
+ autoGenerate: true
+ zookeeperConfigMapName: nifi-interactive-znode
+ extraVolumes:
+ - name: nifi-processors
+ persistentVolumeClaim:
+ claimName: nifi-interactive-processors
+ - name: hdfs-config
+ configMap:
+ name: hdfs
+ - name: nifi-interactive-hive-site
+ configMap:
+ name: nifi-interactive-hive-site
+ - name: kerberos
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ annotations:
+ secrets.stackable.tech/class: kerberos-kuttl-test-united-pheasant
+ secrets.stackable.tech/kerberos.service.names: nifi
+ secrets.stackable.tech/scope: service=nifi
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: "1"
+ storageClassName: secrets.stackable.tech
+ nodes:
+ configOverrides:
+ nifi.properties:
+ nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/
+ nifi.kerberos.krb5.file: /stackable/userdata/kerberos/krb5.conf
+ # Quicker startup
+ nifi.cluster.flow.election.max.wait.time: 3 secs
+ envOverrides:
+ KERBEROS_REALM: PROD.MYCORP
+ roleGroups:
+ default:
+ replicas: 1
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+ name: nifi-interactive-znode
+spec:
+ clusterRef:
+ name: zookeeper
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: nifi-interactive-processors
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 10Gi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: nifi-interactive-hive-site
+data:
+ hive-site.xml: |
+
+
+ hive.metastore.kerberos.principal
+ hive/hive.kuttl-test-united-pheasant.svc.cluster.local@MY.CORP
+
+
+ hive.metastore.sasl.enabled
+ true
+
+
diff --git a/tests/templates/kuttl/smoke/20-assert.yaml b/tests/templates/kuttl/smoke/20-assert.yaml
index e0766c49..49ba7437 100644
--- a/tests/templates/kuttl/smoke/20-assert.yaml
+++ b/tests/templates/kuttl/smoke/20-assert.yaml
@@ -6,7 +6,7 @@ timeout: 600
apiVersion: apps/v1
kind: StatefulSet
metadata:
- name: test-zk-server-default
+ name: zookeeper-server-default
status:
readyReplicas: 1
replicas: 1
diff --git a/tests/templates/kuttl/smoke/20-install-zk.yaml.j2 b/tests/templates/kuttl/smoke/20-install-zookeeper.yaml.j2
similarity index 78%
rename from tests/templates/kuttl/smoke/20-install-zk.yaml.j2
rename to tests/templates/kuttl/smoke/20-install-zookeeper.yaml.j2
index 100927b1..103845c4 100644
--- a/tests/templates/kuttl/smoke/20-install-zk.yaml.j2
+++ b/tests/templates/kuttl/smoke/20-install-zookeeper.yaml.j2
@@ -2,7 +2,7 @@
apiVersion: zookeeper.stackable.tech/v1alpha1
kind: ZookeeperCluster
metadata:
- name: test-zk
+ name: zookeeper
spec:
image:
productVersion: "{{ test_scenario['values']['zookeeper'] }}"
@@ -19,11 +19,3 @@ spec:
roleGroups:
default:
replicas: 1
----
-apiVersion: zookeeper.stackable.tech/v1alpha1
-kind: ZookeeperZnode
-metadata:
- name: test-nifi-znode
-spec:
- clusterRef:
- name: test-zk
diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml
index ae825d11..d075654f 100644
--- a/tests/templates/kuttl/smoke/30-assert.yaml
+++ b/tests/templates/kuttl/smoke/30-assert.yaml
@@ -6,7 +6,7 @@ timeout: 1200
apiVersion: apps/v1
kind: StatefulSet
metadata:
- name: test-nifi-node-default
+ name: nifi-node-default
spec:
template:
spec:
@@ -18,7 +18,7 @@ status:
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
- name: test-nifi-node
+ name: nifi-node
status:
expectedPods: 2
currentHealthy: 2
diff --git a/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2
index aceebaf6..337c37a9 100644
--- a/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2
+++ b/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2
@@ -1,33 +1,8 @@
---
-apiVersion: authentication.stackable.tech/v1alpha1
-kind: AuthenticationClass
-metadata:
- name: simple-nifi-users
-spec:
- provider:
- static:
- userCredentialsSecret:
- name: simple-nifi-admin-credentials
----
-apiVersion: v1
-kind: Secret
-metadata:
- name: simple-nifi-admin-credentials
-stringData:
- admin: >
- passwordWithSpecialCharacter\@<&>"'
----
-apiVersion: v1
-kind: Secret
-metadata:
- name: nifi-sensitive-property-key
-stringData:
- nifiSensitivePropsKey: mYsUp3rS3cr3tk3y
----
apiVersion: nifi.stackable.tech/v1alpha1
kind: NifiCluster
metadata:
- name: test-nifi
+ name: nifi
spec:
image:
{% if test_scenario['values']['nifi'].find(",") > 0 %}
@@ -39,10 +14,10 @@ spec:
{% endif %}
pullPolicy: IfNotPresent
clusterConfig:
- zookeeperConfigMapName: test-nifi-znode
+ zookeeperConfigMapName: nifi-znode
listenerClass: {{ test_scenario['values']['listener-class'] }}
authentication:
- - authenticationClass: simple-nifi-users
+ - authenticationClass: nifi-users
hostHeaderCheck:
allowAll: false
additionalAllowedHosts:
@@ -73,3 +48,36 @@ spec:
"nifi.properties":
"nifi.diagnostics.on.shutdown.enabled": "false"
"nifi.diagnostics.on.shutdown.max.filecount": "20"
+---
+apiVersion: authentication.stackable.tech/v1alpha1
+kind: AuthenticationClass
+metadata:
+ name: nifi-users
+spec:
+ provider:
+ static:
+ userCredentialsSecret:
+ name: nifi-users
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: nifi-users
+stringData:
+ admin: >
+ passwordWithSpecialCharacter\@<&>"'
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: nifi-sensitive-property-key
+stringData:
+ nifiSensitivePropsKey: mYsUp3rS3cr3tk3y
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+ name: nifi-znode
+spec:
+ clusterRef:
+ name: zookeeper
diff --git a/tests/templates/kuttl/smoke/31-assert.yaml.j2 b/tests/templates/kuttl/smoke/31-assert.yaml.j2
index 06b1dc78..7b2eacd4 100644
--- a/tests/templates/kuttl/smoke/31-assert.yaml.j2
+++ b/tests/templates/kuttl/smoke/31-assert.yaml.j2
@@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1
kind: TestAssert
timeout: 30
commands:
-- script: kubectl get cm -n $NAMESPACE test-nifi-node-default -o yaml | grep -- 'nifi.web.proxy.host=.*example.com:1234' | xargs test ! -z
+- script: kubectl get cm -n $NAMESPACE nifi-node-default -o yaml | grep -- 'nifi.web.proxy.host=.*example.com:1234' | xargs test ! -z
diff --git a/tests/templates/kuttl/smoke/32-assert.yaml b/tests/templates/kuttl/smoke/32-assert.yaml
index 00da1613..960b4789 100644
--- a/tests/templates/kuttl/smoke/32-assert.yaml
+++ b/tests/templates/kuttl/smoke/32-assert.yaml
@@ -7,13 +7,13 @@ commands:
# Test envOverrides
#
- script: |
- kubectl -n $NAMESPACE get sts test-nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
- kubectl -n $NAMESPACE get sts test-nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
- kubectl -n $NAMESPACE get sts test-nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'
+ kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
+ kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
+ kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'
#
# Test configOverrides
#
- script: |
- kubectl -n $NAMESPACE get cm test-nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.enabled=false"
- kubectl -n $NAMESPACE get cm test-nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.verbose=false"
- kubectl -n $NAMESPACE get cm test-nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.max.filecount=20"
+ kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.enabled=false"
+ kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.verbose=false"
+ kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.max.filecount=20"
diff --git a/tests/templates/kuttl/smoke/33-assert.yaml b/tests/templates/kuttl/smoke/33-assert.yaml
index b2a98140..271a8bec 100644
--- a/tests/templates/kuttl/smoke/33-assert.yaml
+++ b/tests/templates/kuttl/smoke/33-assert.yaml
@@ -4,4 +4,4 @@ apiVersion: kuttl.dev/v1beta1
kind: TestAssert
timeout: 600
commands:
- - script: kubectl exec -n $NAMESPACE --container nifi test-nifi-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"'
+ - script: kubectl exec -n $NAMESPACE --container nifi nifi-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"'
diff --git a/tests/templates/kuttl/smoke/40-assert.yaml b/tests/templates/kuttl/smoke/40-assert.yaml
index 88f50b77..c3bcd76b 100644
--- a/tests/templates/kuttl/smoke/40-assert.yaml
+++ b/tests/templates/kuttl/smoke/40-assert.yaml
@@ -6,7 +6,7 @@ timeout: 1200
apiVersion: apps/v1
kind: StatefulSet
metadata:
- name: test-nifi-node-default
+ name: nifi-node-default
status:
readyReplicas: 3
replicas: 3
diff --git a/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2 b/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2
index 987b0745..37ded261 100644
--- a/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2
+++ b/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2
@@ -4,5 +4,5 @@ kind: TestStep
commands:
- script: >-
kubectl --namespace $NAMESPACE
- patch nificlusters.nifi.stackable.tech test-nifi
+ patch nificlusters.nifi.stackable.tech nifi
--type=merge --patch '{"spec":{"nodes": {"roleGroups": {"default": {"replicas": 3}}}}}'
diff --git a/tests/templates/kuttl/smoke/70-assert.yaml b/tests/templates/kuttl/smoke/70-assert.yaml
deleted file mode 100644
index 29b0ff4d..00000000
--- a/tests/templates/kuttl/smoke/70-assert.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
----
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-timeout: 600
----
-apiVersion: v1
-kind: Event
-reason: Started
-source:
- component: kubelet
-involvedObject:
- apiVersion: v1
- kind: Pod
- name: test-nifi-node-default-0
----
-apiVersion: v1
-kind: Event
-reason: Started
-source:
- component: kubelet
-involvedObject:
- apiVersion: v1
- kind: Pod
- name: test-nifi-node-default-1
----
-apiVersion: v1
-kind: Event
-reason: Started
-source:
- component: kubelet
-involvedObject:
- apiVersion: v1
- kind: Pod
- name: test-nifi-node-default-2
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: test-nifi-node-default
-status:
- readyReplicas: 3
- replicas: 3
diff --git a/tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2 b/tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2
deleted file mode 100644
index f39ce021..00000000
--- a/tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2
+++ /dev/null
@@ -1,9 +0,0 @@
----
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
- - script: >-
- kubectl --namespace $NAMESPACE
- patch nificlusters.nifi.stackable.tech test-nifi
- --type=merge --patch '{"spec":{"config": {"authentication": {"allowAnonymousAccess": true}}}}'
- - command: kubectl rollout restart statefulset test-nifi-node-default --namespace $NAMESPACE
diff --git a/tests/templates/kuttl/smoke/test_nifi.py b/tests/templates/kuttl/smoke/test_nifi.py
index b885c09f..9d89fb63 100755
--- a/tests/templates/kuttl/smoke/test_nifi.py
+++ b/tests/templates/kuttl/smoke/test_nifi.py
@@ -42,7 +42,7 @@ def get_token(nifi_host, username, password):
# disable warnings as we have specified non-verified https connections
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
- host = f"https://test-nifi-node-default-1.test-nifi-node-default.{args['namespace']}.svc.cluster.local:8443"
+ host = f"https://nifi-node-default-1.nifi-node-default.{args['namespace']}.svc.cluster.local:8443"
token = get_token(host, args['user'], args['password'])
headers = {'Authorization': token}
node_count = int(args['count'])
diff --git a/tests/templates/kuttl/smoke/test_nifi_metrics.py b/tests/templates/kuttl/smoke/test_nifi_metrics.py
index 86076038..2b0682e1 100755
--- a/tests/templates/kuttl/smoke/test_nifi_metrics.py
+++ b/tests/templates/kuttl/smoke/test_nifi_metrics.py
@@ -23,7 +23,7 @@
port = args["port"]
timeout = int(args["timeout"])
- url = f"http://test-nifi-node-default-0.test-nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics"
+ url = f"http://nifi-node-default-0.nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics"
# wait for 'timeout' seconds
t_end = time.time() + timeout
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
index 176bd07a..55e51bc1 100644
--- a/tests/test-definition.yaml
+++ b/tests/test-definition.yaml
@@ -13,22 +13,36 @@ dimensions:
- 1.27.0
- name: nifi_new
values:
- - 2.4.0
- # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
- # as in the example below.
- # - 2.4.0,oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev
+ - 2.4.0 # oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev
- name: nifi-latest
values:
- - 2.4.0
- # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
- # as in the example below.
- # - 2.4.0,oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev
+ - 2.4.0 # oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev
+ - name: nifi-iceberg
+ # Not all NiFi versions support Iceberg with the same functionality!
+ # E.g. our own implementation started with NiFi 2.2.0
+ values:
+ - 2.4.0 # oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev
- name: zookeeper
values:
- 3.9.3
- name: zookeeper-latest
values:
- 3.9.3
+ - name: opa-l
+ values:
+ - 1.4.2
+ - name: hdfs-l
+ values:
+ - 3.4.1
+ - name: hive-l
+ values:
+ - 4.0.1
+ - name: trino-l
+ values:
+ - "470"
+ - name: krb5
+ values:
+ - 1.21.1
- name: ldap-use-tls
values:
- "false"
@@ -37,6 +51,13 @@ dimensions:
values:
- "false"
- "true"
+ - name: iceberg-use-kerberos
+ values:
+ - "false"
+ - "true"
+ - name: kerberos-realm
+ values:
+ - "PROD.MYCORP"
- name: openshift
values:
- "false"
@@ -90,6 +111,18 @@ tests:
- zookeeper-latest
- oidc-use-tls
- openshift
+ - name: iceberg
+ dimensions:
+ - nifi-iceberg
+ - opa-l
+ - zookeeper-latest
+ - hdfs-l
+ - hive-l
+ - trino-l
+ - krb5
+ - iceberg-use-kerberos
+ - kerberos-realm
+ - openshift
suites:
- name: nightly
patch: