test: Add test for Apache Iceberg integration

tests/release.yaml

@@ -14,5 +14,9 @@ releases:
         operatorVersion: 0.0.0-dev
       zookeeper:
         operatorVersion: 0.0.0-dev
+      hive:
+        operatorVersion: 0.0.0-dev
+      trino:
+        operatorVersion: 0.0.0-dev
       nifi:
         operatorVersion: 0.0.0-dev

tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2

@@ -0,0 +1,9 @@
+{% if test_scenario['values']['openshift'] == 'true' %}
+# see https://github.com/stackabletech/issues/issues/566
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
+    timeout: 120
+{% endif %}

tests/templates/kuttl/iceberg/01-create-s3-connection.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: envsubst '$NAMESPACE' < 01_s3-connection.yaml | kubectl apply -n $NAMESPACE -f -

tests/templates/kuttl/iceberg/01_s3-connection.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: s3.stackable.tech/v1alpha1
+kind: S3Connection
+metadata:
+  name: minio
+spec:
+  host: "minio.${NAMESPACE}.svc.cluster.local"
+  port: 9000
+  accessStyle: Path
+  credentials:
+    secretClass: s3-credentials-class
+  tls:
+    verification:
+      server:
+        caCert:
+          secretClass: tls
+---
+apiVersion: secrets.stackable.tech/v1alpha1
+kind: SecretClass
+metadata:
+  name: s3-credentials-class
+spec:
+  backend:
+    k8sSearch:
+      searchNamespace:
+        pod: {}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-credentials
+  labels:
+    secrets.stackable.tech/class: s3-credentials-class
+stringData:
+  accessKey: admin
+  secretKey: adminadmin

tests/templates/kuttl/iceberg/10-assert.yaml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+{% endif %}

tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2

@@ -0,0 +1,9 @@
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+data:
+  ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+{% endif %}

tests/templates/kuttl/iceberg/20-assert.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+status:
+  readyReplicas: 1
+  replicas: 1

tests/templates/kuttl/iceberg/20-install-minio.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl -n $NAMESPACE apply -f https://raw.githubusercontent.com/stackabletech/demos/refs/heads/release-25.3/stacks/_templates/minio-tls/rendered-chart.yaml

tests/templates/kuttl/iceberg/21-assert.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: postgresql
+status:
+  readyReplicas: 1
+  replicas: 1

tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 300
+commands:
+  - script: >-
+      helm upgrade postgresql
+      --install
+      --version=12.5.6
+      --namespace $NAMESPACE
+      -f 21_helm-bitnami-postgresql-values.yaml
+      --repo https://charts.bitnami.com/bitnami postgresql

tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2

@@ -0,0 +1,29 @@
+---
+volumePermissions:
+  enabled: false
+  securityContext:
+    runAsUser: auto
+
+primary:
+  extendedConfiguration: |
+    password_encryption=md5
+  podSecurityContext:
+{% if test_scenario['values']['openshift'] == 'true' %}
+    enabled: false
+{% else %}
+    enabled: true
+{% endif %}
+  containerSecurityContext:
+    enabled: false
+  resources:
+    requests:
+      memory: "512Mi"
+      cpu: "512m"
+    limits:
+      memory: "512Mi"
+      cpu: "1"
+
+auth:
+  username: hive
+  password: hive
+  database: hive

tests/templates/kuttl/iceberg/30-assert.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 900
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: hive-metastore-default
+status:
+  readyReplicas: 2
+  replicas: 2

tests/templates/kuttl/iceberg/30-install-hive.yaml.j2

@@ -0,0 +1,42 @@
+---
+apiVersion: hive.stackable.tech/v1alpha1
+kind: HiveCluster
+metadata:
+  name: hive
+spec:
+  image:
+{% if test_scenario['values']['hive-latest'].find(",") > 0 %}
+    custom: "{{ test_scenario['values']['hive-latest'].split(',')[1] }}"
+    productVersion: "{{ test_scenario['values']['hive-latest'].split(',')[0] }}"
+{% else %}
+    productVersion: "{{ test_scenario['values']['hive-latest'] }}"
+{% endif %}
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    database:
+      connString: jdbc:postgresql://postgresql:5432/hive
+      credentialsSecret: postgres-credentials
+      dbType: postgres
+    # hdfs:
+    #   configMap: hdfs
+    s3:
+      reference: minio
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+    vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  metastore:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 2
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-credentials
+type: Opaque
+stringData:
+  username: hive
+  password: hive

tests/templates/kuttl/iceberg/31-assert.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 720
+commands:
+  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s

tests/templates/kuttl/iceberg/31-install-trino.yaml.j2

@@ -0,0 +1,53 @@
+---
+apiVersion: trino.stackable.tech/v1alpha1
+kind: TrinoCluster
+metadata:
+  name: trino
+spec:
+  image:
+{% if test_scenario['values']['trino-latest'].find(",") > 0 %}
+    custom: "{{ test_scenario['values']['trino-latest'].split(',')[1] }}"
+    productVersion: "{{ test_scenario['values']['trino-latest'].split(',')[0] }}"
+{% else %}
+    productVersion: "{{ test_scenario['values']['trino-latest'] }}"
+{% endif %}
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    catalogLabelSelector:
+      matchLabels:
+        trino: trino
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+    vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  coordinators:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 1
+  workers:
+    config:
+      gracefulShutdownTimeout: 60s # Let the test run faster
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 1
+        config: {}
+---
+apiVersion: trino.stackable.tech/v1alpha1
+kind: TrinoCatalog
+metadata:
+  name: iceberg
+  labels:
+    trino: trino
+spec:
+  connector:
+    iceberg:
+      metastore:
+        configMap: hive
+      s3:
+        reference: minio
+      # hdfs:
+      #   configMap: hdfs

tests/templates/kuttl/iceberg/32-assert.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-iceberg-table
+status:
+  succeeded: 1

tests/templates/kuttl/iceberg/32-create-iceberg-table.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-iceberg-table
+spec:
+  template:
+    spec:
+      containers:
+        - name: create-iceberg-table
+          image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-latest'] }}-stackable0.0.0-dev"
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              cat << 'EOF' | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin
+              CREATE SCHEMA IF NOT EXISTS iceberg.test WITH (location = 's3a://demo/lakehouse/test');
+              CREATE TABLE IF NOT EXISTS iceberg.test.greetings (hello varchar);
+              EOF
+      restartPolicy: OnFailure

tests/templates/kuttl/iceberg/40-assert.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: zookeeper-server-default
+status:
+  readyReplicas: 1
+  replicas: 1

tests/templates/kuttl/iceberg/40-install-zookeeper.yaml.j2

@@ -0,0 +1,21 @@
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperCluster
+metadata:
+  name: zookeeper
+spec:
+  image:
+    productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}"
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    listenerClass: {{ test_scenario['values']['listener-class'] }}
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+    vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  servers:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 1

tests/templates/kuttl/iceberg/50-assert.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 1200
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: nifi-node-default
+status:
+  readyReplicas: 1
+  replicas: 1