chore: Add kuttl smoke test (#79)

sbernauer · web-flow · commit 05154a89d56a · 2025-09-15T11:33:50.000Z
* chore: Add smoke kuttl test

* Add kuttl test for routing

* Update Trino versions

* Switch cert Secret to stringData
diff --git a/scripts/run-tests b/scripts/run-tests
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+./scripts/run-tests "$@"
diff --git a/tests/.gitignore b/tests/.gitignore
@@ -0,0 +1 @@
+_work/
diff --git a/tests/kuttl-test.yaml.jinja2 b/tests/kuttl-test.yaml.jinja2
@@ -0,0 +1,28 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+testDirs:
+  {% for testcase in testinput.tests %}
+  - ./tests/{{ testcase.name }}
+  {% endfor %}
+
+startKIND: false
+suppress: ["events"]
+parallel: 2
+
+# The timeout (in seconds) is used when namespaces are created or
+# deleted, and, if not overridden, in TestSteps, TestAsserts, and
+# Commands. If not set, the timeout is 30 seconds by default.
+#
+# The deletion of a namespace can take a while until all resources,
+# especially PersistentVolumeClaims, are gracefully shut down. If the
+# timeout is reached in the meantime, even a successful test case is
+# considered a failure.
+#
+# For instance, the termination grace period of the Vector aggregator in
+# the logging tests is set to 60 seconds. If there are logs entries
+# which could not be forwarded yet to the external aggregator defined in
+# the VECTOR_AGGREGATOR environment variable, then the test aggregator
+# uses this period of time by trying to forward the events. In this
+# case, deleting a namespace with several Pods takes about 90 seconds.
+timeout: 300
diff --git a/tests/release.yaml b/tests/release.yaml
@@ -0,0 +1,16 @@
+# Contains all operators required to run the test suite.
+---
+releases:
+  # Do not change the name of the release as it's referenced from run-tests
+  tests:
+    releaseDate: 1970-01-01
+    description: Integration test
+    products:
+      commons:
+        operatorVersion: 0.0.0-dev
+      secret:
+        operatorVersion: 0.0.0-dev
+      listener:
+        operatorVersion: 0.0.0-dev
+      trino:
+        operatorVersion: 0.0.0-dev
diff --git a/tests/templates/kuttl/smoke/00-assert.yaml.j2 b/tests/templates/kuttl/smoke/00-assert.yaml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+{% endif %}
diff --git a/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2
@@ -0,0 +1,9 @@
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+data:
+  ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+{% endif %}
diff --git a/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 b/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2
@@ -0,0 +1,9 @@
+{% if test_scenario['values']['openshift'] == 'true' %}
+# see https://github.com/stackabletech/issues/issues/566
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
+    timeout: 120
+{% endif %}
diff --git a/tests/templates/kuttl/smoke/00-rbac.yaml.j2 b/tests/templates/kuttl/smoke/00-rbac.yaml.j2
@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: trino-lb
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: trino-lb
+subjects:
+  - kind: ServiceAccount
+    name: trino-lb
+roleRef:
+  kind: Role
+  name: trino-lb
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: trino-lb
+  labels:
+    app.kubernetes.io/name: trino-lb
+rules:
+  - apiGroups:
+      - trino.stackable.tech
+    resources:
+      - trinoclusters
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+  - apiGroups:
+      - trino.stackable.tech
+    resources:
+      - trinoclusters/status
+    verbs:
+      - get
diff --git a/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml b/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml
@@ -0,0 +1,58 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: trino-lb-certificates
+# Copied from example-configs/self-singed-certs
+stringData:
+  cert.pem: |
+    -----BEGIN CERTIFICATE-----
+    MIIDkzCCAnugAwIBAgIUXVYkRCrM/ge03DVymDtXCuybp7gwDQYJKoZIhvcNAQEL
+    BQAwWTELMAkGA1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+    GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X
+    DTIxMDczMTE0MjIxMloXDTIyMDczMTE0MjIxMlowWTELMAkGA1UEBhMCVVMxEzAR
+    BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5
+    IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+    MIIBCgKCAQEA02V5ZjmqLB/VQwTarrz/35qsa83L+DbAoa0001+jVmmC+G9Nufi0
+    daroFWj/Uicv2fZWETU8JoZKUrX4BK9og5cg5rln/CtBRWCUYIwRgY9R/CdBGPn4
+    kp+XkSJaCw74ZIyLy/Zfux6h8ES1m9YRnBza+s7U+ImRBRf4MRPtXQ3/mqJxAZYq
+    dOnKnvssRyD2qutgVTAxwMUvJWIivRhRYDj7WOpS4CEEeQxP1iH1/T5P7FdtTGdT
+    bVBABCA8JhL96uFGPpOYHcM/7R5EIA3yZ5FNg931QzoDITjtXGtQ6y9/l/IYkWm6
+    J67RWcN0IoTsZhz0WNU4gAeslVtJLofn8QIDAQABo1MwUTAdBgNVHQ4EFgQUzFnK
+    NfS4LAYuKeWwHbzooER0yZ0wHwYDVR0jBBgwFoAUzFnKNfS4LAYuKeWwHbzooER0
+    yZ0wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAk4O+e9jia59W
+    ZwetN4GU7OWcYhmOgSizRSs6u7mTfp62LDMt96WKU3THksOnZ44HnqWQxsSfdFVU
+    XJD12tjvVU8Z4FWzQajcHeemUYiDze8EAh6TnxnUcOrU8IcwiKGxCWRY/908jnWg
+    +MMscfMCMYTRdeTPqD8fGzAlUCtmyzH6KLE3s4Oo/r5+NR+Uvrwpdvb7xe0MwwO9
+    Q/zR4N8ep/HwHVEObcaBofE1ssZLksX7ZgCP9wMgXRWpNAtC5EWxMbxYjBfWFH24
+    fDJlBMiGJWg8HHcxK7wQhFh+fuyNzE+xEWPsI9VL1zDftd9x8/QsOagyEOnY8Vxr
+    AopvZ09uEQ==
+    -----END CERTIFICATE-----
+  key.pem: |
+    -----BEGIN PRIVATE KEY-----
+    MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDTZXlmOaosH9VD
+    BNquvP/fmqxrzcv4NsChrTTTX6NWaYL4b025+LR1qugVaP9SJy/Z9lYRNTwmhkpS
+    tfgEr2iDlyDmuWf8K0FFYJRgjBGBj1H8J0EY+fiSn5eRIloLDvhkjIvL9l+7HqHw
+    RLWb1hGcHNr6ztT4iZEFF/gxE+1dDf+aonEBlip06cqe+yxHIPaq62BVMDHAxS8l
+    YiK9GFFgOPtY6lLgIQR5DE/WIfX9Pk/sV21MZ1NtUEAEIDwmEv3q4UY+k5gdwz/t
+    HkQgDfJnkU2D3fVDOgMhOO1ca1DrL3+X8hiRabonrtFZw3QihOxmHPRY1TiAB6yV
+    W0kuh+fxAgMBAAECggEADltu8k1qTFLhJgsXWxTFAAe+PBgfCT2WuaRM2So+qqjB
+    12Of0MieYPt5hbK63HaC3nfHgqWt7yPhulpXfOH45C8IcgMXl93MMg0MJr58leMI
+    +2ojFrIrerHSFm5R1TxwDEwrVm/mMowzDWFtQCc6zPJ8wNn5RuP48HKfTZ3/2fjw
+    zEjSwPO2wFMfo1EJNTjlI303lFbdFBs67NaX6puh30M7Tn+gznHKyO5a7F57wkIt
+    fkgnEy/sgMedQlwX7bRpUoD6f0fZzV8Qz4cHFywtYErczZJh3VGitJoO/VCIDdty
+    RPXOAqVDd7EpP1UUehZlKVWZ0OZMEfRgKbRCel5abQKBgQDwgwrIQ5+BiZv6a0VT
+    ETeXB+hRbvBinRykNo/RvLc3j1enRh9/zO/ShadZIXgOAiM1Jnr5Gp8KkNGca6K1
+    myhtad7xYPODYzNXXp6T1OPgZxHZLIYzVUj6ypXeV64Te5ZiDaJ1D49czsq+PqsQ
+    XRcgBJSNpFtDFiXWpjXWfx8PxwKBgQDhAnLY5Sl2eeQo+ud0MvjwftB/mN2qCzJY
+    5AlQpRI4ThWxJgGPuHTR29zVa5iWNYuA5LWrC1y/wx+t5HKUwq+5kxvs+npYpDJD
+    ZX/w0Glc6s0Jc/mFySkbw9B2LePedL7lRF5OiAyC6D106Sc9V2jlL4IflmOzt4CD
+    ZTNbLtC6hwKBgHfIzBXxl/9sCcMuqdg1Ovp9dbcZCaATn7ApfHd5BccmHQGyav27
+    k7XF2xMJGEHhzqcqAxUNrSgV+E9vTBomrHvRvrd5Ec7eGTPqbBA0d0nMC5eeFTh7
+    wV0miH20LX6Gjt9G6yJiHYSbeV5G1+vOcTYBEft5X/qJjU7aePXbWh0BAoGBAJlV
+    5tgCCuhvFloK6fHYzqZtdT6O+PfpW20SMXrgkvMF22h2YvgDFrDwqKRUB47NfHzg
+    3yBpxNH1ccA5/w97QO8w3gX3h6qicpJVOAPusu6cIBACFZfjRv1hyszOZwvw+Soa
+    Fj5kHkqTY1YpkREPYS9V2dIW1Wjic1SXgZDw7VM/AoGAP/cZ3ZHTSCDTFlItqy5C
+    rIy2AiY0WJsx+K0qcvtosPOOwtnGjWHb1gdaVdfX/IRkSsX4PAOdnsyidNC5/l/m
+    y8oa+5WEeGFclWFhr4dnTA766o8HrM2UjIgWWYBF2VKdptGnHxFeJWFUmeQC/xeW
+    w37pCS7ykL+7gp7V0WShYsw=
+    -----END PRIVATE KEY-----
diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 720
+commands:
+  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-s-1 --timeout 719s
+  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-m-1 --timeout 719s
diff --git a/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2 b/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: trino.stackable.tech/v1alpha1
+kind: TrinoCatalog
+metadata:
+  name: tpch
+  labels:
+    trino: trino
+spec:
+  connector:
+    tpch: {}
diff --git a/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2 b/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2
@@ -0,0 +1,72 @@
+{% for name in ['trino-s-1', 'trino-m-1'] %}
+---
+apiVersion: trino.stackable.tech/v1alpha1
+kind: TrinoCluster
+metadata:
+  name: {{ name }}
+spec:
+  image:
+{% if test_scenario['values']['trino'].find(",") > 0 %}
+    custom: "{{ test_scenario['values']['trino'].split(',')[1] }}"
+    productVersion: "{{ test_scenario['values']['trino'].split(',')[0] }}"
+{% else %}
+    productVersion: "{{ test_scenario['values']['trino'] }}"
+{% endif %}
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    catalogLabelSelector:
+      matchLabels:
+        trino: trino
+    authentication:
+      - authenticationClass: trino-users-auth
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+    vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  coordinators:
+    config:
+      resources:
+        cpu:
+          min: 250m
+          max: "1"
+        memory:
+          limit: 2Gi
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 1
+  workers:
+    config:
+      resources:
+        cpu:
+          min: 250m
+          max: "1"
+        memory:
+          limit: 3Gi
+      gracefulShutdownTimeout: 60s # Let the test run faster
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 1
+{% endfor %}
+---
+apiVersion: authentication.stackable.tech/v1alpha1
+kind: AuthenticationClass
+metadata:
+  name: trino-users-auth
+spec:
+  provider:
+    static:
+      userCredentialsSecret:
+        name: trino-users
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: trino-users
+type: kubernetes.io/opaque
+stringData:
+  admin: adminadmin
+  alice: alicealice
+  bob: bobbob
diff --git a/tests/templates/kuttl/smoke/20-assert.yaml b/tests/templates/kuttl/smoke/20-assert.yaml
@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trino-lb
+status:
+  replicas: 3
+  readyReplicas: 3
diff --git a/tests/templates/kuttl/smoke/20-install-trino-lb.j2 b/tests/templates/kuttl/smoke/20-install-trino-lb.j2
@@ -0,0 +1,74 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl -n $NAMESPACE create secret generic trino-lb-config --from-file=trino-lb-config.yaml=20_trino-lb-config.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trino-lb
+spec:
+  # I'm a bit surprised that 3 replicas work in combination with inMemory persistence :)
+  # Maybe Kubernetes is sticky for some reason, maybe trino-clients will retry call until they get
+  # to the correct trino-lb instance...
+  replicas: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: trino-lb
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: trino-lb
+    spec:
+      containers:
+        - name: trino-lb
+          image: {{ test_scenario['values']['trino-lb'] }}
+          imagePullPolicy: IfNotPresent
+          command: ["trino-lb", "--config-file", "/etc/stackable/trino-lb/config/trino-lb-config.yaml"]
+          ports:
+            - containerPort: 8080
+            - containerPort: 8443
+            - containerPort: 9090
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          volumeMounts:
+            - mountPath: /etc/stackable/trino-lb/config/
+              name: config
+            - mountPath: /certificates/
+              name: certificates
+      volumes:
+        - name: config
+          secret:
+            secretName: trino-lb-config
+        - name: certificates
+          secret:
+            secretName: trino-lb-certificates
+      serviceAccountName: trino-lb
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: trino-lb
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: trino-lb
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+      name: http
+    - protocol: TCP
+      port: 8443
+      targetPort: 8443
+      name: https
+    - protocol: TCP
+      port: 9090
+      targetPort: 9090
+      name: metrics
diff --git a/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2
@@ -0,0 +1,22 @@
+trinoLb:
+  externalAddress: https://trino-lb:8443
+  tls:
+    enabled: true
+    certPemFile: /certificates/cert.pem
+    keyPemFile: /certificates/key.pem
+  persistence:
+    inMemory: {}
+trinoClusterGroups:
+  default:
+    maxRunningQueries: 1
+    trinoClusters:
+      - name: trino-s-1
+        endpoint: https://trino-s-1-coordinator:8443
+        credentials:
+          username: admin
+          password: adminadmin
+trinoClusterGroupsIgnoreCert: true
+
+# Route all queries to the "default" cluster group
+routers: []
+routingFallback: default
diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-queries
+status:
+  succeeded: 1
diff --git a/tests/templates/kuttl/smoke/30-test-queries.yaml.j2 b/tests/templates/kuttl/smoke/30-test-queries.yaml.j2
diff --git a/tests/templates/kuttl/smoke/40-change-routing.j2 b/tests/templates/kuttl/smoke/40-change-routing.j2
diff --git a/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2
diff --git a/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml b/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml
diff --git a/tests/templates/kuttl/smoke/50-assert.yaml b/tests/templates/kuttl/smoke/50-assert.yaml
diff --git a/tests/templates/kuttl/smoke/50-test-routing.yaml.j2 b/tests/templates/kuttl/smoke/50-test-routing.yaml.j2
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+#!/usr/bin/env bash`
	`2`	`+`
	`3`	`+./scripts/run-tests "$@"`