From 7a63cb45926548c06802a69cc7c2cc6ab6d3c6aa Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 17 Jul 2025 14:24:02 +0200 Subject: [PATCH 1/3] update template component --- tests/templates/kuttl/iceberg/20-install-minio.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/iceberg/20-install-minio.yaml b/tests/templates/kuttl/iceberg/20-install-minio.yaml index bbef7238..52f37d71 100644 --- a/tests/templates/kuttl/iceberg/20-install-minio.yaml +++ b/tests/templates/kuttl/iceberg/20-install-minio.yaml @@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl -n $NAMESPACE apply -f https://raw.githubusercontent.com/stackabletech/demos/refs/heads/release-25.3/stacks/_templates/minio-tls/rendered-chart.yaml + - script: kubectl -n $NAMESPACE apply -f https://raw.githubusercontent.com/stackabletech/demos/refs/heads/main/stacks/_templates/minio-tls/rendered-chart.yaml From 9fabe6695d2a329f9d7de2a66e43676cc10996ae Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 17 Jul 2025 18:09:28 +0200 Subject: [PATCH 2/3] wip: change to pattern used in other tests, not yet working --- tests/templates/kuttl/iceberg/00-secrets.yaml | 22 ++++++++ .../kuttl/iceberg/01_s3-connection.yaml | 4 +- tests/templates/kuttl/iceberg/20-assert.yaml | 7 --- .../kuttl/iceberg/20-install-minio.yaml | 8 ++- .../iceberg/helm-bitnami-minio-values.yaml | 55 +++++++++++++++++++ 5 files changed, 87 insertions(+), 9 deletions(-) create mode 100644 tests/templates/kuttl/iceberg/00-secrets.yaml create mode 100644 tests/templates/kuttl/iceberg/helm-bitnami-minio-values.yaml diff --git a/tests/templates/kuttl/iceberg/00-secrets.yaml b/tests/templates/kuttl/iceberg/00-secrets.yaml new file mode 100644 index 00000000..a89967c4 --- /dev/null +++ b/tests/templates/kuttl/iceberg/00-secrets.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: minio-tls-certificates +spec: + backend: + k8sSearch: + searchNamespace: + pod: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-tls-certificates + labels: + secrets.stackable.tech/class: minio-tls-certificates +# Have a look at the folder certs on how to create this +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQyVENDQXNHZ0F3SUJBZ0lVTmpxdUdZV3R5SjVhNnd5MjNIejJHUmNNbHdNd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2V6RUxNQWtHQTFVRUJoTUNSRVV4R3pBWkJnTlZCQWdNRWxOamFHeGxjM2RwWnkxSWIyeHpkR1ZwYmpFTwpNQXdHQTFVRUJ3d0ZWMlZrWld3eEtEQW1CZ05WQkFvTUgxTjBZV05yWVdKc1pTQlRhV2R1YVc1bklFRjFkR2h2CmNtbDBlU0JKYm1NeEZUQVRCZ05WQkFNTURITjBZV05yWVdKc1pTNWtaVEFnRncweU16QTJNVFl4TWpVeE1ESmEKR0E4eU1USXpNRFV5TXpFeU5URXdNbG93ZXpFTE1Ba0dBMVVFQmhNQ1JFVXhHekFaQmdOVkJBZ01FbE5qYUd4bApjM2RwWnkxSWIyeHpkR1ZwYmpFT01Bd0dBMVVFQnd3RlYyVmtaV3d4S0RBbUJnTlZCQW9NSDFOMFlXTnJZV0pzClpTQlRhV2R1YVc1bklFRjFkR2h2Y21sMGVTQkpibU14RlRBVEJnTlZCQU1NREhOMFlXTnJZV0pzWlM1a1pUQ0MKQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFOblYvdmJ5M1JvNTdhMnF2UVJubjBqZQplS01VMitGMCtsWk5DQXZpR1VENWJtOGprOTFvUFpuazBiaFFxZXlFcm1EUzRXVDB6ZXZFUklCSkpEamZMMEQ4CjQ2QmU3UGlNS2UwZEdqb3FJM3o1Y09JZWpjOGFMUEhTSWxnTjZsVDNmSXJ1UzE2Y29RZ0c0dWFLaUhGNStlV0YKRFJVTGR1NmRzWXV6NmRLanFSaVVPaEh3RHd0VUprRHdQditFSXRxbzBIK01MRkxMWU0wK2xFSWFlN2RONUNRNQpTbzVXaEwyY3l2NVZKN2xqL0VBS0NWaUlFZ0NtekRSRGNSZ1NTald5SDRibjZ5WDIwMjZmUEl5V0pGeUVkTC82CmpBT0pBRERSMEd5aE5PWHJFZXFob2NTTW5JYlFWcXdBVDBrTWh1WFN2d3Zscm5MeVRwRzVqWm00bFVNMzRrTUMKQXdFQUFhTlRNRkV3SFFZRFZSME9CQllFRkVJM1JNTWl5aUJqeVExUlM4bmxPUkpWZDFwQk1COEdBMVVkSXdRWQpNQmFBRkVJM1JNTWl5aUJqeVExUlM4bmxPUkpWZDFwQk1BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0RRWUpLb1pJCmh2Y05BUUVMQlFBRGdnRUJBSHRLUlhkRmR0VWh0VWpvZG1ZUWNlZEFEaEhaT2hCcEtpbnpvdTRicmRrNEhmaEYKTHIvV0ZsY1JlbWxWNm1Cc0xweU11SytUZDhaVUVRNkpFUkx5NmxTL2M2cE9HeG5CNGFDbEU4YXQrQytUakpBTwpWbTNXU0k2VlIxY0ZYR2VaamxkVlE2eGtRc2tNSnpPN2RmNmlNVFB0VjVSa01lSlh0TDZYYW1FaTU0ckJvZ05ICk5yYStFSkJRQmwvWmU5ME5qZVlidjIwdVFwWmFhWkZhYVNtVm9OSERwQndsYTBvdXkrTWpPYkMzU3BnT3ExSUMKUGwzTnV3TkxWOFZiT3I1SHJoUUFvS21nU05iM1A4dmFUVnV4L1gwWWZqeS9TN045a1BCYUs5bUZqNzR6d1Y5dwpxU1ExNEtsNWpPM1YzaHJHV1laRWpET2diWnJyRVgxS1hFdXN0K1E9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR5RENDQXJDZ0F3SUJBZ0lVQ0kyUE5OcnR6cDZRbDdHa3VhRnhtRGE2VUJvd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2V6RUxNQWtHQTFVRUJoTUNSRVV4R3pBWkJnTlZCQWdNRWxOamFHeGxjM2RwWnkxSWIyeHpkR1ZwYmpFTwpNQXdHQTFVRUJ3d0ZWMlZrWld3eEtEQW1CZ05WQkFvTUgxTjBZV05yWVdKc1pTQlRhV2R1YVc1bklFRjFkR2h2CmNtbDBlU0JKYm1NeEZUQVRCZ05WQkFNTURITjBZV05yWVdKc1pTNWtaVEFnRncweU16QTJNVFl4TWpVeE1ESmEKR0E4eU1USXpNRFV5TXpFeU5URXdNbG93WGpFTE1Ba0dBMVVFQmhNQ1JFVXhHekFaQmdOVkJBZ01FbE5qYUd4bApjM2RwWnkxSWIyeHpkR1ZwYmpFT01Bd0dBMVVFQnd3RlYyVmtaV3d4RWpBUUJnTlZCQW9NQ1ZOMFlXTnJZV0pzClpURU9NQXdHQTFVRUF3d0ZiV2x1YVc4d2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUIKQVFDanluVnorWEhCOE9DWTRwc0VFWW1qb2JwZHpUbG93d2NTUU4rWURQQ2tCZW9yMFRiODdFZ0x6SksrSllidQpwb1hCbE5JSlBRYW93SkVvL1N6U2s4ZnUyWFNNeXZBWlk0RldHeEp5Mnl4SXh2UC9pYk9HT1l1aVBHWEsyNHQ2ClpjR1RVVmhhdWlaR1Nna1dyZWpXV2g3TWpGUytjMXZhWVpxQitRMXpQczVQRk1sYzhsNVYvK2I4WjdqTUppODQKbU9mSVB4amt2SXlKcjVVa2VGM1VmTHFKUzV5NExGNHR5NEZ0MmlBZDdiYmZIYW5mdlltdjZVb0RWdE1YdFdvMQpvUVBmdjNzaFdybVJMenc2ZXVJQXRiWGM1Q2pCeUlha0NiaURuQVU4cktnK0IxSjRtdlFnckx3bzNxUHJ5Smd4ClNkaWRtWjJtRVI3RXorYzVCMG0vTGlJaEFnTUJBQUdqWHpCZE1Cc0dBMVVkRVFRVU1CS0NCVzFwYm1sdmdnbHMKYjJOaGJHaHZjM1F3SFFZRFZSME9CQllFRkpRMGdENWtFdFFyK3REcERTWjdrd1o4SDVoR01COEdBMVVkSXdRWQpNQmFBRkVJM1JNTWl5aUJqeVExUlM4bmxPUkpWZDFwQk1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQmNkaGQrClI0Sm9HdnFMQms1OWRxSVVlY2N0dUZzcmRQeHNCaU9GaFlOZ1pxZWRMTTBVTDVEenlmQUhmVk8wTGZTRURkZFgKUkpMOXlMNytrTVUwVDc2Y3ZkQzlYVkFJRTZIVXdUbzlHWXNQcXN1eVpvVmpOcEVESkN3WTNDdm9ubEpWZTRkcQovZ0FiSk1ZQitUU21ZNXlEUHovSkZZL1haellhUGI3T2RlR3VqYlZUNUl4cDk3QXBTOFlJaXY3M0Mwd1ViYzZSCmgwcmNmUmJ5a1NRVWg5dmdWZFhSU1I4RFQzV0NmZHFOek5CWVh2OW1xZlc1ejRzYkdqK2wzd1VsL0kzRi9tSXcKZnlPNEN0aTRha2lHVkhsZmZFeTB3a3pWYUJ4aGNYajJJM0JVVGhCNFpxamxzc2llVmFGa3d2WG1teVJUMG9FVwo1SCtOUEhjcXVTMXpQc2NsCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2QUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktZd2dnU2lBZ0VBQW9JQkFRQ2p5blZ6K1hIQjhPQ1kKNHBzRUVZbWpvYnBkelRsb3d3Y1NRTitZRFBDa0Jlb3IwVGI4N0VnTHpKSytKWWJ1cG9YQmxOSUpQUWFvd0pFbwovU3pTazhmdTJYU015dkFaWTRGV0d4SnkyeXhJeHZQL2liT0dPWXVpUEdYSzI0dDZaY0dUVVZoYXVpWkdTZ2tXCnJlaldXaDdNakZTK2MxdmFZWnFCK1ExelBzNVBGTWxjOGw1Vi8rYjhaN2pNSmk4NG1PZklQeGprdkl5SnI1VWsKZUYzVWZMcUpTNXk0TEY0dHk0RnQyaUFkN2JiZkhhbmZ2WW12NlVvRFZ0TVh0V28xb1FQZnYzc2hXcm1STHp3NgpldUlBdGJYYzVDakJ5SWFrQ2JpRG5BVThyS2crQjFKNG12UWdyTHdvM3FQcnlKZ3hTZGlkbVoybUVSN0V6K2M1CkIwbS9MaUloQWdNQkFBRUNnZ0VBQWQzdDVzdUNFMjdXY0llc3NxZ3NoSFAwZHRzKyswVzF6K3h6WC8xTnhPRFkKWVhWNkJmbi9mRHJ4dFQ4aVFaZ2VVQzJORTFQaHZveXJXdWMvMm9xYXJjdEd1OUFZV29HNjJLdG9VMnpTSFdZLwpJN3VERTFXV2xOdlJZVFdOYW5DOGV4eGpRRzE4d0RKWjFpdFhTeEl0NWJEM3lrL3dUUlh0dCt1SnpyVjVqb2N1CmNoeERMd293aXUxQWo2ZFJDWk5CejlUSnh5TnI1ME5ZVzJVWEJhVC84N1hyRkZkSndNVFZUMEI3SE9uRzdSQlYKUWxLdzhtcVZiYU5lbmhjdk1qUjI5c3hUekhSK2p4SU8zQndPNk9Hai9PRmhGQllVN1RMWGVsZDFxb2UwdmIyRwpiOGhQcEd1cHRyNUF0OWx3MXc1d1EzSWdpdXRQTkg1cXlEeUNwRWw2RVFLQmdRRGNkYnNsT2ZLSmo3TzJMQXlZCkZ0a1RwaWxFMFYzajBxbVE5M0lqclY0K0RSbUxNRUIyOTk0MDdCVVlRUWoxL0RJYlFjb1oyRUVjVUI1cGRlSHMKN0RNRUQ2WExIYjJKVTEyK2E3c1d5Q05kS2VjZStUNy9JYmxJOFR0MzQwVWxIUTZ6U01TRGNqdmZjRkhWZ3YwcwpDYWpoRng3TmtMRVhUWnI4ZlQzWUloajR2UUtCZ1FDK01nWjFVbW9KdzlJQVFqMnVJVTVDeTl4aldlWURUQU8vCllhWEl6d2xnZTQzOE1jYmI0Y04yU2FOU0dEZ1Y3bnU1a3FpaWhwalBZV0lpaU9CcDlrVFJIWE9kUFc0N3N5ZUkKdDNrd3JwMnpWbFVnbGNNWlo2bW1WM1FWYUFOWmdqVTRSU3Y0ZS9WeFVMamJaYWZqUHRaUnNqWkdwSzBZVTFvdApWajhJZVE3Zk5RS0JnQ1ArWk11ekpsSW5VQ1FTRlF4UHpxbFNtN0pNckpPaHRXV2h3TlRxWFZTc050dHV5VmVqCktIaGpneDR1b0JQcFZSVDJMTlVEWmI0RnByRjVPYVhBK3FOVEdyS0s3SU1iUlZidHArSVVVeEhHNGFGQStIUVgKUVhVVFRhNUpRT1RLVmJnWHpWM1lyTVhTUk1valZNcDMyVWJHeTVTc1p2MXpBamJ2QzhYWjYxSFJBb0dBZEJjUQp2aGU1eFpBUzVEbUtjSGkvemlHa3ViZXJuNk9NUGdxYUtJSEdsVytVOExScFR0ajBkNFRtL1Rydk1PUEovVEU1CllVcUtoenBIcmhDaCtjdHBvY0k2U1dXdm5SenpLbzNpbVFaY0Y1VEFqUTBjY3F0RmI5UzlkRHR5bi9YTUNqYWUKYWlNdll5VUVVRll5TFpDelBGWnNycDNoVVpHKzN5RmZoQXB3TzJrQ2dZQkh3WWFQSWRXNld3NytCMmhpbjBvdwpqYTNjZXN2QTRqYU1Qd1NMVDhPTnRVMUdCU01md2N6TWJuUEhMclJ2Qjg3bjlnUGFSMndRR1VtckZFTzNMUFgvCmtSY09HcFlCSHBEWEVqRGhLa1dkUnVMT0ZnNEhMWmRWOEFOWmxRMFZTY0U4dTNkRERVTzg5cEdEbjA4cVRBcmwKeDlreHN1ZEVWcmtlclpiNVV4RlZxUT09Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K diff --git a/tests/templates/kuttl/iceberg/01_s3-connection.yaml b/tests/templates/kuttl/iceberg/01_s3-connection.yaml index 56c30d36..c095461e 100644 --- a/tests/templates/kuttl/iceberg/01_s3-connection.yaml +++ b/tests/templates/kuttl/iceberg/01_s3-connection.yaml @@ -13,7 +13,7 @@ spec: verification: server: caCert: - secretClass: tls + secretClass: minio-tls-certificates --- apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass @@ -34,3 +34,5 @@ metadata: stringData: accessKey: admin secretKey: adminadmin + root-user: admin + root-password: adminadmin diff --git a/tests/templates/kuttl/iceberg/20-assert.yaml b/tests/templates/kuttl/iceberg/20-assert.yaml index e1829b77..477bdd02 100644 --- a/tests/templates/kuttl/iceberg/20-assert.yaml +++ b/tests/templates/kuttl/iceberg/20-assert.yaml @@ -10,10 +10,3 @@ metadata: status: readyReplicas: 1 replicas: 1 ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job -status: - succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/20-install-minio.yaml b/tests/templates/kuttl/iceberg/20-install-minio.yaml index 52f37d71..0be8b7a3 100644 --- a/tests/templates/kuttl/iceberg/20-install-minio.yaml +++ b/tests/templates/kuttl/iceberg/20-install-minio.yaml @@ -2,4 +2,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl -n $NAMESPACE apply -f https://raw.githubusercontent.com/stackabletech/demos/refs/heads/main/stacks/_templates/minio-tls/rendered-chart.yaml + - script: >- + helm install minio + --namespace $NAMESPACE + --version 12.6.4 + -f helm-bitnami-minio-values.yaml + --repo https://charts.bitnami.com/bitnami minio + timeout: 240 diff --git a/tests/templates/kuttl/iceberg/helm-bitnami-minio-values.yaml b/tests/templates/kuttl/iceberg/helm-bitnami-minio-values.yaml new file mode 100644 index 00000000..071da70e --- /dev/null +++ b/tests/templates/kuttl/iceberg/helm-bitnami-minio-values.yaml @@ -0,0 +1,55 @@ +--- +mode: standalone +disableWebUI: false +extraEnvVars: + - name: BITNAMI_DEBUG + value: "true" + - name: MINIO_LOG_LEVEL + value: DEBUG + +provisioning: + enabled: true + buckets: + - name: demo + policy: public + resources: + requests: + memory: 1Gi + cpu: "512m" + limits: + memory: "1Gi" + cpu: "1" + podSecurityContext: + enabled: false + containerSecurityContext: + enabled: false + +volumePermissions: + enabled: false + +podSecurityContext: + enabled: false + +containerSecurityContext: + enabled: false + +persistence: + enabled: false + +resources: + requests: + memory: 1Gi + cpu: "512m" + limits: + memory: "1Gi" + cpu: "1" + +auth: + existingSecret: minio-credentials + +service: + type: NodePort + +tls: + enabled: true + existingSecret: minio-tls-certificates From f8abc9d4ad25cec5fc65054bd88ff13398ff9a6b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 18 Jul 2025 10:25:07 +0200 Subject: [PATCH 3/3] changes to fqdn for minio --- tests/templates/kuttl/iceberg/01_s3-connection.yaml | 2 +- tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json | 2 +- .../templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json | 2 +- tests/templates/kuttl/iceberg/README.md | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/templates/kuttl/iceberg/01_s3-connection.yaml b/tests/templates/kuttl/iceberg/01_s3-connection.yaml index c095461e..ef785029 100644 --- a/tests/templates/kuttl/iceberg/01_s3-connection.yaml +++ b/tests/templates/kuttl/iceberg/01_s3-connection.yaml @@ -4,7 +4,7 @@ kind: S3Connection metadata: name: minio spec: - host: "minio.${NAMESPACE}.svc.cluster.local" + host: minio port: 9000 accessStyle: Path credentials: diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json index bf689ddd..53971f11 100644 --- a/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json +++ b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json @@ -654,7 +654,7 @@ "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml,/stackable/userdata/hive-config/hive-site.xml", "hive-metastore-uri": "thrift://hive-metastore:9083", - "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-endpoint": "https://minio:9000", "s3-path-style-access": "true", "warehouse-location": "s3a://demo/lakehouse" }, diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json index 1a3d6c88..3cab623f 100644 --- a/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json +++ b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json @@ -594,7 +594,7 @@ "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml", "hive-metastore-uri": "thrift://hive-metastore:9083", - "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-endpoint": "https://minio:9000", "s3-path-style-access": "true", "warehouse-location": "s3a://demo/lakehouse" }, diff --git a/tests/templates/kuttl/iceberg/README.md b/tests/templates/kuttl/iceberg/README.md index ce884d9e..fc63e85b 100644 --- a/tests/templates/kuttl/iceberg/README.md +++ b/tests/templates/kuttl/iceberg/README.md @@ -6,6 +6,9 @@ TIP: I used `JSON: Sort Document` in VScode to somewhat have consistent formatti Notable the following diff has been made (may not be up to date!): +NOTE: Due to test changes (in particular changing how Minio is installed) the FQDN for Minio/TLS access is the unqualified service name i.e. m̀inio`. +The `s3-endpoint` has been changed accordingly to `"s3-endpoint": "https://minio:9000"`. + ```diff diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow.json b/tests/templates/kuttl/iceberg/60_nifi-flow.json index 09783fa..23c679f 100644