add metrics test for connect server

Author: razvan

tests/templates/kuttl/spark-connect/40-assert.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: metrics
+commands:
+  - script: kubectl exec -n $NAMESPACE python -- python /test-scripts/metrics.py

tests/templates/kuttl/spark-connect/40-install-test-container.yaml

@@ -0,0 +1,59 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+metadata:
+  name: install-test-container
+timeout: 300
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-scripts
+data:
+  metrics.py: |
+    import sys
+    import logging
+    import requests
+
+    if __name__ == "__main__":
+        LOG_LEVEL = "DEBUG"  # if args.debug else 'INFO'
+        logging.basicConfig(
+            level=LOG_LEVEL,
+            format="%(asctime)s %(levelname)s: %(message)s",
+            stream=sys.stdout,
+        )
+
+        response = requests.get("http://spark-connect-server-metrics:4040/metrics/prometheus")
+
+        assert response.status_code == 200, (
+            f"Expected HTTP return code 200 from the metrics endpoint but got [{response.status_code}]"
+        )
+
+        assert "BlockManager_memory_remainingOnHeapMem_MB_Value" in response.text, (
+            "Expected metric [BlockManager_memory_remainingOnHeapMem_MB_Value] not found"
+        )
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: python
+spec:
+  containers:
+    - name: tools
+      image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+      stdin: true
+      tty: true
+      resources:
+        requests:
+          memory: "128Mi"
+          cpu: "512m"
+        limits:
+          memory: "128Mi"
+          cpu: "1"
+      volumeMounts:
+        - name: test-scripts
+          mountPath: /test-scripts
+  volumes:
+    - name: test-scripts
+      configMap:
+        name: test-scripts