add application testing for kots install

Author: diamonwiggins

.github/workflows/mlflow-ci.yml

@@ -317,14 +317,14 @@ jobs:
             echo "No custom values file provided. Using default values."
           fi
           
-          # Create namespace if it doesn't exist
-          KUBECONFIG="$KUBECONFIG_FILE" kubectl create namespace values-test 2>/dev/null || true
+          # Install into default namespace for consistency with KOTS install
+          echo "Using namespace: default"
           
           # Install infra chart from Replicated registry
           echo "Installing infra chart from Replicated registry..."
           echo "Chart path: $OCI_URL/infra"
           KUBECONFIG="$KUBECONFIG_FILE" helm upgrade --install infra $OCI_URL/infra \
-            --namespace values-test \
+            --namespace default \
             --wait --timeout 5m --debug || { 
             echo "ERROR: Failed to install infra chart from $OCI_URL/infra"
             echo "Please check that registry login was successful and the chart exists in the registry."
@@ -336,7 +336,7 @@ jobs:
           echo "Chart path: $OCI_URL/mlflow"
           echo "Using values args: $MLFLOW_VALUES_ARGS"
           KUBECONFIG="$KUBECONFIG_FILE" helm upgrade --install mlflow $OCI_URL/mlflow \
-            --namespace values-test \
+            --namespace default \
             $MLFLOW_VALUES_ARGS \
             --wait --timeout 5m --debug || { 
             echo "ERROR: Failed to install mlflow chart from $OCI_URL/mlflow"
@@ -370,21 +370,21 @@ jobs:
           # The Helm installation with --wait should already ensure services are ready
           # But let's verify the services are present
           echo "Verifying MLflow service exists..."
-          KUBECONFIG="$KUBECONFIG_FILE" kubectl get svc -n values-test
+          KUBECONFIG="$KUBECONFIG_FILE" kubectl get svc -n default
           
           # Check pod status and wait for them to be running
           echo "Checking pod status..."
-          KUBECONFIG="$KUBECONFIG_FILE" kubectl get pods -n values-test
+          KUBECONFIG="$KUBECONFIG_FILE" kubectl get pods -n default
           
           echo "Waiting for MLflow pods to be running..."
-          KUBECONFIG="$KUBECONFIG_FILE" kubectl wait --for=condition=Ready pods --selector=app.kubernetes.io/name=mlflow -n values-test --timeout=2m || {
+          KUBECONFIG="$KUBECONFIG_FILE" kubectl wait --for=condition=Ready pods --selector=app.kubernetes.io/name=mlflow -n default --timeout=2m || {
             echo "WARNING: Timed out waiting for pods to be ready, will try port-forwarding anyway"
-            KUBECONFIG="$KUBECONFIG_FILE" kubectl describe pods -n values-test
+            KUBECONFIG="$KUBECONFIG_FILE" kubectl describe pods -n default
           }
           
           # Set up port forwarding in the background
           echo "Setting up port forwarding to run in the background"
-          nohup bash -c "KUBECONFIG='$KUBECONFIG_FILE' kubectl port-forward -n values-test svc/mlflow $PORT:5000 &>/tmp/port-forward-${{ github.run_id }}.log" &
+          nohup bash -c "KUBECONFIG='$KUBECONFIG_FILE' kubectl port-forward -n default svc/mlflow $PORT:5000 &>/tmp/port-forward-${{ github.run_id }}.log" &
           PORT_FORWARD_PID=$!
           echo "port_forward_pid=$PORT_FORWARD_PID" >> $GITHUB_OUTPUT
           echo "Set up port forwarding with PID: $PORT_FORWARD_PID"
@@ -409,7 +409,7 @@ jobs:
             
             # If port-forward failed, check pod logs
             echo "Pod logs:"
-            KUBECONFIG="$KUBECONFIG_FILE" kubectl logs -n values-test -l app.kubernetes.io/name=mlflow --tail=20 || true
+            KUBECONFIG="$KUBECONFIG_FILE" kubectl logs -n default -l app.kubernetes.io/name=mlflow --tail=20 || true
           fi
         env:
           KUBECONFIG: ${{ steps.create-cluster.outputs.cluster-kubeconfig }}
@@ -542,6 +542,89 @@ jobs:
           wait-duration: 10m
           shared-password: 'replicatedmlflow'
 
+      # Set up port forwarding after KOTS installation is complete
+      - name: Set up port forwarding
+        id: port-forward
+        run: |
+          # Use kubeconfig file
+          KUBECONFIG_FILE="/tmp/kubeconfig-kots-test-${{ github.run_id }}"
+          echo "$KUBECONFIG" > "$KUBECONFIG_FILE"
+          echo "Saved kubeconfig to $KUBECONFIG_FILE"
+          
+          # Hardcoded port 5000 for simplicity
+          PORT="5000"
+          echo "Using port: $PORT for testing"
+          
+          # Verify services are present
+          echo "Verifying MLflow service exists..."
+          KUBECONFIG="$KUBECONFIG_FILE" kubectl get svc -n default
+          
+          # Check pod status and wait for them to be running
+          echo "Checking pod status..."
+          KUBECONFIG="$KUBECONFIG_FILE" kubectl get pods -n default
+          
+          echo "Waiting for MLflow pods to be running..."
+          KUBECONFIG="$KUBECONFIG_FILE" kubectl wait --for=condition=Ready pods --selector=app.kubernetes.io/name=mlflow -n default --timeout=2m || {
+            echo "WARNING: Timed out waiting for pods to be ready, will try port-forwarding anyway"
+            KUBECONFIG="$KUBECONFIG_FILE" kubectl describe pods -n default
+          }
+          
+          # Set up port forwarding in the background
+          echo "Setting up port forwarding to run in the background"
+          nohup bash -c "KUBECONFIG='$KUBECONFIG_FILE' kubectl port-forward -n default svc/mlflow $PORT:5000 &>/tmp/port-forward-kots-${{ github.run_id }}.log" &
+          PORT_FORWARD_PID=$!
+          echo "port_forward_pid=$PORT_FORWARD_PID" >> $GITHUB_OUTPUT
+          echo "Set up port forwarding with PID: $PORT_FORWARD_PID"
+          
+          # Set hostname for testing
+          echo "hostname=localhost:$PORT" >> $GITHUB_OUTPUT
+          echo "Test endpoint will be: localhost:$PORT"
+          
+          # Give port-forward more time to establish
+          echo "Waiting for port-forward to establish..."
+          sleep 15
+          
+          # Basic connectivity check
+          echo "Checking connectivity to MLflow..."
+          if curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/; then
+            echo "Successfully connected to MLflow service!"
+          else
+            echo "Warning: Initial connection attempt failed, service may still be starting"
+            # Show the port-forward log for debugging
+            echo "Port-forward log:"
+            cat /tmp/port-forward-kots-${{ github.run_id }}.log || true
+            
+            # If port-forward failed, check pod logs
+            echo "Pod logs:"
+            KUBECONFIG="$KUBECONFIG_FILE" kubectl logs -n default -l app.kubernetes.io/name=mlflow --tail=20 || true
+          fi
+        env:
+          KUBECONFIG: ${{ steps.create-cluster.outputs.cluster-kubeconfig }}
+
+      # Application testing with our consolidated test file
+      - name: Run Application Tests
+        run: |
+          cd applications/mlflow
+          
+          # Use the same kubeconfig file from previous steps
+          KUBECONFIG_FILE="/tmp/kubeconfig-kots-test-${{ github.run_id }}"
+          if [ ! -f "$KUBECONFIG_FILE" ]; then
+            echo "$KUBECONFIG" > "$KUBECONFIG_FILE"
+          fi
+          
+          echo "Installing Python dependencies for tests..."
+          pip install mlflow pandas scikit-learn requests urllib3
+          
+          echo "Running MLflow application tests against ${{ steps.port-forward.outputs.hostname }}"
+          echo "This may take some time as it will retry connections for up to 2 minutes"
+          KUBECONFIG="$KUBECONFIG_FILE" python tests/mlflow_test.py ${{ steps.port-forward.outputs.hostname }} \
+            --protocol http \
+            --connection-timeout 180 \
+            --debug
+        timeout-minutes: 5
+        env:
+          KUBECONFIG: ${{ steps.create-cluster.outputs.cluster-kubeconfig }}
+
       - name: Install troubleshoot
         run: curl -L https://github.com/replicatedhq/troubleshoot/releases/latest/download/support-bundle_linux_amd64.tar.gz | tar xzvf -
         if: failure()