wait before port-forward

Author: diamonwiggins

.github/workflows/mlflow-ci.yml

@@ -372,6 +372,16 @@ jobs:
           echo "Verifying MLflow service exists..."
           KUBECONFIG="$KUBECONFIG_FILE" kubectl get svc -n values-test
           
+          # Check pod status and wait for them to be running
+          echo "Checking pod status..."
+          KUBECONFIG="$KUBECONFIG_FILE" kubectl get pods -n values-test
+          
+          echo "Waiting for MLflow pods to be running..."
+          KUBECONFIG="$KUBECONFIG_FILE" kubectl wait --for=condition=Ready pods --selector=app.kubernetes.io/name=mlflow -n values-test --timeout=2m || {
+            echo "WARNING: Timed out waiting for pods to be ready, will try port-forwarding anyway"
+            KUBECONFIG="$KUBECONFIG_FILE" kubectl describe pods -n values-test
+          }
+          
           # Set up port forwarding in the background
           echo "Setting up port forwarding to run in the background"
           nohup bash -c "KUBECONFIG='$KUBECONFIG_FILE' kubectl port-forward -n values-test svc/mlflow $PORT:5000 &>/tmp/port-forward-${{ github.run_id }}.log" &
@@ -383,8 +393,9 @@ jobs:
           echo "hostname=localhost:$PORT" >> $GITHUB_OUTPUT
           echo "Test endpoint will be: localhost:$PORT"
           
-          # Simple wait for port-forward to establish
-          sleep 5
+          # Give port-forward more time to establish
+          echo "Waiting for port-forward to establish..."
+          sleep 15
           
           # Basic connectivity check
           echo "Checking connectivity to MLflow..."
@@ -395,6 +406,10 @@ jobs:
             # Show the port-forward log for debugging
             echo "Port-forward log:"
             cat /tmp/port-forward-${{ github.run_id }}.log || true
+            
+            # If port-forward failed, check pod logs
+            echo "Pod logs:"
+            KUBECONFIG="$KUBECONFIG_FILE" kubectl logs -n values-test -l app.kubernetes.io/name=mlflow --tail=20 || true
           fi
         env:
           KUBECONFIG: ${{ steps.create-cluster.outputs.cluster-kubeconfig }}