fix taskfile

Author: diamonwiggins

.github/workflows/mlflow-ci.yml

@@ -56,6 +56,7 @@ jobs:
         run: |
           # Use Taskfile to add Helm repos, lint charts and generate templates
           task add:helm:repos
+          task update:helm:deps
           task lint
           task template
 
@@ -98,6 +99,8 @@ jobs:
         working-directory: applications/mlflow
         run: |
           # Update and package charts
+          task add:helm:repos
+          task update:helm:deps
           task update:versions
           task package
 

applications/mlflow/Taskfile.yml

@@ -158,11 +158,23 @@ tasks:
       - helm repo add cnpg https://cloudnative-pg.github.io/charts
       - helm repo add minio https://operator.min.io/
       - helm repo update
+  
+  # Dependency update
+  update:helm:deps:
+    desc: Update Helm chart dependencies
+    deps: [add:helm:repos]
+    cmds:
+      - echo "Updating Helm chart dependencies..."
+      - for: { var: CHARTS }
+        cmd: |
+          echo "Updating dependencies for {{.ITEM}} chart..."
+          helm dependency update {{.CHART_DIR}}/{{.ITEM}}
+      - echo "Helm chart dependencies updated successfully."
 
   # Chart linting
   lint:
     desc: Lint Helm charts
-    deps: [add:helm:repos]
+    deps: [add:helm:repos, update:helm:deps]
     cmds:
       - echo "Linting Helm charts..."
       - for: { var: CHARTS }
@@ -174,7 +186,7 @@ tasks:
   # Template rendering
   template:
     desc: Template Helm charts with Replicated SDK disabled and output to stdout
-    deps: [add:helm:repos]
+    deps: [add:helm:repos, update:helm:deps]
     cmds:
       - echo "Templating Helm charts with Replicated SDK disabled..."
       - for: { var: CHARTS }
@@ -200,7 +212,7 @@ tasks:
   # Packaging tasks
   package:
     desc: Package Helm charts for distribution
-    deps: [add:helm:repos, update:versions]
+    deps: [add:helm:repos, update:helm:deps, update:versions]
     cmds:
       - echo "Packaging Helm charts..."
       - for: { var: CHARTS }
@@ -444,8 +456,10 @@ tasks:
           echo "❌ ERROR: MLflow service not found after $((MAX_RETRIES * RETRY_INTERVAL)) seconds."
           echo "Showing all available services in the namespace:"
           kubectl get svc -n {{.NAMESPACE}}
-          echo "Showing all pods in the namespace:"
+          echo "Showing pod status in the namespace:"
           kubectl get pods -n {{.NAMESPACE}}
+          echo "Showing pod details:"
+          kubectl describe pods -n {{.NAMESPACE}} -l app.kubernetes.io/name=mlflow
           exit 1
         fi
         
@@ -471,32 +485,88 @@ tasks:
         
         echo "Setting up port forwarding to $SERVICE_NAME..."
         # Kill any existing port-forward on the same port
-        lsof -i :{{.PORT}} | grep LISTEN | awk '{print $2}' | xargs kill 2>/dev/null || true
+        PORT_FORWARD_LOG="/tmp/port-forward-mlflow-$$.log"
+        
+        # Platform-independent way to check and kill processes on the port
+        PORT_IN_USE=""
+        if command -v lsof >/dev/null 2>&1; then
+          # Linux/Mac approach
+          PORT_IN_USE=$(lsof -i :{{.PORT}} | grep LISTEN | awk '{print $2}')
+          if [ -n "$PORT_IN_USE" ]; then
+            echo "Killing process $PORT_IN_USE using port {{.PORT}}"
+            kill $PORT_IN_USE 2>/dev/null || true
+            sleep 2
+          fi
+        elif command -v netstat >/dev/null 2>&1; then
+          # Windows/generic approach
+          netstat -ano | grep ":{{.PORT}} " | grep "LISTENING" > /dev/null && {
+            echo "Port {{.PORT}} is in use. Please free this port before continuing."
+            exit 1
+          }
+        fi
         
-        # Set up port forwarding in the background
-        kubectl port-forward -n {{.NAMESPACE}} $SERVICE_NAME {{.PORT}}:5000 &
+        # Set up port forwarding in the background with logs
+        kubectl port-forward -n {{.NAMESPACE}} $SERVICE_NAME {{.PORT}}:5000 > $PORT_FORWARD_LOG 2>&1 &
         PORT_FORWARD_PID=$!
         echo "Port forwarding set up with PID: $PORT_FORWARD_PID"
         
         # Give port-forward more time to establish
         echo "Waiting for port-forward to establish..."
         sleep 5
         
+        # Check if port-forward process is still running
+        if ! ps -p $PORT_FORWARD_PID > /dev/null; then
+          echo "ERROR: Port forwarding process died. Check the logs:"
+          cat $PORT_FORWARD_LOG
+          exit 1
+        fi
+        
         # Basic connectivity check
         echo "Checking connectivity to MLflow on localhost:{{.PORT}}..."
-        for i in {1..5}; do
-          if curl -s -o /dev/null -w "%{http_code}" http://localhost:{{.PORT}}/; then
+        MAX_CONN_RETRIES=5
+        CONN_RETRY_COUNT=0
+        CONN_SUCCESS=false
+        
+        while [ $CONN_RETRY_COUNT -lt $MAX_CONN_RETRIES ]; do
+          CONN_RETRY_COUNT=$((CONN_RETRY_COUNT+1))
+          echo "Connection attempt $CONN_RETRY_COUNT/$MAX_CONN_RETRIES..."
+          
+          # Try curling the MLflow endpoint
+          if curl -s -o /dev/null -w "%{http_code}" http://localhost:{{.PORT}}/ > /dev/null 2>&1; then
             echo "Successfully connected to MLflow service!"
+            CONN_SUCCESS=true
             break
           else
-            echo "Connection attempt $i failed, retrying..."
-            sleep 5
-            if [ $i -eq 5 ]; then
-              echo "Warning: Could not connect to MLflow service after multiple attempts"
-              echo "Port forwarding may not be working correctly"
+            echo "Connection attempt $CONN_RETRY_COUNT failed, retrying in 5 seconds..."
+            
+            # Check if port-forward is still running
+            if ! ps -p $PORT_FORWARD_PID > /dev/null; then
+              echo "ERROR: Port forwarding process died during connection attempts."
+              echo "Port forwarding log:"
+              cat $PORT_FORWARD_LOG
+              
+              # Restart port forwarding as a fallback
+              echo "Attempting to restart port forwarding..."
+              kubectl port-forward -n {{.NAMESPACE}} $SERVICE_NAME {{.PORT}}:5000 > $PORT_FORWARD_LOG 2>&1 &
+              PORT_FORWARD_PID=$!
+              echo "Restarted port forwarding with PID: $PORT_FORWARD_PID"
             fi
+            
+            sleep 5
           fi
         done
+        
+        if [ "$CONN_SUCCESS" != "true" ]; then
+          echo "WARNING: Could not connect to MLflow service after $MAX_CONN_RETRIES attempts."
+          echo "This may indicate issues with the service or port forwarding."
+          echo "Port forwarding log:"
+          cat $PORT_FORWARD_LOG
+          echo "Pod logs:"
+          kubectl logs -n {{.NAMESPACE}} -l app.kubernetes.io/name=mlflow --tail=20 || true
+          echo "Continuing anyway, but tests may fail."
+        fi
+        
+        echo "Port forwarding setup completed."
 
   test:local:
     desc: Run Helm installation test with local charts (no Replicated registry)
@@ -549,7 +619,7 @@ tasks:
       - echo "Running application tests against MLflow on localhost:{{.PORT}}..."
       - |
         echo "Installing Python dependencies for tests..."
-        pip3 install mlflow pandas scikit-learn requests urllib3
+        pip3 install mlflow==2.11.0 pandas>=2.0.0 scikit-learn>=1.3.0 requests>=2.31.0 urllib3>=2.0.0
         
         echo "Running MLflow application tests"
         python {{.TESTS_DIR}}/mlflow_test.py localhost:{{.PORT}} \