Skip to content

Commit 5528edc

Browse files
diamonwigginsdiamonwiggins
committed
fix port forward
1 parent 488dcb4 commit 5528edc

File tree

1 file changed

+68
-47
lines changed

1 file changed

+68
-47
lines changed

‎applications/mlflow/Taskfile.yml‎

Lines changed: 68 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -602,47 +602,81 @@ tasks:
602602
fi
603603
604604
echo "Setting up port forwarding to $SERVICE_NAME..."
605-
# Kill any existing port-forward on the same port
606-
PORT_FORWARD_LOG="/tmp/port-forward-mlflow-$$.log"
607-
608-
# Platform-independent way to check and kill processes on the port
609-
PORT_IN_USE=""
610-
if command -v lsof >/dev/null 2>&1; then
611-
# Linux/Mac approach
612-
PORT_IN_USE=$(lsof -i :{{.PORT}} | grep LISTEN | awk '{print $2}')
613-
if [ -n "$PORT_IN_USE" ]; then
614-
echo "Killing process $PORT_IN_USE using port {{.PORT}}"
615-
kill $PORT_IN_USE 2>/dev/null || true
616-
sleep 2
617-
fi
618-
elif command -v netstat >/dev/null 2>&1; then
619-
# Windows/generic approach
620-
netstat -ano | grep ":{{.PORT}} " | grep "LISTENING" > /dev/null && {
621-
echo "Port {{.PORT}} is in use. Please free this port before continuing."
622-
exit 1
623-
}
624-
fi
625-
626605
# Set up port forwarding in the background with logs
627-
kubectl port-forward -n {{.NAMESPACE}} $SERVICE_NAME {{.PORT}}:5000 > $PORT_FORWARD_LOG 2>&1 &
606+
echo "Setting up port forwarding using nohup..."
607+
# Use nohup to ensure the process runs in the background even if the parent process exits
608+
PORT_FORWARD_LOG="/tmp/port-forward-mlflow-$$.log"
609+
nohup kubectl port-forward -n {{.NAMESPACE}} $SERVICE_NAME {{.PORT}}:5000 > $PORT_FORWARD_LOG 2>&1 &
628610
PORT_FORWARD_PID=$!
629-
echo "Port forwarding set up with PID: $PORT_FORWARD_PID"
611+
612+
# Give port-forward a moment to start
613+
sleep 2
630614
631615
# Verify the PID was captured properly
632-
if [ -z "$PORT_FORWARD_PID" ] || [ "$PORT_FORWARD_PID" -eq "0" ]; then
616+
if [ -z "$PORT_FORWARD_PID" ] || [ "$PORT_FORWARD_PID" = "0" ]; then
633617
echo "ERROR: Failed to capture port-forward process PID"
634-
exit 1
618+
echo "Attempting alternate port forwarding method..."
619+
620+
# Alternative approach - use a fixed port file to track the PID
621+
PID_FILE="/tmp/mlflow-portforward.pid"
622+
rm -f $PID_FILE
623+
624+
# Use a background task with PID file
625+
(
626+
kubectl port-forward -n {{.NAMESPACE}} $SERVICE_NAME {{.PORT}}:5000 > $PORT_FORWARD_LOG 2>&1 &
627+
echo $! > $PID_FILE
628+
wait
629+
) &
630+
631+
# Wait a moment and check if the PID file was created
632+
sleep 3
633+
if [ -f $PID_FILE ]; then
634+
PORT_FORWARD_PID=$(cat $PID_FILE)
635+
echo "Port forwarding set up with alternate method, PID: $PORT_FORWARD_PID"
636+
else
637+
echo "ERROR: Both port forwarding methods failed. Continuing anyway..."
638+
# Continue anyway and rely on curl checks to verify connectivity
639+
PORT_FORWARD_PID=""
640+
fi
641+
else
642+
echo "Port forwarding set up with PID: $PORT_FORWARD_PID"
635643
fi
636644
637645
# Give port-forward more time to establish
638646
echo "Waiting for port-forward to establish..."
639647
sleep 5
640648
641-
# Check if port-forward process is still running
642-
if ! ps -p $PORT_FORWARD_PID > /dev/null 2>&1; then
643-
echo "ERROR: Port forwarding process died. Check the logs:"
644-
cat $PORT_FORWARD_LOG
645-
exit 1
649+
# Only check process if we have a PID
650+
if [ -n "$PORT_FORWARD_PID" ]; then
651+
# Check if port-forward process is still running
652+
if ! ps -p $PORT_FORWARD_PID > /dev/null 2>&1; then
653+
echo "WARNING: Port forwarding process with PID $PORT_FORWARD_PID is not running"
654+
echo "Port forwarding log:"
655+
cat $PORT_FORWARD_LOG || echo "No log file found"
656+
echo "Will try to connect anyway..."
657+
fi
658+
fi
659+
660+
# Check if port-forward is still running
661+
if [ -n "$PORT_FORWARD_PID" ] && ! ps -p $PORT_FORWARD_PID > /dev/null 2>&1; then
662+
echo "ERROR: Port forwarding process died during connection attempts."
663+
echo "Port forwarding log:"
664+
cat $PORT_FORWARD_LOG || echo "No log file found"
665+
666+
# Restart port forwarding as a fallback
667+
echo "Attempting to restart port forwarding..."
668+
nohup kubectl port-forward -n {{.NAMESPACE}} $SERVICE_NAME {{.PORT}}:5000 > $PORT_FORWARD_LOG 2>&1 &
669+
PORT_FORWARD_PID=$!
670+
sleep 3
671+
672+
if [ -z "$PORT_FORWARD_PID" ] || [ "$PORT_FORWARD_PID" = "0" ]; then
673+
echo "WARNING: Failed to capture restarted port-forward process PID"
674+
echo "Will continue without checking process status"
675+
else
676+
echo "Restarted port forwarding with PID: $PORT_FORWARD_PID"
677+
fi
678+
679+
sleep 5 # Give the new port-forward time to establish
646680
fi
647681
648682
# Basic connectivity check
@@ -664,24 +698,11 @@ tasks:
664698
echo "Connection attempt $CONN_RETRY_COUNT failed, retrying in 5 seconds..."
665699
666700
# Check if port-forward is still running
667-
if ! ps -p $PORT_FORWARD_PID > /dev/null 2>&1; then
668-
echo "ERROR: Port forwarding process died during connection attempts."
701+
if [ -n "$PORT_FORWARD_PID" ] && ! ps -p $PORT_FORWARD_PID > /dev/null 2>&1; then
702+
echo "WARNING: Port forwarding process with PID $PORT_FORWARD_PID is not running"
669703
echo "Port forwarding log:"
670-
cat $PORT_FORWARD_LOG
671-
672-
# Restart port forwarding as a fallback
673-
echo "Attempting to restart port forwarding..."
674-
kubectl port-forward -n {{.NAMESPACE}} $SERVICE_NAME {{.PORT}}:5000 > $PORT_FORWARD_LOG 2>&1 &
675-
PORT_FORWARD_PID=$!
676-
677-
# Verify the PID was captured properly after restart
678-
if [ -z "$PORT_FORWARD_PID" ] || [ "$PORT_FORWARD_PID" -eq "0" ]; then
679-
echo "ERROR: Failed to capture restarted port-forward process PID"
680-
exit 1
681-
fi
682-
683-
echo "Restarted port forwarding with PID: $PORT_FORWARD_PID"
684-
sleep 5 # Give the new port-forward time to establish
704+
cat $PORT_FORWARD_LOG || echo "No log file found"
705+
echo "Will try to connect anyway..."
685706
fi
686707
687708
sleep 5

0 commit comments

Comments
 (0)