Adding ray port range to the free port checker (#997)

Signed-off-by: Abhinav Garg <abhinavg@stanford.edu>
Co-authored-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com>

Author: abhinavg4

nemo_curator/core/client.py

@@ -135,12 +135,3 @@ def stop(self) -> None:
             logger.info(msg)
             # Clear the process to prevent double execution (atexit handler)
             self.ray_process = None
-
-
-if __name__ == "__main__":
-    client = RayClient()
-    client.start()
-    import time
-
-    time.sleep(10)  # Wait for ray to start
-    client.stop()

nemo_curator/core/constants.py

@@ -17,6 +17,10 @@
 DEFAULT_RAY_TEMP_DIR = "/tmp/ray"  # noqa: S108
 DEFAULT_RAY_METRICS_PORT = 8080
 DEFAULT_RAY_DASHBOARD_HOST = "127.0.0.1"
-DEFAULT_RAY_CLIENT_SERVER_PORT = 20001
+DEFAULT_RAY_CLIENT_SERVER_PORT = 10001
 DEFAULT_RAY_AUTOSCALER_METRIC_PORT = 44217
 DEFAULT_RAY_DASHBOARD_METRIC_PORT = 44227
+
+# We cannot use a free port between 10000 and 19999 as it is used by Ray.
+DEFAULT_RAY_MIN_WORKER_PORT = 10002
+DEFAULT_RAY_MAX_WORKER_PORT = 19999

nemo_curator/core/utils.py

@@ -23,6 +23,8 @@
 from nemo_curator.core.constants import (
     DEFAULT_RAY_AUTOSCALER_METRIC_PORT,
     DEFAULT_RAY_DASHBOARD_METRIC_PORT,
+    DEFAULT_RAY_MAX_WORKER_PORT,
+    DEFAULT_RAY_MIN_WORKER_PORT,
 )
 
 if TYPE_CHECKING:
@@ -35,6 +37,8 @@ def get_free_port(start_port: int, get_next_free_port: bool = True) -> int:
     Else, it will raise an error if the free port is not equal to start_port.
     """
     for port in range(start_port, 65535):
+        if port >= DEFAULT_RAY_MIN_WORKER_PORT and port <= DEFAULT_RAY_MAX_WORKER_PORT:
+            continue
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
             # SO_REUSEADDR to avoid TIME_WAIT issues on some OSes
             s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)