[bugfix] always force spawn instead of fork (#852)

Author: Eigensystem

fastvideo/envs.py

@@ -21,7 +21,7 @@
     FASTVIDEO_TRACE_FUNCTION: int = 0
     FASTVIDEO_ATTENTION_BACKEND: str | None = None
     FASTVIDEO_ATTENTION_CONFIG: str | None = None
-    FASTVIDEO_WORKER_MULTIPROC_METHOD: str = "fork"
+    FASTVIDEO_WORKER_MULTIPROC_METHOD: str = "spawn"
     FASTVIDEO_TARGET_DEVICE: str = "cuda"
     MAX_JOBS: str | None = None
     NVCC_THREADS: str | None = None
@@ -212,9 +212,8 @@ def maybe_convert_int(value: str | None) -> int | None:
              os.path.expanduser(os.getenv("FASTVIDEO_ATTENTION_CONFIG", "."))),
 
     # Use dedicated multiprocess context for workers.
-    # Both spawn and fork work
     "FASTVIDEO_WORKER_MULTIPROC_METHOD":
-    lambda: os.getenv("FASTVIDEO_WORKER_MULTIPROC_METHOD", "fork"),
+    lambda: os.getenv("FASTVIDEO_WORKER_MULTIPROC_METHOD", "spawn"),
 
     # Enables torch profiler if set. Path to the directory where torch profiler
     # traces are saved. Note that it must be an absolute path.

fastvideo/utils.py

@@ -1119,34 +1119,9 @@ def xpu_is_initialized() -> bool:
     return torch.xpu.is_initialized()
 
 
-def maybe_force_spawn() -> None:
-    """Check if we need to force the use of the `spawn` multiprocessing start
-    method.
-    """
-    if os.environ.get("FASTVIDEO_WORKER_MULTIPROC_METHOD") == "spawn":
-        return
-
-    reasons = []
-
-    from fastvideo.worker.ray_utils import is_in_ray_actor
-    if is_in_ray_actor():
-        # even if we choose to spawn, we need to pass the ray address
-        # to the subprocess so that it knows how to connect to the ray cluster.
-        # env vars are inherited by subprocesses, even if we use spawn.
-        import ray
-        os.environ["RAY_ADDRESS"] = ray.get_runtime_context().gcs_address
-        reasons.append("In a Ray actor and can only be spawned")
-
-    if cuda_is_initialized():
-        reasons.append("CUDA is initialized")
-    elif xpu_is_initialized():
-        reasons.append("XPU is initialized")
-
-    if reasons:
-        logger.warning(
-            "We must use the `spawn` multiprocessing start method. "
-            "Overriding FASTVIDEO_WORKER_MULTIPROC_METHOD to 'spawn'. "
-            "Reasons: %s", "; ".join(reasons))
+def force_spawn() -> None:
+    if os.environ.get("FASTVIDEO_WORKER_MULTIPROC_METHOD") == "fork":
+        logger.warning("We must use the `spawn` multiprocessing start method.")
         os.environ["FASTVIDEO_WORKER_MULTIPROC_METHOD"] = "spawn"
 
 
@@ -1157,7 +1132,7 @@ def get_mp_context() -> BaseContext:
     certain conditions, we may enforce spawn and override the value of
     FASTVIDEO_WORKER_MULTIPROC_METHOD.
     """
-    maybe_force_spawn()
+    force_spawn()
     mp_method = envs.FASTVIDEO_WORKER_MULTIPROC_METHOD
     return multiprocessing.get_context(mp_method)
 

fastvideo/worker/multiproc_executor.py

@@ -19,7 +19,7 @@
 from fastvideo.fastvideo_args import FastVideoArgs
 from fastvideo.logger import init_logger
 from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
-from fastvideo.utils import decorate_logs, get_distributed_init_method, get_exception_traceback, get_loopback_ip, get_mp_context, get_open_port, kill_itself_when_parent_died, maybe_force_spawn
+from fastvideo.utils import decorate_logs, get_distributed_init_method, get_exception_traceback, get_loopback_ip, get_mp_context, get_open_port, kill_itself_when_parent_died, force_spawn
 from fastvideo.worker.executor import Executor
 from fastvideo.worker.worker_base import WorkerWrapperBase
 
@@ -503,4 +503,4 @@ def set_multiproc_executor_envs() -> None:
     in a multiprocessing environment. This should be called by the parent 
     process before worker processes are created"""
 
-    maybe_force_spawn()
+    force_spawn()