Fixed config check issue with multi-node spawn method (#2424)

* Fixed config check issue with multi-node spawn method

* [skip ci] updated clearml docs link

* Update test_launcher.py

Author: vfdev-5

ignite/contrib/handlers/clearml_logger.py

@@ -51,7 +51,7 @@ class ClearMLLogger(BaseLogger):
     Args:
         kwargs: Keyword arguments accepted from
             `clearml.Task
-            <https://clear.ml/docs/latest/docs/references/sdk/task#taskinit>`_.
+            <https://clear.ml/docs/latest/docs/references/sdk/task/#taskinit>`_.
             All arguments are optional. If a ClearML Task has already been created,
             kwargs will be ignored and the current ClearML Task will be used.
 

ignite/distributed/launcher.py

@@ -265,9 +265,9 @@ def _setup_spawn_params(
             node_rank = 0
         if node_rank >= nnodes or node_rank < 0:
             raise ValueError(f"Argument node_rank should be between 0 and {nnodes - 1}, but given {node_rank}")
-        if nnodes > 1 and (master_addr is None or master_port is None or init_method is None):
+        if nnodes > 1 and (master_addr is None or master_port is None) and init_method is None:
             raise ValueError(
-                "If number of nodes larger than one, arguments master_addr and master_port or init_method"
+                "If number of nodes larger than one, arguments master_addr and master_port or init_method "
                 f"should be specified, but given master_addr={master_addr}, master_port={master_port} and "
                 f"init_method={init_method}."
             )

tests/ignite/distributed/test_launcher.py

@@ -40,11 +40,11 @@ def exec_filepath():
     yield fp.as_posix()
 
 
-def execute(cmd):
+def execute(cmd, env=None):
 
     import ignite
 
-    env = dict(os.environ)
+    env = dict(os.environ) if env is None else env
     env["PYTHONPATH"] = f"{os.path.dirname(ignite.__path__[0])}"
     process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
     process.wait()
@@ -153,6 +153,38 @@ def test_check_idist_parallel_spawn_n_procs_gloo(exec_filepath):
     _test_check_idist_parallel_spawn(exec_filepath, "gloo", np)
 
 
+@pytest.mark.distributed
+@pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
+@pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
+def test_smoke_test_check_idist_parallel_spawn_multinode_n_procs_gloo(exec_filepath):
+    # Just a smoke test from check_idist_parallel.py for an emulated multi-node configuration
+    cmd1 = "export CUDA_VISIBLE_DEVICES= && "
+    cmd1 += 'bash -c "python tests/ignite/distributed/check_idist_parallel.py --backend=gloo --nproc_per_node=2 '
+    cmd1 += '--nnodes=2 --node_rank=0 --master_addr=localhost --master_port=3344 &"'
+    os.system(cmd1)
+
+    cmd2 = [
+        sys.executable,
+        exec_filepath,
+        "--backend=gloo",
+        "--nproc_per_node=2",
+        "--nnodes=2",
+        "--node_rank=1",
+        "--master_addr=localhost",
+        "--master_port=3344",
+    ]
+    env = dict(os.environ)
+    env["CUDA_VISIBLE_DEVICES"] = ""
+    out = execute(cmd2, env=env)
+
+    assert "backend=gloo" in out
+    assert "nproc_per_node: 2" in out
+    assert "nnodes: 2" in out
+    assert "master_addr: localhost" in out
+    assert "master_port: 3344" in out
+    assert "End of run" in out
+
+
 @pytest.mark.distributed
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")