Update docs with torchrun instead of torch.distributed.launch #2415 (#2420)

* Make Checkpoint.load_objects to accept str and load internally (#2303)

* modify error message

* Add test for Checkpoint.load_objects

* fix test messages to match function error message

* Update docs with torchrun -- launcher module (#2415)

* Update docs with torchrun -- utils module (#2415)

* change codeblock to bash

* Update docs with torchrun -- check_idist_parallel (#2415)

* Update docs with torchrun -- check_idist_parallel (#2415)

Co-authored-by: alialhawas <[email protected]>

* Update launcher.py

* Update check_idist_parallel.py

* Updates according to the review

Co-authored-by: alialhawas <[email protected]>
Co-authored-by: vfdev <[email protected]>

Author: 3 people

ignite/distributed/launcher.py

@@ -23,7 +23,7 @@ class Parallel:
     provided ``backend`` (useful for standalone scripts).
 
     2) Only initialize a processing group given the ``backend``
-    (useful with tools like `torch.distributed.launch`_, `horovodrun`_, etc).
+    (useful with tools like `torchrun`_, `horovodrun`_, etc).
 
     Args:
         backend: backend to use: `nccl`, `gloo`, `xla-tpu`, `horovod`. If None, no distributed
@@ -50,14 +50,14 @@ class Parallel:
         spawn_kwargs: kwargs to ``idist.spawn`` function.
 
     Examples:
-        1) Single node or Multi-node, Multi-GPU training launched with `torch.distributed.launch`_ or `horovodrun`_
+        1) Single node or Multi-node, Multi-GPU training launched with `torchrun` or `horovodrun`_
         tools
 
         Single node option with 4 GPUs
 
         .. code-block:: bash
 
-            python -m torch.distributed.launch --nproc_per_node=4 --use_env main.py
+            torchrun --nproc_per_node=4 main.py
             # or if installed horovod
             horovodrun -np=4 python main.py
 
@@ -66,15 +66,15 @@ class Parallel:
         .. code-block:: bash
 
             ## node 0
-            python -m torch.distributed.launch --nnodes=2 --node_rank=0 --master_addr=master \
-                --master_port=3344 --nproc_per_node=8 --use_env main.py
+            torchrun --nnodes=2 --node_rank=0 --master_addr=master --master_port=3344 \
+            --nproc_per_node=8 main.py
 
             # or if installed horovod
             horovodrun -np 16 -H hostname1:8,hostname2:8 python main.py
 
             ## node 1
-            python -m torch.distributed.launch --nnodes=2 --node_rank=1 --master_addr=master \
-                --master_port=3344 --nproc_per_node=8 --use_env main.py
+            torchrun --nnodes=2 --node_rank=1 --master_addr=master --master_port=3344 \
+            --nproc_per_node=8 main.py
 
 
         User code is the same for both options:
@@ -92,6 +92,8 @@ def training(local_rank, config, **kwargs):
 
             backend = "nccl"  # or "horovod" if package is installed
 
+            config = {"key": "value"}
+
             with idist.Parallel(backend=backend) as parallel:
                 parallel.run(training, config, a=1, b=2)
 
@@ -152,6 +154,8 @@ def training(local_rank, config, **kwargs):
                 print(idist.get_rank(), ": run with config:", config, "- backend=", idist.backend())
                 # ...
 
+            config = {"key": "value"}
+
             with idist.Parallel(backend="xla-tpu", nproc_per_node=8) as parallel:
                 parallel.run(training, config, a=1, b=2)
 
@@ -188,12 +192,13 @@ def training(local_rank, config, **kwargs):
                 "master_port": 15000
             }
 
+            config = {"key": "value"}
+
             with idist.Parallel(backend="nccl", **dist_config) as parallel:
                 parallel.run(training, config, a=1, b=2)
 
 
-
-    .. _torch.distributed.launch: https://pytorch.org/docs/stable/distributed.html#launch-utility
+    .. _torchrun: https://pytorch.org/docs/stable/elastic/run.html#launcher-api
     .. _horovodrun: https://horovod.readthedocs.io/en/latest/api.html#module-horovod.run
     .. _dist.init_process_group: https://pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group
     .. versionchanged:: 0.4.2
@@ -294,6 +299,8 @@ def training(local_rank, config, **kwargs):
                     print(idist.get_rank(), ": run with config:", config, "- backend=", idist.backend())
                     # ...
 
+                config = {"key": "value"}
+
                 with idist.Parallel(backend=backend) as parallel:
                     parallel.run(training, config, a=1, b=2)
 

ignite/distributed/utils.py

@@ -480,12 +480,11 @@ def initialize(backend: str, **kwargs: Any) -> None:
             - | "horovod" : comm(=None), more info: `hvd_init`_.
 
     Examples:
-        Launch single node multi-GPU training with ``torch.distributed.launch`` utility.
+        Launch single node multi-GPU training with ``torchrun`` utility.
 
         .. code-block:: python
 
-            # >>> python -m torch.distributed.launch --nproc_per_node=4 main.py
-
+            # >>> torchrun --nproc_per_node=4 main.py
             # main.py
 
             import ignite.distributed as idist

tests/ignite/distributed/check_idist_parallel.py

@@ -38,19 +38,20 @@ def training(local_rank, config, **kwargs):
         python tests/ignite/distributed/check_idist_parallel.py
         ```
 
-        - Launch 4 procs using gloo backend with `torch.distributed.launch`
+        - Launch 4 procs using gloo backend with `torchrun`:
+
         ```
-        python -m torch.distributed.launch --nproc_per_node=4 --use_env \
-            tests/ignite/distributed/check_idist_parallel.py --backend=gloo
+        torchrun --nproc_per_node=4 tests/ignite/distributed/check_idist_parallel.py --backend=gloo
         ```
 
-        - Launch 2 procs in 2 nodes using gloo backend with `torch.distributed.launch`:
+        - Launch 2 procs in 2 nodes using gloo backend with `torchrun` or `torch.distributed.launch`:
+
         ```
-        bash -c "python -m torch.distributed.launch --nnodes=2 --node_rank=0 \
-            --master_addr=localhost --master_port=3344 --nproc_per_node=2 --use_env \
+        bash -c "torchrun --nnodes=2 --node_rank=0 \
+            --master_addr=localhost --master_port=3344 --nproc_per_node=2 \
             tests/ignite/distributed/check_idist_parallel.py --backend=gloo &" \
-        && bash -c "python -m torch.distributed.launch --nnodes=2 --node_rank=1 \
-            --master_addr=localhost --master_port=3344 --nproc_per_node=2 --use_env \
+        && bash -c "torchrun --nnodes=2 --node_rank=1 \
+            --master_addr=localhost --master_port=3344 --nproc_per_node=2 \
             tests/ignite/distributed/check_idist_parallel.py --backend=gloo &"
         ```
 

tests/ignite/distributed/test_launcher.py

@@ -63,15 +63,11 @@ def test_check_idist_parallel_no_dist(exec_filepath):
 
 
 def _test_check_idist_parallel_torch_launch(init_method, fp, backend, nprocs):
-    # python -m torch.distributed.launch --nproc_per_node=nprocs --use_env \
-    #   tests/ignite/distributed/check_idist_parallel.py --backend=backend
+    # torchrun --nproc_per_node=nprocs tests/ignite/distributed/check_idist_parallel.py --backend=backend
 
     cmd = [
-        sys.executable,
-        "-m",
-        "torch.distributed.launch",
+        "torchrun",
         f"--nproc_per_node={nprocs}",
-        "--use_env",
         fp,
         f"--backend={backend}",
     ]