Refactor script to use 'overwrites' variable for command-line arguments in training scripts (#1473)

The goal of this PR is to add support for command line arguments to the
bash training scripts. The `run_train.sh` had support for `overrides`,
however, the `multinode_trainer.slurm` script did not. This `overrides`
flag add supports for commands like: `sbatch multinode_trainer.slurm
--job.description="TEST_RUN"`

However, there is a problem with the current `overrides` implementation,
when passing arguments with space such as `"TEST RUN"` instead of
`"TEST_RUN"` then the variable `job.description` would only get `TEST`
as input and the training script throws an error for unrecognizing the
argument `RUN` which is passed in a different line. To address this I
simplify the code and directly pass the additional overrides through
`$@`. This solves the issue for commands such as: `sbatch
multinode_trainer.slurm --job.description="TEST RUN"`

Author: idoh

multinode_trainer.slurm

@@ -34,7 +34,7 @@ export LOGLEVEL=INFO
 export FI_PROVIDER="efa"
 # Ensure that P2P is available
 # export NCCL_P2P_DISABLE=1
-export NCCL_IB_DISABLE=1
+# export NCCL_IB_DISABLE=1
 
 # debugging flags (optional)
 export NCCL_DEBUG=WARN
@@ -59,5 +59,5 @@ CONFIG_FILE=${CONFIG_FILE:-"./torchtitan/models/llama3/train_configs/llama3_8b.t
 dcgmi profile --pause
 # adjust sbatch --ntasks and sbatch --nodes above and --nnodes below
 # to your specific node count, and update target launch file.
-srun torchrun --nnodes 4 --nproc_per_node 8 --rdzv_id 101 --rdzv_backend c10d --rdzv_endpoint "$head_node_ip:29500" ./torchtitan/train.py --job.config_file ${CONFIG_FILE}
+srun torchrun --nnodes 4 --nproc_per_node 8 --rdzv_id 101 --rdzv_backend c10d --rdzv_endpoint "$head_node_ip:29500" ./torchtitan/train.py --job.config_file ${CONFIG_FILE} "$@"
 dcgmi profile --resume

run_train.sh

@@ -7,22 +7,17 @@
 
 set -ex
 
-# use envs as local overrides for convenience
+# use envs as local overwrites for convenience
 # e.g.
 # LOG_RANK=0,1 NGPU=4 ./run_train.sh
 NGPU=${NGPU:-"8"}
 export LOG_RANK=${LOG_RANK:-0}
 CONFIG_FILE=${CONFIG_FILE:-"./torchtitan/models/llama3/train_configs/debug_model.toml"}
 
-overrides=""
-if [ $# -ne 0 ]; then
-    overrides="$*"
-fi
-
 TORCHFT_LIGHTHOUSE=${TORCHFT_LIGHTHOUSE:-"http://localhost:29510"}
 
 PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True" \
 TORCHFT_LIGHTHOUSE=${TORCHFT_LIGHTHOUSE} \
 torchrun --nproc_per_node=${NGPU} --rdzv_backend c10d --rdzv_endpoint="localhost:0" \
 --local-ranks-filter ${LOG_RANK} --role rank --tee 3 \
--m torchtitan.train --job.config_file ${CONFIG_FILE} $overrides
+-m torchtitan.train --job.config_file ${CONFIG_FILE} "$@"