optimizer work_dir

Author: kkscilife

autotest/config/gptoss.py

@@ -50,6 +50,6 @@
     tokenizer_path=GPTOSS_21B_PATH,
     global_batch_size=16,
     total_epoch=1,
-    work_dir=f"/mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output/{os.environ['GITHUB_RUN_ID']}/gptoss-sft/sft",
+    work_dir=f"{os.environ['WORK_DIR']}",
     seed=0,
 )

autotest/module/train.py

@@ -24,10 +24,11 @@ def get_cmd(config):
                     train_type,
                 ]
             )
+            config["work_dir"] = work_dir
 
             if train_type == "sft":
                 command = (
-                    f"cd {current_dir}; pwd; pip install -e .[all]; pip install more-itertools; export GITHUB_RUN_ID={config.get('run_id')}; "
+                    f"cd {current_dir}; pwd; pip install -e .[all]; pip install more-itertools; export GITHUB_RUN_ID={config.get('run_id')}; export WORK_DIR={work_dir}; "
                     + f"torchrun --nproc-per-node {nproc_per_node} --master_addr=${{MASTER_ADDR}} --master_port=${{MASTER_PORT}} --nnodes=${{WORLD_SIZE}} --node_rank=${{RANK}} "
                     + f"xtuner/v1/train/cli/{train_type}.py"
                 )
@@ -46,11 +47,9 @@ def get_cmd(config):
                         command += f" --dataset {dataset_path}"
                     command += f" --work_dir {work_dir}"
 
-                config["work_dir"] = work_dir
                 return command, config
             elif train_type == "rl":
                 infer_type = config.get("parameters", {}).get("infer_backend", "lmdeploy")
-                config["work_dir"] = work_dir
                 command = (
                     f"cd {current_dir}; pwd; pip install -e .[all]; export GITHUB_RUN_ID={config.get('run_id')}; export WORK_DIR={work_dir}; "
                     + f"bash -x examples/v1/scripts/run_rl.sh {config_path} {infer_type} ${{MODEL_PATH}} ${{DATA_PATH}} ${{EVAL_DATA_PATH}}"