[Fix]: sft multi-turn training error

Author: realtmxi

docs/SFT_GUIDE_EN.md

@@ -1,20 +1,16 @@
+# SFT GUIDE
+
+
 ```
-nohup ./run_qwen_05_sp2.sh 4 /data1/models/openmanus_rl/Qwen/Qwen3-3b-sft \
+nohup ./run_sft.sh 4 /data1/models/openmanus_rl/Qwen/Qwen3-3b-sft \
     data.truncation=right \
     trainer.total_training_steps=1000 \
-    ++actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 \
-    ++critic.model.fsdp_config.model_dtype=bfloat16 \
     trainer.logger="['console','wandb']" \
     trainer.project_name="OpenManus-rl" \
     > training_run.log 2>&1 &
 ```
 
-You need to clone a new verl codebase, and use verl conda environment to run this multi-turn sft script.
-
-You should copy openmanus-rl/scripts/run_sft.sh to verl/examples/sft/multiturn/
-then run the script
-
 
 ```
-./run_qwen_05_sp2.sh 4 /data1/models/openmanus_rl/Qwen/Qwen3-3b-sft     data.truncation=right     trainer.total_training_steps=30    trainer.logger="['console','wandb']"     trainer.project_name="OpenManus-rl"
-```
+./run_sft.sh 4 /data1/models/openmanus_rl/Qwen/Qwen3-3b-sft     data.truncation=right     trainer.total_training_steps=30    trainer.logger="['console','wandb']"     trainer.project_name="OpenManus-rl"
+```

scripts/run_sft.sh

@@ -17,6 +17,8 @@ if [ -f "$CONDA_BASE_DIR/etc/profile.d/conda.sh" ]; then
 else
     echo "Conda base profile script not found at $CONDA_BASE_DIR/etc/profile.d/conda.sh"
 fi
+export WANDB_API_KEY= # TODO: add your wandb api key here
+wandb login
 
 nproc_per_node=$1
 save_path=$2
@@ -31,7 +33,7 @@ if [ "$use_all_gpu" = "true" ]; then
     tensor_parallel_size=8
     echo "Configured to use 8 GPUs: CUDA_VISIBLE_DEVICES=$visible_devices, tensor_parallel_size=$tensor_parallel_size"
 else
-    visible_devices="4,5,6,7"
+    visible_devices="0,1,2,3"
     tensor_parallel_size=4
     echo "Configured to use 4 GPUs: CUDA_VISIBLE_DEVICES=$visible_devices, tensor_parallel_size=$tensor_parallel_size"
 fi
@@ -41,12 +43,12 @@ fi
 CUDA_VISIBLE_DEVICES="$visible_devices" \
 torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \
      -m verl.trainer.fsdp_sft_trainer \
-    data.train_files=$HOME/muxin/OpenManus-RL/data/train.parquet \
-    data.val_files=$HOME/muxin/OpenManus-RL/data/test.parquet \
+    data.train_files=../data/train.parquet \
+    data.val_files=../data/test.parquet \
     data.multiturn.enable=true \
     data.multiturn.messages_key=conversations \
     data.micro_batch_size=4 \
-    model.partial_pretrain=/data1/models/Qwen/Qwen3-4B \
+    model.partial_pretrain=/data1/models/Qwen/Qwen2.5-3B \ # TODO: add your model path here
     trainer.default_local_dir=$save_path \
     trainer.project_name=multiturn-sft \
     trainer.experiment_name=multiturn-sft-qwen-3-4b \

verl/trainer/fsdp_sft_trainer.py

@@ -172,7 +172,7 @@ def _build_model_optimizer(self):
         with init_context():
             self.model: PreTrainedModel = AutoModelForCausalLM.from_pretrained(local_model_path,
                                                                                config=config,
-                                                                               torch_dtype=torch.float32,
+                                                                               torch_dtype="auto",
                                                                                attn_implementation='flash_attention_2',
                                                                                trust_remote_code=trust_remote_code)
             if self.config.model.get('lora_rank', 0) > 0:

verl/utils/dataset/multiturn_sft_dataset.py

@@ -64,6 +64,23 @@ def series_to_item(ls):
 
             while isinstance(ls, (pandas.core.series.Series, numpy.ndarray)) and len(ls) == 1:
                 ls = ls[0]
+            
+            # Convert numpy array to list if needed
+            if isinstance(ls, numpy.ndarray):
+                ls = ls.tolist()
+            elif isinstance(ls, pandas.core.series.Series):
+                ls = ls.tolist()
+
+            # If ls is a single dictionary with 'role' and 'content', wrap it in a list
+            if isinstance(ls, dict) and 'role' in ls and 'content' in ls:
+                ls = [ls]
+    
+            # Verify the structure if it's a list
+            if isinstance(ls, list):
+                for i, item in enumerate(ls):
+                    if not isinstance(item, dict) or 'role' not in item or 'content' not in item:
+                        raise ValueError(f"Invalid message format at index {i}: {item}")
+
             return ls
 
         dataframes = []
@@ -75,14 +92,21 @@ def series_to_item(ls):
         # Extract messages list from dataframe
         self.messages = self.dataframe[self.messages_key].apply(series_to_item).tolist()
 
+        dataframes = []
+        for parquet_file in self.parquet_files:
+            dataframe = pd.read_parquet(parquet_file)
+            dataframes.append(dataframe)
+        self.dataframe = pd.concat(dataframes)
+
+        # Extract messages list from dataframe
+        self.messages = self.dataframe[self.messages_key].apply(series_to_item).tolist()
+
     def __len__(self):
         return len(self.messages)
 
     def __getitem__(self, item):
         tokenizer = self.tokenizer
         messages = self.messages[item]
-
-        # First, get the full conversation tokens
         full_tokens = tokenizer.apply_chat_template(messages, tokenize=True, return_tensors="pt", add_generation_prompt=False)
         input_ids = full_tokens[0]  # The output is already a tensor
         attention_mask = torch.ones_like(input_ids)
@@ -143,4 +167,4 @@ def __getitem__(self, item):
             "attention_mask": attention_mask,
             "position_ids": position_ids,
             "loss_mask": loss_mask,
-        }
+        }