first commit

Author: Felipe Mello

apps/sft/eval_utils.py

@@ -26,32 +26,30 @@ optimizer:
 lr_scheduler:
   warmup_steps: 200
 
-# Unified dataset configuration
-# First dataset with split='train' is used for training
-dataset_val:
-  datasets:
-    - name: "train"
-      path: "yahma/alpaca-cleaned"
-      split: "train[:95%]"
-
-    - name: "val"
-      path: "yahma/alpaca-cleaned"
-      split: "train[95%:]"
-
 training:
   local_batch_size: 1
   seq_len: 2048
   max_norm: 1.0
   steps: 1000
   compile: false
-  dataset: "c4"
+  datasets:
+    - path: "yahma/alpaca-cleaned"
+      split: "train[:95%]"
+
+eval:
+  eval_every_n_steps: 5   # (null = disabled)
+  max_eval_steps: 0    # Max batches per eval dataset (null = run until epoch completes)
+  batch_size: ${training.local_batch_size}  # Batch size for evaluation
+  datasets:
+      - path: "yahma/alpaca-cleaned"
+        split: "train[95%:]"
 
 parallelism:
   data_parallel_replicate_degree: 1
   data_parallel_shard_degree: -1
-  tensor_parallel_degree: 1
+  tensor_parallel_degree: 2
   pipeline_parallel_degree: 1
-  context_parallel_degree: 1
+  context_parallel_degree: 2
   expert_parallel_degree: 1
   disable_loss_parallel: false