[Qwen3] Add 1.7b Config (#1671)

Contents:
1. Minor fix in qwen 0.6b training config
2. Add qwen 1.7b training cofnig
3. Fix the number of parameters calculation when weight tying is enabled

Author: wwwjn

torchtitan/experiments/qwen3/README.md

@@ -9,8 +9,9 @@ QWEN3 0.6B Dense model is available for:
 Other model sizes are added to the args, but toml file configs need to be added and tested.
 
 #### Download Qwen3 tokenizer
+```python scripts/download_hf_assets.py --repo_id <hf_repo_name> --assets tokenizer```
 
-```python scripts/download_hf_assets.py --repo_id Qwen/Qwen3-0.6B --assets tokenizer```
+eg, for Qwen3 0.6B model, the HF repo name is `Qwen/Qwen3-0.6B`. For 1.7B model, the HF repo name is `Qwen/Qwen3-1.7B`.
 
 #### Parity with HF
 
@@ -22,5 +23,6 @@ Model parity test has been done and results suggest parity with HF implementatio
     - MoE alternatives
 
 - Testing
+    - Learning rate verifying: verify learning rate and schedule with real training jobs (eg, 3k stps), or find official references.
     - The model should be tested against established performance benchmarks
     - CI integration

torchtitan/experiments/qwen3/model/args.py

@@ -64,4 +64,8 @@ def get_nparams_and_flops(self, model: nn.Module, seq_len: int) -> tuple[int, in
         )
         num_flops_per_token = 6 * (nparams - nparams_embedding) + 12 * l * h * q * t
 
+        if self.enable_weight_tying:
+            # exclude model.token_embedding parameters from nparams
+            nparams = nparams - nparams_embedding
+
         return nparams, num_flops_per_token

torchtitan/experiments/qwen3/train_configs/qwen3_0.6b.toml

@@ -44,7 +44,7 @@ context_parallel_degree = 1
 enable = false
 folder = "checkpoint"
 interval = 500
-last_save_model_weights_only = false
+last_save_model_only = false
 export_dtype = "float16"
 async_mode = "disabled" # ["disabled", "async", "async_with_pinned_mem"]
 

torchtitan/experiments/qwen3/train_configs/qwen3_1.7b.toml

@@ -0,0 +1,62 @@
+[job]
+dump_folder = "./outputs"
+description = "Qwen 3 1.7B training"
+
+[profiling]
+enable_profiling = false
+save_traces_folder = "profile_trace"
+profile_freq = 100
+
+[metrics]
+log_freq = 10
+enable_tensorboard = false
+save_tb_folder = "tb"
+
+[model]
+name = "qwen3"
+flavor = "1.7B"
+hf_assets_path = "./assets/hf/Qwen3-1.7B"
+# converters = ["float8"]
+
+[optimizer]
+name = "AdamW"
+lr = 3e-4
+eps = 1e-8
+
+[lr_scheduler]
+warmup_steps = 20  # lr scheduler warm up, 20% total steps
+
+[training]
+local_batch_size = 4
+seq_len = 4096
+max_norm = 1.0  # grad norm clipping
+steps = 100
+dataset = "c4"
+
+[parallelism]
+data_parallel_replicate_degree = 1
+data_parallel_shard_degree = -1
+fsdp_reshard_after_forward = "default" # default / never / always
+tensor_parallel_degree = 1
+context_parallel_degree = 1
+
+[checkpoint]
+enable = false
+folder = "checkpoint"
+interval = 500
+last_save_model_only = false
+export_dtype = "float16"
+async_mode = "disabled" # ["disabled", "async", "async_with_pinned_mem"]
+
+[activation_checkpoint]
+mode = "selective"  # ["none", "selective", "full"]
+selective_ac_option = "op"  # "int" = ac every positive int layer or 'op', ac based on ops policy
+
+[compile]
+enable=false
+components = ["model", "loss"]
+
+[float8]
+enable_fsdp_float8_all_gather = false
+precompute_float8_dynamic_scale_for_fsdp = false
+filter_fqns = ["output"]