added better guidance for if deprecated tokenizer path fails (#1568)

Adds a check to see if the old tokenizer path is being used when
tokenizer path fails. This way it can provide guidance to people to
update to the supported `hf_assets_path` and `download_hf_assets.py`
script

Author: wesleytruong

torchtitan/components/tokenizer.py

@@ -82,7 +82,16 @@ def _load_config(self, config_path: str) -> Optional[dict]:
     def _load_tokenizer_from_path(self, tokenizer_path: str) -> Tokenizer:
         """Load tokenizer from various file formats."""
         if not os.path.exists(tokenizer_path):
-            raise FileNotFoundError(f"Tokenizer path '{tokenizer_path}' does not exist")
+            if "assets/tokenizer" in tokenizer_path:
+                raise FileNotFoundError(
+                    "Detected ./assets/tokenizer path which was deprecated in https://github.com/pytorch/torchtitan/pull/1540.\n"
+                    "Remove --model.tokenizer_path and download to --model.hf_assets_path using ./scripts/download_hf_assets.py\n"
+                    "See example: https://github.com/pytorch/torchtitan/tree/main/torchtitan/models/deepseek_v3#download-tokenizer"
+                )
+            else:
+                raise FileNotFoundError(
+                    f"Tokenizer path '{tokenizer_path}' does not exist"
+                )
 
         # Define paths for different tokenizer file types
         tokenizer_json_path = os.path.join(tokenizer_path, "tokenizer.json")