match titan config update (#48)

Author: ebsmothers

apps/sft/llama3_8b.yaml

@@ -15,7 +15,7 @@ comm:
 model:
   name: llama3
   flavor: 8B
-  tokenizer_path: /tmp/Meta-Llama-3.1-8B-Instruct
+  hf_assets_path: /tmp/Meta-Llama-3.1-8B-Instruct
 
 optimizer:
   name: AdamW

apps/sft/main.py

@@ -22,7 +22,6 @@
 from omegaconf import DictConfig, OmegaConf
 from torch import nn
 from torchdata.stateful_dataloader import StatefulDataLoader
-from torchtitan.components.checkpoint import ModelWrapper
 from torchtitan.components.loss import LossFunction
 from torchtitan.components.lr_scheduler import LRSchedulersContainer
 from torchtitan.components.optimizer import OptimizersContainer
@@ -76,22 +75,20 @@ def setup(self):
         #     self.train_config.packing_config,
         # )
 
-        # TODO: confirm that this is working properly
-        # Should also use load, not dcp_load
         self.checkpointer.load(step=self.current_step)
         # self.profiler = self.setup_profiler(self.train_config.profiler_config)
         # self.logger = self.setup_logger(self.train_config.logger_config)
 
     def setup_data(self):
         tokenizer = HuggingFaceModelTokenizer(
             tokenizer_json_path=os.path.join(
-                self.job_config.model.tokenizer_path, "tokenizer.json"
+                self.job_config.model.hf_assets_path, "tokenizer.json"
             ),
             tokenizer_config_json_path=os.path.join(
-                self.job_config.model.tokenizer_path, "tokenizer_config.json"
+                self.job_config.model.hf_assets_path, "tokenizer_config.json"
             ),
             generation_config_path=os.path.join(
-                self.job_config.model.tokenizer_path, "generation_config.json"
+                self.job_config.model.hf_assets_path, "generation_config.json"
             ),
         )