Remove Qwen tokenizer modification (#390)

Signed-off-by: Chenjie Luo <[email protected]>

Author: cjluo-nv

examples/llm_eval/gen_model_answer.py

@@ -180,14 +180,6 @@ def get_model_answers(
     # Model Optimizer modification
     tokenizer = get_tokenizer(model_path, trust_remote_code=args.trust_remote_code)
     if checkpoint_dir:
-        # get model type
-        last_part = os.path.basename(checkpoint_dir)
-        model_type = last_part.split("_")[0]
-        # Some models require to set pad_token and eos_token based on external config (e.g., qwen)
-        if model_type == "qwen":
-            tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
-            tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
-
         assert LLM is not None, "tensorrt_llm APIs could not be imported."
         model = LLM(checkpoint_dir, tokenizer=tokenizer)
     elif not nim_model:

examples/llm_eval/mmlu.py

@@ -253,14 +253,6 @@ def main(
     model_path = kwargs["model_path"]
     tokenizer = get_tokenizer(model_path, trust_remote_code=kwargs.get("trust_remote_code", False))
     if kwargs.get("checkpoint_dir"):
-        # get model type
-        last_part = os.path.basename(kwargs["checkpoint_dir"])
-        model_type = last_part.split("_")[0]
-        # Some models require to set pad_token and eos_token based on external config (e.g., qwen)
-        if model_type == "qwen":
-            tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
-            tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
-
         assert LLM is not None, "tensorrt_llm APIs could not be imported."
         medusa_choices = kwargs.get("medusa_choices")
         model = LLM(

examples/llm_eval/quantization_utils.py

@@ -53,10 +53,6 @@ def get_tokenizer(ckpt_path, max_seq_len=MAX_SEQ_LEN, trust_remote_code=False):
         padding_side="left",
         trust_remote_code=trust_remote_code,
     )
-    if type(tokenizer).__name__ == "QWenTokenizer":
-        # qwen use token id 151643 as pad and eos tokens
-        tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
-        tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
 
     # can't set attribute 'pad_token' for "<unk>"
     if tokenizer.pad_token != "<unk>":

examples/llm_ptq/example_utils.py

@@ -54,11 +54,6 @@ def get_tokenizer(ckpt_path, trust_remote_code=False, **kwargs):
         ckpt_path, trust_remote_code=trust_remote_code, **kwargs
     )
 
-    if "qwen" in type(tokenizer).__name__.lower():
-        # qwen use token id 151643 as pad and eos tokens
-        tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
-        tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
-
     # can't set attribute 'pad_token' for "<unk>"
     # We skip this step for Nemo models
     if tokenizer.pad_token != "<unk>" or tokenizer.pad_token is None:

examples/windows/accuracy_benchmark/quantization_utils.py

@@ -37,10 +37,6 @@ def get_tokenizer(ckpt_path, max_seq_len=MAX_SEQ_LEN, trust_remote_code=False):
         padding_side="left",
         trust_remote_code=trust_remote_code,
     )
-    if type(tokenizer).__name__ == "QWenTokenizer":
-        # qwen use token id 151643 as pad and eos tokens
-        tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
-        tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
 
     # can't set attribute 'pad_token' for "<unk>"
     if tokenizer.pad_token != "<unk>":