Update tokenizer.py (#435)

matthewkotila · web-flow · commit f5203fa8e1a5 · 2025-08-15T14:56:04.000-07:00
diff --git a/genai-perf/genai_perf/tokenizer.py b/genai-perf/genai_perf/tokenizer.py
@@ -53,7 +53,7 @@ def set_tokenizer(self, name: str, trust_remote_code: bool, revision: str) -> No
                 from transformers import logging as token_logger
 
                 token_logger.set_verbosity_error()
-                tokenizer = AutoTokenizer.from_pretrained(
+                tokenizer = AutoTokenizer.from_pretrained( # nosec
                     name, trust_remote_code=trust_remote_code, revision=revision
                 )
         except Exception as e:

Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@ def set_tokenizer(self, name: str, trust_remote_code: bool, revision: str) -> No`
`53`	`53`	`from transformers import logging as token_logger`
`54`	`54`
`55`	`55`	`token_logger.set_verbosity_error()`
`56`		`- tokenizer = AutoTokenizer.from_pretrained(`
	`56`	`+ tokenizer = AutoTokenizer.from_pretrained( # nosec`
`57`	`57`	`name, trust_remote_code=trust_remote_code, revision=revision`
`58`	`58`	`)`
`59`	`59`	`except Exception as e:`