@@ -40,33 +40,23 @@ static constexpr auto kUseSDPAWithKVCache = "use_sdpa_with_kv_cache";
4040
4141std::unique_ptr<::tokenizers::Tokenizer> load_tokenizer (
4242 const std::string& tokenizer_path) {
43- std::unique_ptr<::tokenizers::Tokenizer> tokenizer = nullptr ;
44-
45- // First try to load as a json tokenizer.
46- {
47- auto tokenizer = std::make_unique<tokenizers::HFTokenizer>();
48- if (tokenizer->load (tokenizer_path) == ::tokenizers::Error::Ok) {
49- ET_LOG (Info, " Loaded json tokenizer" );
50- return tokenizer;
51- }
43+
44+ auto json_tokenizer = std::make_unique<tokenizers::HFTokenizer>();
45+ if (json_tokenizer->load (tokenizer_path) == ::tokenizers::Error::Ok) {
46+ ET_LOG (Info, " Loaded json tokenizer" );
47+ return json_tokenizer;
5248 }
5349
54- // Try to load as tiktoken tokenizer.
55- {
56- auto tokenizer = get_tiktoken_for_llama ();
57- if (tokenizer->load (tokenizer_path) == ::tokenizers::Error::Ok) {
58- ET_LOG (Info, " Loaded TikToken tokenizer" );
59- return tokenizer;
60- }
50+ auto tiktoken_tokenizer = get_tiktoken_for_llama ();
51+ if (tiktoken_tokenizer->load (tokenizer_path) == ::tokenizers::Error::Ok) {
52+ ET_LOG (Info, " Loaded TikToken tokenizer" );
53+ return tiktoken_tokenizer;
6154 }
6255
63- // Try to load as BPE tokenizer.
64- {
65- auto tokenizer = std::make_unique<::tokenizers::Llama2cTokenizer>();
66- if (tokenizer->load (tokenizer_path) == ::tokenizers::Error::Ok) {
67- ET_LOG (Info, " Loaded BPE tokenizer" );
68- return tokenizer;
69- }
56+ auto bpe_tokenizer = std::make_unique<::tokenizers::Llama2cTokenizer>();
57+ if (bpe_tokenizer->load (tokenizer_path) == ::tokenizers::Error::Ok) {
58+ ET_LOG (Info, " Loaded BPE tokenizer" );
59+ return bpe_tokenizer;
7060 }
7161
7262 return nullptr ;
0 commit comments