Skip to content

Commit c908968

Browse files
committed
Fix local shadowing
1 parent ac83e12 commit c908968

File tree

1 file changed

+13
-23
lines changed

1 file changed

+13
-23
lines changed

examples/models/llama/runner/runner.cpp

Lines changed: 13 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -40,33 +40,23 @@ static constexpr auto kUseSDPAWithKVCache = "use_sdpa_with_kv_cache";
4040

4141
std::unique_ptr<::tokenizers::Tokenizer> load_tokenizer(
4242
const std::string& tokenizer_path) {
43-
std::unique_ptr<::tokenizers::Tokenizer> tokenizer = nullptr;
44-
45-
// First try to load as a json tokenizer.
46-
{
47-
auto tokenizer = std::make_unique<tokenizers::HFTokenizer>();
48-
if (tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
49-
ET_LOG(Info, "Loaded json tokenizer");
50-
return tokenizer;
51-
}
43+
44+
auto json_tokenizer = std::make_unique<tokenizers::HFTokenizer>();
45+
if (json_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
46+
ET_LOG(Info, "Loaded json tokenizer");
47+
return json_tokenizer;
5248
}
5349

54-
// Try to load as tiktoken tokenizer.
55-
{
56-
auto tokenizer = get_tiktoken_for_llama();
57-
if (tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
58-
ET_LOG(Info, "Loaded TikToken tokenizer");
59-
return tokenizer;
60-
}
50+
auto tiktoken_tokenizer = get_tiktoken_for_llama();
51+
if (tiktoken_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
52+
ET_LOG(Info, "Loaded TikToken tokenizer");
53+
return tiktoken_tokenizer;
6154
}
6255

63-
// Try to load as BPE tokenizer.
64-
{
65-
auto tokenizer = std::make_unique<::tokenizers::Llama2cTokenizer>();
66-
if (tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
67-
ET_LOG(Info, "Loaded BPE tokenizer");
68-
return tokenizer;
69-
}
56+
auto bpe_tokenizer = std::make_unique<::tokenizers::Llama2cTokenizer>();
57+
if (bpe_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
58+
ET_LOG(Info, "Loaded BPE tokenizer");
59+
return bpe_tokenizer;
7060
}
7161

7262
return nullptr;

0 commit comments

Comments
 (0)