Update src/llama.cpp

ggerganov · web-flow · commit cd145b11c003 · 2024-09-25T16:19:55.000+03:00
diff --git a/src/llama.cpp b/src/llama.cpp
@@ -6459,7 +6459,7 @@ static void llm_load_vocab(
         GGML_ASSERT(!ids.empty() && "model vocab missing newline token");
         vocab.linefeed_id = ids[0];
     } else {
-        const std::vector<int> ids = llama_tokenize_internal(model.vocab, "\xC4\x8A", false); // U+010A
+        const std::vector<int> ids = llama_tokenize_internal(vocab, "\xC4\x8A", false); // U+010A
         GGML_ASSERT(!ids.empty() && "model vocab missing newline token");
         vocab.linefeed_id = ids[0];
     }

Original file line number	Diff line number	Diff line change
`@@ -6459,7 +6459,7 @@ static void llm_load_vocab(`
`6459`	`6459`	`GGML_ASSERT(!ids.empty() && "model vocab missing newline token");`
`6460`	`6460`	`vocab.linefeed_id = ids[0];`
`6461`	`6461`	`} else {`
`6462`		`- const std::vector<int> ids = llama_tokenize_internal(model.vocab, "\xC4\x8A", false); // U+010A`
	`6462`	`+ const std::vector<int> ids = llama_tokenize_internal(vocab, "\xC4\x8A", false); // U+010A`
`6463`	`6463`	`GGML_ASSERT(!ids.empty() && "model vocab missing newline token");`
`6464`	`6464`	`vocab.linefeed_id = ids[0];`
`6465`	`6465`	`}`