@@ -201,8 +201,7 @@ struct llm_tokenizer_spm : llm_tokenizer {
201201};
202202
203203struct llm_tokenizer_spm_session {
204- llm_tokenizer_spm_session (const llama_vocab & vocab) : vocab(vocab),
205- spm_tokenizer (static_cast <const llm_tokenizer_spm *>(vocab.tokenizer)) {}
204+ llm_tokenizer_spm_session (const llama_vocab & vocab) : vocab(vocab) {}
206205
207206 void tokenize (const std::string & text, std::vector<llama_vocab::id> & output) {
208207
@@ -320,7 +319,8 @@ struct llm_tokenizer_spm_session {
320319 }
321320
322321 const llama_vocab & vocab;
323- const llm_tokenizer_spm * spm_tokenizer; // currently unused
322+ // currently unused
323+ // const llm_tokenizer_spm * spm_tokenizer;
324324
325325 std::vector<llm_symbol> symbols;
326326 llm_bigram_spm::queue work_queue;
@@ -668,8 +668,7 @@ struct llm_tokenizer_wpm : llm_tokenizer {
668668};
669669
670670struct llm_tokenizer_wpm_session {
671- llm_tokenizer_wpm_session (const llama_vocab & vocab) : vocab(vocab),
672- wpm_tokenizer (static_cast <const llm_tokenizer_wpm *>(vocab.tokenizer)) {}
671+ llm_tokenizer_wpm_session (const llama_vocab & vocab) : vocab(vocab) {}
673672
674673 void tokenize (const std::string & text, std::vector<llama_vocab::id> & output) {
675674 const auto & token_map = vocab.token_to_id ;
@@ -773,7 +772,8 @@ struct llm_tokenizer_wpm_session {
773772
774773private:
775774 const llama_vocab & vocab;
776- const llm_tokenizer_wpm * wpm_tokenizer;
775+ // currently unused
776+ // const llm_tokenizer_wpm * wpm_tokenizer;
777777};
778778
779779//
0 commit comments