Skip to content

Commit 768c43f

Browse files
committed
remove unused fileds to avoid unused filed build error
1 parent 25d4599 commit 768c43f

File tree

1 file changed

+6
-6
lines changed

1 file changed

+6
-6
lines changed

src/llama-vocab.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -201,8 +201,7 @@ struct llm_tokenizer_spm : llm_tokenizer {
201201
};
202202

203203
struct llm_tokenizer_spm_session {
204-
llm_tokenizer_spm_session(const llama_vocab & vocab) : vocab(vocab),
205-
spm_tokenizer(static_cast<const llm_tokenizer_spm *>(vocab.tokenizer)) {}
204+
llm_tokenizer_spm_session(const llama_vocab & vocab) : vocab(vocab) {}
206205

207206
void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
208207

@@ -320,7 +319,8 @@ struct llm_tokenizer_spm_session {
320319
}
321320

322321
const llama_vocab & vocab;
323-
const llm_tokenizer_spm * spm_tokenizer; // currently unused
322+
// currently unused
323+
// const llm_tokenizer_spm * spm_tokenizer;
324324

325325
std::vector<llm_symbol> symbols;
326326
llm_bigram_spm::queue work_queue;
@@ -668,8 +668,7 @@ struct llm_tokenizer_wpm : llm_tokenizer {
668668
};
669669

670670
struct llm_tokenizer_wpm_session {
671-
llm_tokenizer_wpm_session(const llama_vocab & vocab) : vocab(vocab),
672-
wpm_tokenizer(static_cast<const llm_tokenizer_wpm *>(vocab.tokenizer)) {}
671+
llm_tokenizer_wpm_session(const llama_vocab & vocab) : vocab(vocab) {}
673672

674673
void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
675674
const auto & token_map = vocab.token_to_id;
@@ -773,7 +772,8 @@ struct llm_tokenizer_wpm_session {
773772

774773
private:
775774
const llama_vocab & vocab;
776-
const llm_tokenizer_wpm * wpm_tokenizer;
775+
// currently unused
776+
// const llm_tokenizer_wpm * wpm_tokenizer;
777777
};
778778

779779
//

0 commit comments

Comments
 (0)