File tree Expand file tree Collapse file tree 1 file changed +3
-4
lines changed Expand file tree Collapse file tree 1 file changed +3
-4
lines changed Original file line number Diff line number Diff line change @@ -5582,15 +5582,14 @@ def set_vocab(self):
55825582                merged  =  QwenModel .bpe (mergeable_ranks , token , max_rank = rank )
55835583                if  len (merged ) ==  2 :
55845584                    merges .append (' ' .join (map (QwenModel .token_bytes_to_string , merged )))
5585-             
55865585
55875586            # Build token list 
55885587            vocab_size  =  self .hparams ["vocab_size" ]
55895588            special_tokens  =  tokenizer .special_tokens 
55905589            reverse_vocab  =  {id_  : encoded_tok  for  encoded_tok , id_  in  {** vocab , ** special_tokens }.items ()}
55915590            tokens : list [str ] =  []
55925591            toktypes : list [int ] =  []
5593-              
5592+ 
55945593            for  i  in  range (vocab_size ):
55955594                if  i  not  in   reverse_vocab :
55965595                    tokens .append (f"[PAD{ i }  ]" )
@@ -5602,13 +5601,13 @@ def set_vocab(self):
56025601                        toktypes .append (gguf .TokenType .CONTROL )
56035602                    else :
56045603                        toktypes .append (gguf .TokenType .NORMAL )
5605-              
5604+ 
56065605            self .gguf_writer .add_tokenizer_model ("gpt2" )
56075606            self .gguf_writer .add_tokenizer_pre (tokpre )
56085607            self .gguf_writer .add_token_list (tokens )
56095608            self .gguf_writer .add_token_types (toktypes )
56105609            self .gguf_writer .add_token_merges (merges )
5611-              
5610+ 
56125611            special_vocab  =  gguf .SpecialVocab (self .dir_model , load_merges = False )
56135612            special_vocab .add_to_gguf (self .gguf_writer )
56145613        else :
    
 
   
 
     
   
   
          
     
  
    
     
 
    
      
     
 
     
    You can’t perform that action at this time.
  
 
    
  
     
    
      
        
     
 
       
      
     
   
 
    
    
  
 
  
 
     
    
0 commit comments