File tree Expand file tree Collapse file tree 1 file changed +6
-5
lines changed Expand file tree Collapse file tree 1 file changed +6
-5
lines changed Original file line number Diff line number Diff line change @@ -240,11 +240,6 @@ def get_existing_models(convert_py):
240240 if tokt == TOKENIZER_TYPE .SPM or tokt == TOKENIZER_TYPE .UGM :
241241 continue
242242
243- # Skip if the tokenizer folder does not exist or there are other download issues previously
244- if not os .path .exists (f"models/tokenizers/{ name } " ):
245- logger .warning (f"Directory for tokenizer { name } not found. Skipping..." )
246- continue
247-
248243 # create the tokenizer
249244 if chkhsh is not None :
250245 # if the model has a pre-computed hash, use it
@@ -254,6 +249,12 @@ def get_existing_models(convert_py):
254249 chkhsh = existing_models [name ]
255250 else :
256251 # otherwise, compute the hash of the tokenizer
252+
253+ # Skip if the tokenizer folder does not exist or there are other download issues previously
254+ if not os .path .exists (f"models/tokenizers/{ name } " ):
255+ logger .warning (f"Directory for tokenizer { name } not found. Skipping..." )
256+ continue
257+
257258 try :
258259 logger .info (f"Loading tokenizer from { f'models/tokenizers/{ name } ' } ..." )
259260 if name == "t5" :
You can’t perform that action at this time.
0 commit comments