6666    "mistralai/Mixtral-8x7B-Instruct-v0.1" ,
6767]
6868
69+ 
6970def  raise_exception (message : str ):
7071    raise  ValueError (message )
7172
73+ 
7274def  tojson (x , ensure_ascii = False , indent = None , separators = None , sort_keys = False ):
7375    return  json .dumps (x , ensure_ascii = ensure_ascii , indent = indent , separators = separators , sort_keys = sort_keys )
7476
77+ 
7578def  strftime_now (format ):
7679    return  datetime .now ().strftime (format )
7780
81+ 
7882def  handle_chat_template (model_id , variant , template_src ):
7983    print (f"# { model_id }   @ { variant }  " , flush = True )
8084    model_name  =  model_id .replace ("/" , "-" )
@@ -87,12 +91,12 @@ def handle_chat_template(model_id, variant, template_src):
8791    print (f"- { template_file }  " , flush = True )
8892
8993    env  =  jinja2 .Environment (
90-       trim_blocks = True ,
91-       lstrip_blocks = True ,
92-       # keep_trailing_newline=False, 
93-       extensions = [
94-         jinja2 .ext .loopcontrols 
95-       ])
94+          trim_blocks = True ,
95+          lstrip_blocks = True ,
96+          # keep_trailing_newline=False, 
97+          extensions = [
98+              jinja2 .ext .loopcontrols 
99+          ])
96100    env .filters ['tojson' ] =  tojson 
97101    env .globals ['raise_exception' ] =  raise_exception 
98102    env .globals ['strftime_now' ] =  strftime_now 
@@ -118,23 +122,24 @@ def handle_chat_template(model_id, variant, template_src):
118122        print (f"- { output_file }  " , flush = True )
119123        try :
120124            output  =  template .render (** context )
121-         except :
125+         except   Exception   as   e1 :
122126            # Some templates (e.g. Phi-3-medium-128k's) expect a non-null "content" key in each message. 
123127            for  message  in  context ["messages" ]:
124128                if  message .get ("content" ) is  None :
125129                    message ["content" ] =  "" 
126130
127131            try :
128132                output  =  template .render (** context )
129-             except  Exception  as  e :
130-                 print (f"  ERROR: { e }  " , flush = True )
131-                 output  =  f"ERROR: { e }  " 
133+             except  Exception  as  e2 :
134+                 print (f"  ERROR: { e2 }  (after first error:  { e1 } ) " , flush = True )
135+                 output  =  f"ERROR: { e2 }  " 
132136
133137        with  open (output_file , 'w' ) as  f :
134138            f .write (output )
135139
136140    print ()
137141
142+ 
138143def  main ():
139144    for  dir  in  ['tests/chat/templates' , 'tests/chat/goldens' ]:
140145        if  not  os .path .isdir (dir ):
@@ -149,7 +154,7 @@ def main():
149154
150155        try :
151156            config  =  json .loads (config_str )
152-         except  json .JSONDecodeError   as   e :
157+         except  json .JSONDecodeError :
153158            # Fix https://huggingface.co/NousResearch/Meta-Llama-3-8B-Instruct/blob/main/tokenizer_config.json 
154159            # (Remove extra '}' near the end of the file) 
155160            config  =  json .loads (re .sub (r'\}([\n\s]*\}[\n\s]*\],[\n\s]*"clean_up_tokenization_spaces")' , r'\1' , config_str ))
@@ -161,5 +166,6 @@ def main():
161166            for  ct  in  chat_template :
162167                handle_chat_template (model_id , ct ['name' ], ct ['template' ])
163168
169+ 
164170if  __name__  ==  '__main__' :
165171    main ()
0 commit comments