@@ -53,6 +53,7 @@ def __init__(
5353 super ().__init__ (specs , test_class , resource_settings = resource_settings )
5454 self .annotations = []
5555 self .overall_comments = []
56+ self .tags = []
5657
5758 def call_ai_feedback (self ) -> dict :
5859 """
@@ -81,7 +82,17 @@ def call_ai_feedback(self) -> dict:
8182 return results
8283
8384 submission_file = config .get ("submission" )
84- if self ._term_in_file (submission_file ):
85+ try :
86+ disallowed_term_in_file = self ._term_in_file (submission_file )
87+ except FileNotFoundError :
88+ results [test_label ] = {
89+ "title" : test_label ,
90+ "status" : "error" ,
91+ "message" : f'Could not file submission file "{ submission_file } ,
92+ }
93+ return results
94+
95+ if disallowed_term_in_file :
8596 results [test_label ] = {
8697 "title" : test_label ,
8798 "status" : "success" ,
@@ -96,15 +107,26 @@ def call_ai_feedback(self) -> dict:
96107 try :
97108 result = subprocess .run (cmd , capture_output = True , text = True , check = True , timeout = timeout , env = env )
98109 output = result .stdout
110+
111+ parsed = None
112+ try :
113+ parsed = json .loads (output )
114+ except json .JSONDecodeError :
115+ pass
116+ if isinstance (parsed , dict ):
117+ if "tags" in parsed :
118+ tags = parsed ["tags" ]
119+ self .tags .extend (tags )
120+ if "output" in parsed :
121+ output = parsed ["output" ]
122+
99123 if output_mode == "overall_comment" :
100124 self .overall_comments .append (output )
101125 results [test_label ] = {"title" : test_label , "status" : "success" }
102126 elif output_mode == "annotations" :
103- try :
104- annotations_data = json .loads (output )
105- annotations = annotations_data .get ("annotations" , annotations_data )
106- except json .JSONDecodeError as e :
107- raise ValueError (f"Invalid JSON in output for { test_label } { e } )
127+ if parsed is None :
128+ raise ValueError (f"Unable to parse the output of '{ output } )
129+ annotations = parsed .get ("annotations" , parsed )
108130 self .annotations .extend (annotations )
109131 results [test_label ] = {"title" : test_label , "status" : "success" }
110132 elif output_mode == "message" :
@@ -122,23 +144,20 @@ def _term_in_file(self, file_path: str) -> bool:
122144 term = "NO_EXTERNAL_AI_FEEDBACK"
123145 path = Path (file_path )
124146
125- try :
126- if path .suffix .lower () == ".pdf" :
127- with open (file_path , "rb" ) as f :
128- reader = PyPDF2 .PdfReader (f )
129- for page in reader .pages :
130- text = page .extract_text () or ""
131- if term in text :
132- return True
133- return False
134- else :
135- with open (file_path , "r" , encoding = "utf-8" ) as f :
136- for line in f :
137- if term in line :
138- return True
139- return False
140- except FileNotFoundError :
141- return True
147+ if path .suffix .lower () == ".pdf" :
148+ with open (file_path , "rb" ) as f :
149+ reader = PyPDF2 .PdfReader (f )
150+ for page in reader .pages :
151+ text = page .extract_text () or ""
152+ if term in text :
153+ return True
154+ return False
155+ else :
156+ with open (file_path , "r" , encoding = "utf-8" ) as f :
157+ for line in f :
158+ if term in line :
159+ return True
160+ return False
142161
143162 @Tester .run_decorator
144163 def run (self ) -> None :
@@ -157,3 +176,5 @@ def after_tester_run(self) -> None:
157176 print (self .test_class .format_annotations (self .annotations ))
158177 if self .overall_comments :
159178 print (self .test_class .format_overall_comment (self .overall_comments , separator = "\n \n " ))
179+ if self .tags :
180+ print (self .test_class .format_tags (self .tags ))
0 commit comments