contest: subcases: handle retry cases

In case of failure, some tests are restarted. That's great, but when
parsing the subcases (nested tests), the previous results were
overridden. Also, when displaying the results, the 'retry' field from
the main test was re-used in each subcases: quite confusing for the
reader.

Now, in case of retry, a new dictionary under the 'results-retry' key is
stored in the 'json_full' entry in the DB. This dictionary uses the
subcase titles as keys, and the results as values. Thanks to that, when
querying the subcases results, this dictionary is used to properly set
the 'retry' field for each subcase.

Signed-off-by: Matthieu Baerts (NGI0) <[email protected]>

Author: matttbe

contest/backend/query.py

@@ -61,10 +61,15 @@ def result_as_l2(raw):
             for case in l1["results"]:
                 data = l1.copy()
                 del data["results"]
-                if "time" in data:
-                    del data["time"]
+                for del_key in ("results-retry", "time", "retry"):
+                    if del_key in data:
+                        del data[del_key]
+
                 data |= case
                 data["test"] = l1["test"] + '.' + case["test"]
+                if "results-retry" in l1 and case["test"] in l1["results-retry"]:
+                    data["retry"] = l1["results-retry"][case["test"]]
+
                 flat.append(data)
     row["results"] = flat
     return json.dumps(row)

contest/remote/vmksft-p.py

@@ -69,8 +69,11 @@ def get_prog_list(vm, targets):
     return [(e.split(":")[0].strip(), e.split(":")[1].strip()) for e in targets]
 
 
-def _parse_nested_tests(full_run):
-    tests = []
+def _parse_nested_tests(full_run, flat):
+    if flat:
+        tests = {}
+    else:
+        tests = []
     nested_tests = False
 
     result_re = re.compile(r"(not )?ok (\d+)( -)? ([^#]*[^ ])( +# +)?([^ ].*)?$")
@@ -110,7 +113,10 @@ def _parse_nested_tests(full_run):
 
         r['result'] = result
 
-        tests.append(r)
+        if flat:
+            tests[r['test']] = r['result']
+        else:
+            tests.append(r)
 
     return tests
 
@@ -196,9 +202,10 @@ def _vm_thread(config, results_path, thr_id, hard_stop, in_queue, out_queue):
 
         if config.getboolean('ksft', 'nested_tests', fallback=False):
             # this will only parse nested tests inside the TAP comments
-            nested_tests = _parse_nested_tests(vm.log_out)
+            nested_tests = _parse_nested_tests(vm.log_out, is_retry)
             if nested_tests:
-                outcome['results'] = nested_tests
+                nested_key = 'results-retry' if is_retry else 'results'
+                outcome[nested_key] = nested_tests
 
             print(f"INFO: thr-{thr_id} {prog} >> nested tests: {len(nested_tests)}")
 
@@ -318,7 +325,7 @@ def test(binfo, rinfo, cbarg):
             'result': r["result"],
             'link': link + '/' + r['file_name']
         }
-        for key in ['time', 'retry', 'crashes', 'results']:
+        for key in ['time', 'retry', 'crashes', 'results', 'results-retry']:
             if key in r:
                 outcome[key] = r[key]
         cases.append(outcome)

contest/results-fetcher.py

@@ -96,9 +96,10 @@ def psql_json_split(self, data):
         full = None
 
         for row in normal["results"]:
-            if "results" in row:
-                full = True
-                del row["results"]
+            for rkey in ("results", "results-retry"):
+                if rkey in row:
+                    full = True
+                    del row[rkey]
 
         if full:
             full = json.dumps(data)