Include tasks in flattened json

Author: mkardas

flatten_evaltab.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-jq -c '.. | select(.datasets?).datasets | .[] | .dataset as $dataset | .sota.rows[] | {paper_url, paper_title, model_name} as $paper | .metrics | . as $metrics | keys[] | {dataset: $dataset, metric_name: ., metric_value: $metrics[.], paper_url: $paper.paper_url, paper_title: $paper.paper_title, model_name: $paper.model_name }' "$1" | grep arxiv\.org | jq -s '.'
+jq -c '.. | select(.datasets?) | .task as $task | .datasets | .[] | .dataset as $dataset | .sota.rows[] | {paper_url, paper_title, model_name} as $paper | .metrics | . as $metrics | keys[] | {dataset: $dataset, metric_name: ., metric_value: $metrics[.], paper_url: $paper.paper_url, paper_title: $paper.paper_title, model_name: $paper.model_name, task: $task }' "$1" | grep arxiv\.org | jq -s '.'

label_tables.py

@@ -152,14 +152,18 @@ def match_metric(metric, tables, value):
 ]
 
 
+def empty_celltags_like(table):
+    return = pd.DataFrame().reindex_like(table).fillna('')
+
+
 def mark_with_best_comparator(task_name, dataset_name, metric_name, arxiv_id, table, values):
     max_hits = 0
     best_tags = None
     rows, cols = table.shape
 
     for comparator in comparators:
         hits = 0
-        cell_tags = pd.DataFrame().reindex_like(table).fillna('')
+        cell_tags = empty_celltags_like(table)
         for col in range(cols):
             for row in range(rows):
                 for val in table.iloc[row, col]:
@@ -180,15 +184,36 @@ def mark_with_best_comparator(task_name, dataset_name, metric_name, arxiv_id, ta
             best_tags = cell_tags
 
     return best_tags
+
+
+def normalize_string(s):
+    return s.lower.strip()
+
+
+def match_str(a, b):
+    return normalize_string(a) == normalize_string(b)
+
+
+def mark_strings(table, tags, values):
+    cell_tags = empty_celltags_like(table)
+    beg, end = tags
+    rows, cols = table.shape
+    for col in range(cols):
+            for row in range(rows):
+                for s in values:
+                    real = table.iloc[row, col]
+                    if match_str(real, s):
+                        cell_tags += f"{beg}{s}{end}"
+    return cell_tags
 
 
 metatables = {}
 def match_many(output_dir, task_name, dataset_name, metric_name, tables, values):
     for arxiv_id in tables:
         for table in tables[arxiv_id]:
             best = mark_with_best_comparator(task_name, dataset_name, metric_name, arxiv_id, tables[arxiv_id][table], values)
+            global metatables
             if best is not None:
-                global metatables
                 key = (arxiv_id, table)
                 if key in metatables:
                     metatables[key] += best