Clean redundant tags

Author: mkardas

extract_tables.py

@@ -134,9 +134,22 @@ def fix_layout(layout):
                     cell.borders -= {"b", "bb", "t", "tt"}
 
 
+# does not deal with nested tags
+# f.e., </bold></red><red><bold>
+# or <bold><bold>
+whitespace_tag_re = re.compile(r"<(bold|italic|red|green|blue)>(\s*)</\1>")
+dummy_close_tag_re = re.compile(r"</(bold|italic|red|green|blue)>(\s*)<\1>")
+def clear_cell(s):
+    if "BP by Lyapunov equatio" in s:
+        print(s)
+    s = whitespace_tag_re.sub(r"\2", s)
+    s = dummy_close_tag_re.sub(r"\2", s)
+    return s.strip()
+
+
 def decouple_layout(df):
     split = df.applymap(lambda x: ("", "") if x == "" else x.split(";", 1))
-    tab = split.applymap(lambda x: x[1])
+    tab = split.applymap(lambda x: clear_cell(x[1]))
     layout = split.applymap(lambda x: to_layout(x[0]))
     fix_layout(layout)
     return tab, layout

sota_extractor2/config.py

@@ -10,7 +10,7 @@
 
 # otherwise use this files
 data = Path("/mnt/efs/pwc/data")
-goldtags_dump = data / "dumps" / "goldtags-2019.07.16_2214.json.gz"
+goldtags_dump = data / "dumps" / "goldtags-2019.07.31_1454-htlatex-latexml.json.gz"
 
 
 elastic = dict(hosts=['localhost'], timeout=20)