Adapt tests to layouts and styles

* compare all output files
* sort styles to make output deterministic
* fix repeated anchors to appendix and bibliography
* flush output at the end of processing

Author: mkardas

Makefile

@@ -20,13 +20,11 @@ all:	$(ANNOTATIONS_DIR)/pdfs-urls.csv $(ANNOTATIONS_DIR)/sources-urls.csv extrac
 
 .PHONY: test
 test: DATA_DIR = test/data
-test: TABLE_FILE = $(TABLES_DIR)/paper/table_01.csv
 test:
 	mkdir -p $(ARCHIVES_DIR)
 	tar czf $(ARCHIVES_DIR)/paper.gz -C test/src .
 	$(MAKE) DATA_DIR=$(DATA_DIR) --always-make extract_all
-	cat $(TABLE_FILE)
-	diff $(TABLE_FILE) test/src/table_01.csv
+	diff -r $(TABLES_DIR) test/expected
 
 .PHONY: extract_all extract_texts extract_tables fix_htmls_all convert_all unpack_all
 

extract_tables.py

@@ -80,9 +80,9 @@ class LayoutCell:
     span: Set[str]
 
     def __str__(self):
-        borders = ['border-'+x for x in self.borders]
-        align = ['align-'+x for x in self.align]
-        span = ['span-'+x for x in self.span]
+        borders = ['border-'+x for x in sorted(list(self.borders))]
+        align = ['align-'+x for x in sorted(list(self.align))]
+        span = ['span-'+x for x in sorted(list(self.span))]
         header = ["header"] if self.header else []
         return ' '.join(borders + align + span + header)
 
@@ -296,7 +296,7 @@ def set_ids_by_labels(soup):
 
 alg_id_re = re.compile(r"^alg(orithm)?[0-9]+")
 def perhaps_not_tabular(table, float_div):
-    classes = float_div.attrs.get("class")
+    classes = float_div.attrs.get("class", [])
     if 'ltx_table' in classes:
         return False
     if 'ltx_figure' in classes:

sota_extractor2/data/doc_utils.py

@@ -20,7 +20,7 @@ def _insert_anchor(el, anchor_id, prefix="xxanchor"):
 
 def put_dummy_anchors(soup):
     for elem in soup.select(
-            '.ltx_appendix, .ltx_bibliography, .ltx_bibitem, ' + \
+            '.ltx_bibitem, ' + \
             '.ltx_figure, .ltx_float, ' + \
             '.ltx_picture, .ltx_theorem'):
         id_str = elem.get('id', '')
@@ -248,7 +248,7 @@ def flush(self):
             yield r
 
     def new_section(self, header_el):
-        if not self.section_output: # output (possibly) empty section so header won't be lost
+        if not self.section_output or self.out: # output (possibly) empty section so header won't be lost
             yield from self.flush()
         self.section_output = False
         self.in_section = True
@@ -260,6 +260,8 @@ def append(self, el):
         t = get_text(el).strip()
         if t != "":
             self.out.append(t)
+            return True
+        return False
 
     def group_content(self, doc):
         for el in walk(doc):
@@ -269,10 +271,14 @@ def group_content(self, doc):
             elif el.name == "h1":
                 continue
             elif 'ltx_para' in classes or el.name == "figure" or 'ltx_bibitem' in classes:
-                self.append(el)
-                yield from self.flush()
+                has_content = self.append(el)
+                if has_content:
+                    yield from self.flush()
             else:
                 self.append(el)
+        self.in_section = True
+        if not self.section_output or self.out:
+            yield from self.flush()
 
 
 def group_content(doc):

sota_extractor2/data/elastic.py

@@ -207,8 +207,6 @@ def parse_html(cls, soup, paper_id):
         idx = 0
         for idx, idx2, section_header, content in group_content(doc):
             content = content.strip()
-            if content == "":
-                continue
             if p.abstract == "" and "abstract" in section_header.lower():
                 p.abstract = clean_abstract(content)
             else:

test/expected/paper/layout_01.csv

@@ -0,0 +1,5 @@
+border-tt align-left,border-tt align-center,border-tt align-right
+border-t align-left,border-t align-center,border-t align-right
+align-left,align-center,align-right
+border-t align-left,border-t align-center,border-t align-right
+border-bb align-left,border-bb align-center,border-bb align-right

test/expected/paper/layout_02.csv

@@ -0,0 +1,4 @@
+border-tt align-left,border-tt align-center,border-tt align-right
+border-t align-left,border-t align-center,border-t align-right
+align-left,align-center,align-right
+border-bb border-t align-left,border-bb border-t align-center,border-bb border-t

test/expected/paper/metadata.json

@@ -0,0 +1 @@
+[{"filename": "table_01.csv", "layout": "layout_01.csv", "caption": "Table 1: A table.", "figure_id": "S3.T1"}, {"filename": "table_02.csv", "layout": "layout_02.csv", "caption": "Table 2: A table.", "figure_id": "S3.T2"}]

test/src/table_01.csv renamed to test/expected/paper/table_01.csv

@@ -0,0 +1,4 @@
+<bold>bold text</bold>,<italic>italic text</italic>,<italic><bold>bold italic text</bold></italic>
+<red>red text</red>,<green>green text</green>,<blue>blue text</blue>
+<bold>5.4%</bold>,<italic>3.8%</italic>,<bold>11.2</bold>±0.15
+<bold>an <italic>italic</italic> text inside bold</bold>,<red><bold>bold red</bold></red>,

test/expected/paper/table_02.csv

@@ -0,0 +1 @@
+{"title":"DILBERT: Distilling Inner Latent BERT variables","authors":"John Doe","abstract":"In this paper we achieve state-of-the-art performance in random number generation.","fragments":[{"paper_id":"paper","order":1000,"header":"xxanchor-S1 1 Introduction","text":""},{"paper_id":"paper","order":2000,"header":"xxanchor-S2 2 Model","text":""},{"paper_id":"paper","order":3000,"header":"xxanchor-S2SS1 2.1 Preprocessing","text":""},{"paper_id":"paper","order":4000,"header":"xxanchor-S2SS2 2.2 Architecture","text":""},{"paper_id":"paper","order":5000,"header":"xxanchor-S3 3 Experiments","text":"In this section we present Table xxref-S3T1."},{"paper_id":"paper","order":5001,"header":"xxanchor-S3 3 Experiments","text":"xxtable-xxanchor-S3T1 Table 1: A table."},{"paper_id":"paper","order":5002,"header":"xxanchor-S3 3 Experiments","text":"xxtable-xxanchor-S3T2 Table 2: A table."}]}