Fixes

Author: mkardas

extract_tables.py

@@ -140,8 +140,6 @@ def fix_layout(layout):
 whitespace_tag_re = re.compile(r"<(bold|italic|red|green|blue)>(\s*)</\1>")
 dummy_close_tag_re = re.compile(r"</(bold|italic|red|green|blue)>(\s*)<\1>")
 def clear_cell(s):
-    if "BP by Lyapunov equatio" in s:
-        print(s)
     s = whitespace_tag_re.sub(r"\2", s)
     s = dummy_close_tag_re.sub(r"\2", s)
     return s.strip()

sota_extractor2/data/doc_utils.py

@@ -51,7 +51,8 @@ def put_footnote_anchors(soup):
     for elem in soup.select('.ltx_role_footnote > .ltx_note_mark'):
         ft = elem.parent
         id_str = ft.get('id')
-        elem.string = f" xxref-{_simplify_anchor(id_str)} "
+        if id_str:
+            elem.string = f" xxref-{_simplify_anchor(id_str)} "
 
     for elem in soup.select('.ltx_note_content > .ltx_tag_note'):
         ft = elem.find_parent(class_="ltx_role_footnote")

sota_extractor2/data/elastic.py

@@ -12,6 +12,7 @@
 from sota_extractor2.data.doc_utils import get_text, content_in_section, group_content, read_html, put_dummy_anchors, clean_abstract
 from .. import config
 from pathlib import Path
+import sys
 
 
 def setup_default_connection():
@@ -200,38 +201,41 @@ def parse_html(cls, soup, paper_id):
 
         fragments = Fragments()
         doc = soup.find("article")
-        footnotes = doc.select(".ltx_role_footnote > .ltx_note_outer")
-        for ft in footnotes:
-            ft.extract()
-
-        idx = 0
-        for idx, idx2, section_header, content in group_content(doc):
-            content = content.strip()
-            if p.abstract == "" and "abstract" in section_header.lower():
-                p.abstract = clean_abstract(content)
-            else:
+        if doc:
+            footnotes = doc.select(".ltx_role_footnote > .ltx_note_outer")
+            for ft in footnotes:
+                ft.extract()
+
+            idx = 0
+            for idx, idx2, section_header, content in group_content(doc):
+                content = content.strip()
+                if p.abstract == "" and "abstract" in section_header.lower():
+                    p.abstract = clean_abstract(content)
+                else:
+                    order = (idx + 1) * 1000 + idx2
+                    f = Fragment(
+                        paper_id=paper_id,
+                        order=order,
+                        header=section_header,
+                        text=content,
+                        meta={'id': f"{paper_id}-{order}"}
+                    )
+                    fragments.append(f)
+            idx += 1
+            idx2 = 0
+            for ft in footnotes:
                 order = (idx + 1) * 1000 + idx2
                 f = Fragment(
-                    paper_id=paper_id,
-                    order=order,
-                    header=section_header,
-                    text=content,
-                    meta={'id': f"{paper_id}-{order}"}
+                        paper_id=paper_id,
+                        order=order,
+                        header="xxanchor-footnotes Footnotes",
+                        text=get_text(ft),
+                        meta={'id': f"{paper_id}-{order}"}
                 )
                 fragments.append(f)
-        idx += 1
-        idx2 = 0
-        for ft in footnotes:
-            order = (idx + 1) * 1000 + idx2
-            f = Fragment(
-                    paper_id=paper_id,
-                    order=order,
-                    header="xxanchor-footnotes Footnotes",
-                    text=get_text(ft),
-                    meta={'id': f"{paper_id}-{order}"}
-            )
-            fragments.append(f)
-            idx2 += 1
+                idx2 += 1
+        else:
+            print(f"No article found for {paper_id}", file=sys.stderr)
         p.fragments = fragments
         return p
 

sota_extractor2/data/table.py

@@ -42,7 +42,7 @@ def str2cell(s):
 
 def read_str_csv(filename):
     try:
-        df = pd.read_csv(filename, header=None, dtype=str).fillna('')
+        df = pd.read_csv(filename, header=None, dtype=str, keep_default_na=False)
     except pd.errors.EmptyDataError:
         df = pd.DataFrame()
     return df