feat: add metadata tracking to document elements (#225)

* add metadata field to elements

* metadata tracking for pdf/image

* metadata for html

* update expected outputs

* metadata for the rest of the document types

* take out file metadata for now

* add url to tables

* added metadata to test_auto

* bump version

* added coordinates to __init__

* fix coordinates in tests

Author: MthwRobinson

CHANGELOG.md

@@ -1,8 +1,9 @@
-## 0.4.9-dev2
+## 0.4.9
 
 * Added ingest modules and s3 connector
 * Default to `url=None` for `partition_pdf` and `partition_image`
 * Add ability to skip English specific check by setting the `UNSTRUCTURED_LANGUAGE` env var to `""`.
+* Document `Element` objects now track metadata
 
 ## 0.4.8
 

test_unstructured/partition/test_auto.py

@@ -25,6 +25,7 @@ def test_auto_partition_email_from_filename():
     elements = partition(filename=filename)
     assert len(elements) > 0
     assert elements == EXPECTED_EMAIL_OUTPUT
+    assert elements[0].metadata.filename == filename
 
 
 def test_auto_partition_email_from_file():
@@ -83,6 +84,7 @@ def test_auto_partition_docx_with_filename(mock_docx_document, expected_docx_ele
 
     elements = partition(filename=filename)
     assert elements == expected_docx_elements
+    assert elements[0].metadata.filename == filename
 
 
 def test_auto_partition_docx_with_file(mock_docx_document, expected_docx_elements, tmpdir):
@@ -98,6 +100,7 @@ def test_auto_partition_html_from_filename():
     filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "example-10k.html")
     elements = partition(filename=filename)
     assert len(elements) > 0
+    assert elements[0].metadata.filename == filename
 
 
 def test_auto_partition_html_from_file():
@@ -129,6 +132,7 @@ def test_auto_partition_text_from_filename():
     elements = partition(filename=filename)
     assert len(elements) > 0
     assert elements == EXPECTED_TEXT_OUTPUT
+    assert elements[0].metadata.filename == filename
 
 
 def test_auto_partition_text_from_file():
@@ -149,6 +153,8 @@ def test_auto_partition_pdf_from_filename():
     assert isinstance(elements[1], NarrativeText)
     assert elements[1].text.startswith("Zejiang Shen 1")
 
+    assert elements[0].metadata.filename == filename
+
 
 def test_auto_partition_pdf_from_file():
     filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "layout-parser-paper-fast.pdf")
@@ -206,6 +212,7 @@ def test_auto_partition_pptx_from_filename():
     filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "fake-power-point.pptx")
     elements = partition(filename=filename)
     assert elements == EXPECTED_PPTX_OUTPUT
+    assert elements[0].metadata.filename == filename
 
 
 def test_auto_with_page_breaks():

test_unstructured/partition/test_common.py

@@ -18,7 +18,7 @@ def test_normalize_layout_element_dict():
         "text": "Some lovely text",
     }
     element = common.normalize_layout_element(layout_element)
-    assert element == Title(text="Some lovely text", coordinates=[[2, 2], [3, 4], [5, 6], [7, 8]])
+    assert element == Title(text="Some lovely text", coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]])
 
 
 def test_normalize_layout_element_dict_caption():
@@ -29,7 +29,7 @@ def test_normalize_layout_element_dict_caption():
     }
     element = common.normalize_layout_element(layout_element)
     assert element == FigureCaption(
-        text="Some lovely text", coordinates=[[2, 2], [3, 4], [5, 6], [7, 8]]
+        text="Some lovely text", coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]]
     )
 
 
@@ -40,7 +40,7 @@ def test_normalize_layout_element_dict_misc():
         "text": "Some lovely text",
     }
     element = common.normalize_layout_element(layout_element)
-    assert element == Text(text="Some lovely text", coordinates=[[2, 2], [3, 4], [5, 6], [7, 8]])
+    assert element == Text(text="Some lovely text", coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]])
 
 
 def test_normalize_layout_element_layout_element():
@@ -51,7 +51,7 @@ def test_normalize_layout_element_layout_element():
     )
     element = common.normalize_layout_element(layout_element)
     assert element == NarrativeText(
-        text="Some lovely text", coordinates=[[2, 2], [3, 4], [5, 6], [7, 8]]
+        text="Some lovely text", coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]]
     )
 
 

test_unstructured/staging/test_base_staging.py

@@ -48,10 +48,9 @@ def test_convert_to_isd_csv(output_csv_file):
         isd_csv_string = base.convert_to_isd_csv(elements)
         csv_file.write(isd_csv_string)
 
-    fieldnames = ["type", "text"]
     with open(output_csv_file, "r") as csv_file:
         csv_rows = csv.DictReader(csv_file)
-        assert all(set(row.keys()) == set(fieldnames) for row in csv_rows)
+        assert all(set(row.keys()) == set(base.TABLE_FIELDNAMES) for row in csv_rows)
 
 
 def test_convert_to_dataframe():