feat: add sender and receive info to element metadata for emails (#439)

* add header metadata for .eml messages

* sent to and from are lists

* add metadata for outlook emails

* version and changelog

Author: MthwRobinson

CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.5.10-dev0
+
+### Enhancements
+
+* Add sender, recipient, date, and subject to element metadata for emails
+
+### Features
+
+### Fixes
+
 ## 0.5.9
 
 ### Enhancements

test_unstructured/partition/test_email.py

@@ -5,7 +5,13 @@
 
 import pytest
 
-from unstructured.documents.elements import Image, ListItem, NarrativeText, Title
+from unstructured.documents.elements import (
+    ElementMetadata,
+    Image,
+    ListItem,
+    NarrativeText,
+    Title,
+)
 from unstructured.documents.email_elements import (
     MetaData,
     ReceivedInfo,
@@ -160,6 +166,21 @@ def test_partition_email_header():
     assert elements == RECEIVED_HEADER_OUTPUT
 
 
+def test_partition_email_has_metadata():
+    filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "fake-email-header.eml")
+    elements = partition_email(filename=filename)
+    assert len(elements) > 0
+    assert elements[0].metadata == ElementMetadata(
+        filename=filename,
+        date="2022-12-16T17:04:16-05:00",
+        page_number=None,
+        url=None,
+        sent_from=["Matthew Robinson <[email protected]>"],
+        sent_to=["Matthew Robinson <[email protected]>"],
+        subject="Test Email",
+    )
+
+
 def test_extract_email_text_matches_html():
     filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "fake-email-attachment.eml")
     elements_from_text = partition_email(filename=filename, content_source="text/plain")

test_unstructured/partition/test_msg.py

@@ -4,7 +4,12 @@
 import msg_parser
 import pytest
 
-from unstructured.documents.elements import ListItem, NarrativeText, Title
+from unstructured.documents.elements import (
+    ElementMetadata,
+    ListItem,
+    NarrativeText,
+    Title,
+)
 from unstructured.partition.msg import partition_msg
 
 DIRECTORY = pathlib.Path(__file__).parent.resolve()
@@ -22,6 +27,15 @@ def test_partition_msg_from_filename():
     filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "fake-email.msg")
     elements = partition_msg(filename=filename)
     assert elements == EXPECTED_MSG_OUTPUT
+    assert elements[0].metadata == ElementMetadata(
+        filename=filename,
+        date="2022-12-16T17:04:16-05:00",
+        page_number=None,
+        url=None,
+        sent_from=["Matthew Robinson <[email protected]>"],
+        sent_to=["Matthew Robinson (None)"],
+        subject="Test Email",
+    )
 
 
 class MockMsOxMessage:

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.9"  # pragma: no cover
+__version__ = "0.5.10-dev0"  # pragma: no cover

unstructured/documents/elements.py

@@ -14,9 +14,19 @@ class NoID(ABC):
 @dataclass
 class ElementMetadata:
     filename: Optional[str] = None
+    date: Optional[str] = None
+
+    # Page numbers currenlty supported for PDF, HTML and PPT documents
     page_number: Optional[int] = None
+
+    # Webpage specific metadata fields
     url: Optional[str] = None
 
+    # E-mail specific metadata fields
+    sent_from: Optional[List[str]] = None
+    sent_to: Optional[List[str]] = None
+    subject: Optional[str] = None
+
     def __post_init__(self):
         if isinstance(self.filename, pathlib.Path):
             self.filename = str(self.filename)

unstructured/partition/email.py

@@ -1,3 +1,4 @@
+import datetime
 import email
 import re
 import sys
@@ -87,6 +88,35 @@ def partition_email_header(msg: Message) -> List[Element]:
     return elements
 
 
+def build_email_metadata(msg: Message) -> ElementMetadata:
+    """Creates an ElementMetadata object from the header information in the email."""
+    header_dict = dict(msg.raw_items())
+    email_date = header_dict.get("Date")
+    if email_date is not None:
+        email_date = convert_to_iso_8601(email_date)
+
+    sent_from = header_dict.get("To")
+    if sent_from is not None:
+        sent_from = [sender.strip() for sender in sent_from.split(",")]
+
+    sent_to = header_dict.get("To")
+    if sent_to is not None:
+        sent_to = [recipient.strip() for recipient in sent_to.split(",")]
+
+    return ElementMetadata(
+        sent_to=sent_to,
+        sent_from=sent_from,
+        subject=header_dict.get("Subject"),
+        date=email_date,
+    )
+
+
+def convert_to_iso_8601(time: str) -> str:
+    """Converts the datetime from the email output to ISO-8601 format."""
+    datetime_object = datetime.datetime.strptime(time, "%a, %d %b %Y %H:%M:%S %z")
+    return datetime_object.isoformat()
+
+
 def extract_attachment_info(
     message: Message,
     output_dir: Optional[str] = None,
@@ -234,6 +264,8 @@ def partition_email(
         header = partition_email_header(msg)
     all_elements = header + elements
 
+    metadata = build_email_metadata(msg)
+    metadata.filename = filename
     for element in all_elements:
-        element.metadata = ElementMetadata(filename=filename)
+        element.metadata = metadata
     return all_elements

unstructured/partition/msg.py

@@ -3,8 +3,9 @@
 
 import msg_parser
 
-from unstructured.documents.elements import Element
+from unstructured.documents.elements import Element, ElementMetadata
 from unstructured.partition.common import exactly_one
+from unstructured.partition.email import convert_to_iso_8601
 from unstructured.partition.html import partition_html
 from unstructured.partition.text import partition_text
 
@@ -38,4 +39,31 @@ def partition_msg(
     else:
         elements = partition_text(text=text)
 
+    metadata = build_msg_metadata(msg_obj)
+    metadata.filename = filename
+    for element in elements:
+        element.metadata = metadata
+
     return elements
+
+
+def build_msg_metadata(msg_obj: msg_parser.MsOxMessage) -> ElementMetadata:
+    """Creates an ElementMetadata object from the header information in the emai."""
+    email_date = getattr(msg_obj, "sent_date", None)
+    if email_date is not None:
+        email_date = convert_to_iso_8601(email_date)
+
+    sent_from = getattr(msg_obj, "sender", None)
+    if sent_from is not None:
+        sent_from = [str(sender) for sender in sent_from]
+
+    sent_to = getattr(msg_obj, "recipients", None)
+    if sent_to is not None:
+        sent_to = [str(recipient) for recipient in sent_to]
+
+    return ElementMetadata(
+        sent_to=sent_to,
+        sent_from=sent_from,
+        subject=getattr(msg_obj, "subject", None),
+        date=email_date,
+    )