fix: validate encodings from email headers (#881)

* add validate encoding function

* remove extraneous file

* added test case for malformed encoding

* version and changelog

Author: MthwRobinson

CHANGELOG.md

@@ -10,6 +10,8 @@
 * Fix KeyError when `isd_to_elements` doesn't find a type
 * Fix _output_filename for local connector, allowing single files to be written correctly to the disk
 
+* Fix for cases where an invalid encoding is extracted from an email header.
+
 ### BREAKING CHANGES
 
 * Information about an element's location is no longer returned as top-level attributes of an element. Instead, it is returned in the `coordinates` attribute of the element's metadata.

example-docs/eml/fake-email-malformed-encoding.eml

@@ -0,0 +1,24 @@
+MIME-Version: 1.0
+Date: Fri, 16 Dec 2022 17:04:16 -0500
+Message-ID: <CADc-_xaLB2FeVQ7mNsoX+NJb_7hAJhBKa_zet-rtgPGenj0uVw@mail.gmail.com>
+Subject: Test Email
+From: Matthew Robinson <[email protected]>
+To: Matthew Robinson <[email protected]>
+Content-Type: multipart/alternative; boundary="00000000000095c9b205eff92630"
+
+--00000000000095c9b205eff92630
+Content-Type: text/plain; charset = "UTF-8"Content-Transfer-Encoding: 8bit
+
+This is a test email to use for unit tests.
+
+Important points:
+
+   - Roses are red
+   - Violets are blue
+
+--00000000000095c9b205eff92630
+Content-Type: text/html; charset = "UTF-8"Content-Transfer-Encoding: 8bit
+
+<div dir="ltr"><div>This is a test email to use for unit tests.</div><div><br></div><div>Important points:</div><div><ul><li>Roses are red</li><li>Violets are blue</li></ul></div></div>
+
+--00000000000095c9b205eff92630--

test_file.html

@@ -117,6 +117,13 @@ def test_partition_email_from_filename_with_metadata_filename():
     assert all(element.metadata.filename == "test" for element in elements)
 
 
+def test_partition_email_from_filename_malformed_encoding():
+    filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "fake-email-malformed-encoding.eml")
+    elements = partition_email(filename=filename)
+    assert len(elements) > 0
+    assert elements == EXPECTED_OUTPUT
+
+
 @pytest.mark.parametrize(
     ("filename", "expected_output"),
     [

test_unstructured/partition/test_email.py

@@ -49,6 +49,15 @@ def format_encoding_str(encoding: str) -> str:
     return formatted_encoding
 
 
+def validate_encoding(encoding: str) -> bool:
+    """Checks if an encoding string is valid. Helps to avoid errors in cases where
+    invalid encodings are extracted from malformed documents."""
+    for common_encoding in COMMON_ENCODINGS:
+        if format_encoding_str(common_encoding) == format_encoding_str(encoding):
+            return True
+    return False
+
+
 def detect_file_encoding(
     filename: str = "",
     file: Optional[Union[bytes, IO[bytes]]] = None,

unstructured/file_utils/encoding.py

@@ -12,6 +12,7 @@
     COMMON_ENCODINGS,
     format_encoding_str,
     read_txt_file,
+    validate_encoding,
 )
 from unstructured.partition.common import (
     convert_to_bytes,
@@ -208,7 +209,7 @@ def parse_email(
     encoding = None
     charsets = msg.get_charsets() or []
     for charset in charsets:
-        if charset and charset.strip():
+        if charset and charset.strip() and validate_encoding(charset):
             encoding = charset
             break