rfctr(file): make FileType enum a file-type descriptor (#3411)

**Summary**
Elaborate the `FileType` enum to be a complete descriptor of file-types.
Add methods to allow `STR_TO_FILETYPE`, `EXT_TO_FILETYPE` and
`FILETYPE_TO_MIMETYPE` mappings to be replaced, removing those redundant
and noisy declarations.

In the process, fix some lingering file-type identification and
`.metadata.filetype` errors that had been skipped in the tests.

**Additional Context**
Gathering the various attributes of a file-type into the `FileType` enum
eliminates the duplication inherent in the separate `STR_TO_FILETYPE`
etc. mappings and makes access to those values convenient for callers.
These attributes include what MIME-type a file-type should record in
metadata and what MIME-types and extensions map to that file-type. These
values and others are made available as methods and properties directly
on the `FileType` class and members. Because all attributes are defined
in the `FileType` enum there is no risk of inconsistency across multiple
locations and any changes happen in one and only one place. Further
attributes and methods will be added in later commits to support other
file-type related operations like mapping to a partitioner and verifying
its dependencies are installed.

Author: scanny

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.15.0-dev14
+## 0.15.0-dev15
 
 ### Enhancements
 

test_unstructured/file_utils/test_exploration.py

@@ -5,7 +5,7 @@
 import pytest
 
 from unstructured.file_utils import exploration
-from unstructured.file_utils.filetype import FileType
+from unstructured.file_utils.model import FileType
 
 DIRECTORY = pathlib.Path(__file__).parent.resolve()
 

test_unstructured/file_utils/test_filetype.py

@@ -22,13 +22,13 @@
 )
 from unstructured.file_utils import filetype
 from unstructured.file_utils.filetype import (
-    FileType,
     _detect_filetype_from_octet_stream,
     _is_code_mime_type,
     _is_text_file_a_csv,
     _is_text_file_a_json,
     detect_filetype,
 )
+from unstructured.file_utils.model import FileType
 
 is_in_docker = os.path.exists("/.dockerenv")
 

test_unstructured/file_utils/test_model.py

@@ -0,0 +1,70 @@
+"""Test suite for `unstructured.file_utils.filetype`."""
+
+from __future__ import annotations
+
+import pytest
+
+from unstructured.file_utils.model import FileType
+
+
+class DescribeFileType:
+    """Unit-test suite for `unstructured.file_utils.model.Filetype`."""
+
+    @pytest.mark.parametrize(
+        ("ext", "file_type"),
+        [
+            (".bmp", FileType.BMP),
+            (".html", FileType.HTML),
+            (".eml", FileType.EML),
+            (".p7s", FileType.EML),
+            (".java", FileType.TXT),
+        ],
+    )
+    def it_can_recognize_a_file_type_from_an_extension(self, ext: str, file_type: FileType | None):
+        assert FileType.from_extension(ext) is file_type
+
+    @pytest.mark.parametrize("ext", [".foobar", ".xyz", ".mdx", "", "."])
+    def but_not_when_that_extension_is_empty_or_not_registered(self, ext: str):
+        assert FileType.from_extension(ext) is None
+
+    @pytest.mark.parametrize(
+        ("mime_type", "file_type"),
+        [
+            ("image/bmp", FileType.BMP),
+            ("text/x-csv", FileType.CSV),
+            ("application/msword", FileType.DOC),
+            ("message/rfc822", FileType.EML),
+            ("text/plain", FileType.TXT),
+            ("text/yaml", FileType.TXT),
+            ("application/xml", FileType.XML),
+            ("text/xml", FileType.XML),
+            ("inode/x-empty", FileType.EMPTY),
+        ],
+    )
+    def it_can_recognize_a_file_type_from_a_mime_type(
+        self, mime_type: str, file_type: FileType | None
+    ):
+        assert FileType.from_mime_type(mime_type) is file_type
+
+    @pytest.mark.parametrize("mime_type", ["text/css", "image/gif", "audio/mpeg", "foo/bar"])
+    def but_not_when_that_mime_type_is_not_registered_by_a_file_type(self, mime_type: str):
+        assert FileType.from_mime_type(mime_type) is None
+
+    @pytest.mark.parametrize(
+        ("file_type", "mime_type"),
+        [
+            (FileType.BMP, "image/bmp"),
+            (FileType.CSV, "text/csv"),
+            (FileType.DOC, "application/msword"),
+            (FileType.EML, "message/rfc822"),
+            (FileType.HTML, "text/html"),
+            (FileType.JPG, "image/jpeg"),
+            (FileType.PDF, "application/pdf"),
+            (FileType.TXT, "text/plain"),
+            (FileType.XML, "application/xml"),
+            (FileType.EMPTY, "inode/x-empty"),
+            (FileType.UNK, "application/octet-stream"),
+        ],
+    )
+    def it_knows_its_canonical_MIME_type(self, file_type: FileType, mime_type: str):
+        assert file_type.mime_type == mime_type

test_unstructured/partition/test_auto.py

@@ -45,7 +45,7 @@
     Text,
     Title,
 )
-from unstructured.file_utils.filetype import FILETYPE_TO_MIMETYPE, FileType
+from unstructured.file_utils.model import FileType
 from unstructured.partition import auto
 from unstructured.partition.auto import IMAGE_FILETYPES, _get_partition_with_extras, partition
 from unstructured.partition.utils.constants import PartitionStrategy
@@ -1245,7 +1245,7 @@ def test_auto_partition_applies_the_correct_filetype_for_all_filetypes(filetype:
 
     assert elements
     assert all(
-        e.metadata.filetype == FILETYPE_TO_MIMETYPE[filetype]
+        e.metadata.filetype == filetype.mime_type
         for e in elements
         if e.metadata.filetype is not None
     )

test_unstructured/partition/test_json.py

@@ -10,7 +10,8 @@
 from pytest_mock import MockFixture
 
 from unstructured.documents.elements import CompositeElement
-from unstructured.file_utils.filetype import FileType, detect_filetype
+from unstructured.file_utils.filetype import detect_filetype
+from unstructured.file_utils.model import FileType
 from unstructured.partition.email import partition_email
 from unstructured.partition.html import partition_html
 from unstructured.partition.json import partition_json

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.15.0-dev14"  # pragma: no cover
+__version__ = "0.15.0-dev15"  # pragma: no cover

unstructured/file_utils/filetype.py

@@ -12,13 +12,7 @@
 
 from unstructured.documents.elements import Element
 from unstructured.file_utils.encoding import detect_file_encoding, format_encoding_str
-from unstructured.file_utils.model import (
-    EXT_TO_FILETYPE,
-    FILETYPE_TO_MIMETYPE,
-    PLAIN_TEXT_EXTENSIONS,
-    STR_TO_FILETYPE,
-    FileType,
-)
+from unstructured.file_utils.model import PLAIN_TEXT_EXTENSIONS, FileType
 from unstructured.logger import logger
 from unstructured.nlp.patterns import EMAIL_HEAD_RE, LIST_OF_DICTS_PATTERN
 from unstructured.partition.common import (
@@ -49,9 +43,9 @@ def detect_filetype(
 
     # first check (content_type)
     if content_type:
-        filetype = STR_TO_FILETYPE.get(content_type)
-        if filetype:
-            return filetype
+        file_type = FileType.from_mime_type(content_type)
+        if file_type:
+            return file_type
 
     # second check (filename/file_name/file)
     # continue if successfully define mime_type
@@ -68,7 +62,7 @@ def detect_filetype(
 
             mime_type = ft.guess_mime(_filename)
         if mime_type is None:
-            return EXT_TO_FILETYPE.get(extension, FileType.UNK)
+            return FileType.from_extension(extension) or FileType.UNK
 
     elif file is not None:
         if hasattr(file, "name"):
@@ -92,7 +86,7 @@ def detect_filetype(
                 "libmagic is unavailable but assists in filetype detection on file-like objects. "
                 "Please consider installing libmagic for better results.",
             )
-            return EXT_TO_FILETYPE.get(extension, FileType.UNK)
+            return FileType.from_extension(extension) or FileType.UNK
 
     else:
         raise ValueError("No filename, file, nor file_filename were specified.")
@@ -128,7 +122,7 @@ def detect_filetype(
             ".tsv",
             ".json",
         ]:
-            return EXT_TO_FILETYPE.get(extension)
+            return FileType.from_extension(extension)
 
         # NOTE(crag): for older versions of the OS libmagic package, such as is currently
         # installed on the Unstructured docker image, .json files resolve to "text/plain"
@@ -151,11 +145,11 @@ def detect_filetype(
             return FileType.EML
 
         if extension in PLAIN_TEXT_EXTENSIONS:
-            return EXT_TO_FILETYPE.get(extension, FileType.UNK)
+            return FileType.from_extension(extension) or FileType.UNK
 
         # Safety catch
-        if mime_type in STR_TO_FILETYPE:
-            return STR_TO_FILETYPE[mime_type]
+        if file_type := FileType.from_mime_type(mime_type):
+            return file_type
 
         return FileType.TXT
 
@@ -165,21 +159,22 @@ def detect_filetype(
         elif file:
             return _detect_filetype_from_octet_stream(file=file)
         else:
-            return EXT_TO_FILETYPE.get(extension, FileType.UNK)
+            return FileType.from_extension(extension) or FileType.UNK
 
     elif mime_type == "application/zip":
-        filetype = FileType.UNK
+        file_type = FileType.UNK
         if file:
-            filetype = _detect_filetype_from_octet_stream(file=file)
+            file_type = _detect_filetype_from_octet_stream(file=file)
         elif filename is not None:
             with open(filename, "rb") as f:
-                filetype = _detect_filetype_from_octet_stream(file=f)
+                file_type = _detect_filetype_from_octet_stream(file=f)
 
         extension = extension if extension else ""
-        if filetype == FileType.UNK:
-            return FileType.ZIP
-        else:
-            return EXT_TO_FILETYPE.get(extension, filetype)
+        return (
+            FileType.ZIP
+            if file_type in (FileType.UNK, FileType.ZIP)
+            else FileType.from_extension(extension) or file_type
+        )
 
     elif _is_code_mime_type(mime_type):
         # NOTE(robinson) - we'll treat all code files as plain text for now.
@@ -191,14 +186,14 @@ def detect_filetype(
         return FileType.EMPTY
 
     # For everything else
-    elif mime_type in STR_TO_FILETYPE:
-        return STR_TO_FILETYPE[mime_type]
+    elif file_type := FileType.from_mime_type(mime_type):
+        return file_type
 
     logger.warning(
         f"The MIME type{f' of {filename!r}' if filename else ''} is {mime_type!r}. "
         "This file type is not currently supported in unstructured.",
     )
-    return EXT_TO_FILETYPE.get(extension, FileType.UNK)
+    return FileType.from_extension(extension) or FileType.UNK
 
 
 def is_json_processable(
@@ -260,7 +255,7 @@ def _detect_filetype_from_octet_stream(file: IO[bytes]) -> FileType:
 
         # Infer mime type using magic if octet-stream is not zip file
         mime_type = magic.from_buffer(file.read(4096), mime=True)
-        return STR_TO_FILETYPE.get(mime_type, FileType.UNK)
+        return FileType.from_mime_type(mime_type) or FileType.UNK
     logger.warning(
         "Could not detect the filetype from application/octet-stream MIME type.",
     )
@@ -439,7 +434,7 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> List[Element]:
                     # NOTE(robinson) - Attached files have already run through this logic
                     # in their own partitioning function
                     if element.metadata.attached_to_filename is None:
-                        add_element_metadata(element, filetype=FILETYPE_TO_MIMETYPE[filetype])
+                        add_element_metadata(element, filetype=filetype.mime_type)
 
                 return elements
             else: