bugfix/fix ndjson detection (#3905)

### Description
NDJSON files were being detected as JSON due to having the same
mime-type. This adds additional logic to skip mime-type based detection
if extension is `.ndjson`

Author: rbiseck3

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.16.21-dev2
+## 0.16.21-dev3
 
 ### Enhancements
 
@@ -8,6 +8,8 @@
 
 ### Fixes
 
+- **Fix file type detection for NDJSON files** NDJSON files were being detected as JSON due to having the same mime-type.
+
 ## 0.16.20
 
 ### Enhancements

test_unstructured/file_utils/test_filetype.py

@@ -77,7 +77,6 @@ def test_it_detects_correct_file_type_for_CFB_and_ZIP_subtypes_detected_by_direc
         (FileType.HEIC, "img/DA-1p.heic", "image/heic"),
         (FileType.HTML, "example-10k-1p.html", "text/html"),
         (FileType.JPG, "img/example.jpg", "image/jpeg"),
-        (FileType.JSON, "spring-weather.html.json", "application/json"),
         (FileType.MD, "README.md", "text/markdown"),
         (FileType.ORG, "README.org", "text/org"),
         (FileType.PDF, "pdf/layout-parser-paper-fast.pdf", "application/pdf"),
@@ -116,7 +115,6 @@ def test_it_detects_correct_file_type_from_file_path_with_correct_asserted_conte
         (FileType.HEIC, "img/DA-1p.heic", "image/heic"),
         (FileType.HTML, "example-10k-1p.html", "text/html"),
         (FileType.JPG, "img/example.jpg", "image/jpeg"),
-        (FileType.JSON, "spring-weather.html.json", "application/json"),
         (FileType.MD, "README.md", "text/markdown"),
         (FileType.ORG, "README.org", "text/org"),
         (FileType.PDF, "pdf/layout-parser-paper-fast.pdf", "application/pdf"),
@@ -154,10 +152,10 @@ def test_it_identifies_NDJSON_for_file_like_object_with_no_name_but_NDJSON_conte
     assert detect_filetype(file=file, content_type=FileType.NDJSON.mime_type) == FileType.NDJSON
 
 
-# TODO: ideally this test should pass, currently fails
-# def test_it_identifies_NDJSON_for_file_with_ndjson_extension_but_JSON_content_type():
-#     file_path = example_doc_path("simple.ndjson")
-#     assert detect_filetype(file_path, content_type=FileType.JSON.mime_type) == FileType.NDJSON
+def test_it_identifies_NDJSON_for_file_with_ndjson_extension_but_JSON_content_type():
+    file_path = example_doc_path("simple.ndjson")
+    assert detect_filetype(file_path, content_type=FileType.JSON.mime_type) == FileType.NDJSON
+
 
 # ================================================================================================
 # STRATEGY #3 - GUESS MIME-TYPE WITH LIBMAGIC/FILETYPE LIBRARY
@@ -268,7 +266,6 @@ def test_it_detects_most_file_types_using_mime_guessing_when_libmagic_guesses_mi
         (FileType.UNK, "stanley-cups.csv"),
         (FileType.UNK, "eml/fake-email.eml"),
         (FileType.UNK, "example-10k-1p.html"),
-        (FileType.UNK, "spring-weather.html.json"),
         (FileType.UNK, "README.md"),
         (FileType.UNK, "README.org"),
         (FileType.UNK, "README.rst"),
@@ -333,6 +330,7 @@ def test_detect_filetype_from_file_warns_when_libmagic_is_not_installed(
         (FileType.TXT, "norwich-city.txt"),
         (FileType.WAV, "CantinaBand3.wav"),
         (FileType.XML, "factbook.xml"),
+        (FileType.NDJSON, "simple.ndjson"),
     ],
 )
 def test_it_detects_correct_file_type_from_extension_when_that_maps_to_a_file_type(
@@ -395,6 +393,27 @@ def test_it_detects_HTML_from_guessed_mime_type_ending_with_xml_and_html_extensi
     assert file_type is FileType.HTML
 
 
+@pytest.mark.parametrize(
+    ("expected_value", "file_name"),
+    [(FileType.NDJSON, "simple.ndjson"), (FileType.JSON, "spring-weather.html.json")],
+)
+def test_it_detects_correct_json_type_without_extension(expected_value: FileType, file_name: str):
+    with open(example_doc_path(file_name), "rb") as f:
+        file = io.BytesIO(f.read())
+
+    filetype = detect_filetype(file=file)
+    assert filetype == expected_value
+
+
+@pytest.mark.parametrize(
+    ("expected_value", "file_name"),
+    [(FileType.NDJSON, "simple.ndjson"), (FileType.JSON, "spring-weather.html.json")],
+)
+def test_it_detects_correct_json_type_with_extension(expected_value: FileType, file_name: str):
+    filetype = detect_filetype(file_path=example_doc_path(file_name))
+    assert filetype == expected_value
+
+
 @pytest.mark.parametrize(
     ("mime_type", "file_name"),
     [

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.16.21-dev2"  # pragma: no cover
+__version__ = "0.16.21-dev3"  # pragma: no cover

unstructured/file_utils/filetype.py

@@ -46,7 +46,7 @@
 from unstructured.file_utils.encoding import detect_file_encoding, format_encoding_str
 from unstructured.file_utils.model import FileType
 from unstructured.logger import logger
-from unstructured.nlp.patterns import DICT_PATTERN, EMAIL_HEAD_RE, LIST_OF_DICTS_PATTERN
+from unstructured.nlp.patterns import EMAIL_HEAD_RE, LIST_OF_DICTS_PATTERN
 from unstructured.partition.common.common import add_element_metadata, exactly_one
 from unstructured.partition.common.metadata import set_element_hierarchy
 from unstructured.utils import get_call_args_applying_defaults, lazyproperty
@@ -140,8 +140,7 @@ def is_ndjson_processable(
         file_text = _FileTypeDetectionContext.new(
             file_path=filename, file=file, encoding=encoding
         ).text_head
-
-    return re.match(DICT_PATTERN, file_text) is not None
+    return file_text.lstrip().startswith("{")
 
 
 class _FileTypeDetector:
@@ -179,7 +178,11 @@ def _file_type(self) -> FileType:
         if file_type := self._file_type_from_file_extension:
             return file_type
 
-        # -- strategy 5: give up and report FileType.UNK --
+        # -- strategy 5: edge case where JSON/NDJSON content without file extension --
+        if file_type := self._disambiguate_json_file_type:
+            return file_type
+
+        # -- strategy 6: give up and report FileType.UNK --
         return FileType.UNK
 
     # == STRATEGIES ============================================================
@@ -210,6 +213,20 @@ def _file_type_from_content_type(self) -> FileType | None:
         # -- otherwise we trust the passed `content_type` as long as `FileType` recognizes it --
         return FileType.from_mime_type(self._ctx.content_type)
 
+    @property
+    def _disambiguate_json_file_type(self) -> FileType | None:
+        """Disambiguate JSON/NDJSON file-type based on file contents.
+
+        This method is used when the content-type is `application/json` and the file is not empty.
+        """
+        if self._ctx.content_type is not None and self._ctx.content_type != "application/json":
+            return None
+        if is_json_processable(file_text=self._ctx.text_head):
+            return FileType.JSON
+        if is_ndjson_processable(file_text=self._ctx.text_head):
+            return FileType.NDJSON
+        return None
+
     @property
     def _file_type_from_guessed_mime_type(self) -> FileType | None:
         """FileType based on auto-detection of MIME-type by libmagic.
@@ -240,6 +257,9 @@ def _file_type_from_guessed_mime_type(self) -> FileType | None:
         if mime_type.endswith("empty"):
             return FileType.EMPTY
 
+        if mime_type.endswith("json") and self._ctx.extension == ".ndjson":
+            return FileType.NDJSON
+
         # -- if no more-specific rules apply, use the MIME-type -> FileType mapping when present --
         file_type = FileType.from_mime_type(mime_type)
         return file_type if file_type != FileType.UNK else None

unstructured/file_utils/model.py

@@ -82,7 +82,8 @@ def from_mime_type(cls, mime_type: str | None) -> FileType | None:
         Returns `None` when `mime_type` is `None` or does not map to the canonical MIME-type of a
         `FileType` member or one of its alias MIME-types.
         """
-        if mime_type is None:
+        if mime_type is None or mime_type == "application/json":
+            # application/json is ambiguous as it may point ot JSON and NDJSON file types
             return None
         # -- not super efficient but plenty fast enough for once-or-twice-per-file use and avoids
         # -- limitations on defining a class variable on an Enum.

unstructured/nlp/patterns.py

@@ -120,7 +120,6 @@
 # format for document elements
 LIST_OF_DICTS_PATTERN = r"\A\s*\[\s*{?"
 
-DICT_PATTERN = r"\A\s*{?"
 
 # (?s) dot all (including newline characters)
 # \{(?=.*:) opening brace and at least one colon
@@ -133,6 +132,7 @@
 # or the closing bracket to handle cases where the JSON array is cut off
 JSON_PATTERN = r"(?s)\{(?=.*:).*?(?:\}|$)|\[(?s:.*?)\](?:$|,|\])"
 
+
 # taken from https://stackoverflow.com/a/3845829/12406158
 VALID_JSON_CHARACTERS = r"[,:{}\[\]0-9.\-+Eaeflnr-u \n\r\t]"