refactor: Introduce 'exactly_one' to simplify partitioning functions (#343)

Author: tomaarsen

CHANGELOG.md

@@ -1,7 +1,8 @@
-## 0.5.3-dev3
+## 0.5.3-dev4
 
 ### Enhancements
 
+* Simplify partitioning functions.
 * Improve logging for ingest CLI.
 
 ### Features

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.3-dev3"  # pragma: no cover
+__version__ = "0.5.3-dev4"  # pragma: no cover

unstructured/partition/common.py

@@ -52,12 +52,17 @@ def layout_list_to_list_items(text: str, coordinates: List[float]) -> List[Eleme
     list_items: List[Element] = []
     for text_segment in split_items:
         if len(text_segment.strip()) > 0:
-            list_items.append(ListItem(text=text_segment.strip(), coordinates=coordinates))
+            list_items.append(
+                ListItem(text=text_segment.strip(), coordinates=coordinates),
+            )
 
     return list_items
 
 
-def document_to_element_list(document, include_page_breaks: bool = False) -> List[Element]:
+def document_to_element_list(
+    document,
+    include_page_breaks: bool = False,
+) -> List[Element]:
     """Converts a DocumentLayout object to a list of unstructured elements."""
     elements: List[Element] = []
     num_pages = len(document.pages)
@@ -124,3 +129,19 @@ def convert_office_doc(input_filename: str, output_directory: str, target_format
 - Mac: https://formulae.brew.sh/cask/libreoffice
 - Debian: https://wiki.debian.org/LibreOffice""",
         )
+
+
+def exactly_one(**kwargs) -> None:
+    """
+    Verify arguments; exactly one of all keyword arguments must not be None.
+
+    Example:
+        >>> exactly_one(filename=filename, file=file, text=text, url=url)
+    """
+    if sum([(arg is not None) for arg in kwargs.values()]) != 1:
+        names = list(kwargs.keys())
+        if len(names) > 1:
+            message = f"Exactly one of {', '.join(names[:-1])} and {names[-1]} must be specified."
+        else:
+            message = f"{names[0]} must be specified."
+        raise ValueError(message)

unstructured/partition/doc.py

@@ -3,7 +3,7 @@
 from typing import IO, List, Optional
 
 from unstructured.documents.elements import Element
-from unstructured.partition.common import convert_office_doc
+from unstructured.partition.common import convert_office_doc, exactly_one
 from unstructured.partition.docx import partition_docx
 
 
@@ -12,25 +12,25 @@ def partition_doc(filename: Optional[str] = None, file: Optional[IO] = None) ->
 
     Parameters
     ----------
-     filename
+    filename
         A string defining the target filename path.
     file
         A file-like object using "rb" mode --> open(filename, "rb").
     """
-    if not any([filename, file]):
-        raise ValueError("One of filename or file must be specified.")
+    # Verify that only one of the arguments was provided
+    exactly_one(filename=filename, file=file)
 
-    if filename is not None and not file:
+    if filename is not None:
         _, filename_no_path = os.path.split(os.path.abspath(filename))
         base_filename, _ = os.path.splitext(filename_no_path)
-    elif file is not None and not filename:
+    elif file is not None:
         tmp = tempfile.NamedTemporaryFile(delete=False)
         tmp.write(file.read())
         tmp.close()
         filename = tmp.name
         _, filename_no_path = os.path.split(os.path.abspath(tmp.name))
     else:
-        raise ValueError("Only one of filename or file can be specified.")
+        raise ValueError("Exactly one of filename and file must be specified.")
 
     if not os.path.exists(filename):
         raise ValueError(f"The file {filename} does not exist.")

unstructured/partition/docx.py

@@ -12,6 +12,7 @@
     Text,
     Title,
 )
+from unstructured.partition.common import exactly_one
 from unstructured.partition.text_type import (
     is_bulleted_text,
     is_possible_narrative_text,
@@ -75,15 +76,13 @@ def partition_docx(
         metadata.
     """
 
-    if not any([filename, file]):
-        raise ValueError("One of filename or file must be specified.")
+    # Verify that only one of the arguments was provided
+    exactly_one(filename=filename, file=file)
 
-    if filename is not None and not file:
+    if filename is not None:
         document = docx.Document(filename)
-    elif file is not None and not filename:
+    elif file is not None:
         document = docx.Document(file)
-    else:
-        raise ValueError("Only one of filename or file can be specified.")
 
     metadata_filename = metadata_filename or filename
     elements: List[Element] = []

unstructured/partition/email.py

@@ -4,6 +4,8 @@
 from email.message import Message
 from typing import IO, Dict, List, Optional, Tuple, Union
 
+from unstructured.partition.common import exactly_one
+
 if sys.version_info < (3, 8):
     from typing_extensions import Final
 else:
@@ -166,14 +168,14 @@ def partition_email(
             f"Valid content sources are: {VALID_CONTENT_SOURCES}",
         )
 
-    if not any([filename, file, text]):
-        raise ValueError("One of filename, file, or text must be specified.")
+    # Verify that only one of the arguments was provided
+    exactly_one(filename=filename, file=file, text=text)
 
-    if filename is not None and not file and not text:
+    if filename is not None:
         with open(filename) as f:
             msg = email.message_from_file(f)
 
-    elif file is not None and not filename and not text:
+    elif file is not None:
         file_content = file.read()
         if isinstance(file_content, bytes):
             file_text = file_content.decode(encoding)
@@ -182,13 +184,10 @@ def partition_email(
 
         msg = email.message_from_string(file_text)
 
-    elif text is not None and not filename and not file:
+    elif text is not None:
         _text: str = str(text)
         msg = email.message_from_string(_text)
 
-    else:
-        raise ValueError("Only one of filename, file, or text can be specified.")
-
     content_map: Dict[str, str] = {}
     for part in msg.walk():
         # NOTE(robinson) - content dispostiion is None for the content of the email itself.

unstructured/partition/html.py

@@ -5,7 +5,11 @@
 from unstructured.documents.elements import Element
 from unstructured.documents.html import HTMLDocument
 from unstructured.documents.xml import VALID_PARSERS
-from unstructured.partition.common import add_element_metadata, document_to_element_list
+from unstructured.partition.common import (
+    add_element_metadata,
+    document_to_element_list,
+    exactly_one,
+)
 
 
 def partition_html(
@@ -40,16 +44,16 @@ def partition_html(
     parser
         The parser to use for parsing the HTML document. If None, default parser will be used.
     """
-    if not any([filename, file, text, url]):
-        raise ValueError("One of filename, file, or text must be specified.")
+    # Verify that only one of the arguments was provided
+    exactly_one(filename=filename, file=file, text=text, url=url)
 
     if not encoding:
         encoding = "utf-8"
 
-    if filename is not None and not file and not text and not url:
+    if filename is not None:
         document = HTMLDocument.from_file(filename, parser=parser, encoding=encoding)
 
-    elif file is not None and not filename and not text and not url:
+    elif file is not None:
         file_content = file.read()
         if isinstance(file_content, bytes):
             file_text = file_content.decode(encoding)
@@ -58,11 +62,11 @@ def partition_html(
 
         document = HTMLDocument.from_string(file_text, parser=parser)
 
-    elif text is not None and not filename and not file and not url:
+    elif text is not None:
         _text: str = str(text)
         document = HTMLDocument.from_string(_text, parser=parser)
 
-    elif url is not None and not filename and not file and not text:
+    elif url is not None:
         response = requests.get(url)
         if not response.ok:
             raise ValueError(f"URL return an error: {response.status_code}")
@@ -73,9 +77,6 @@ def partition_html(
 
         document = HTMLDocument.from_string(response.text, parser=parser)
 
-    else:
-        raise ValueError("Only one of filename, file, or text can be specified.")
-
     layout_elements = document_to_element_list(document, include_page_breaks=include_page_breaks)
     if include_metadata:
         return add_element_metadata(

unstructured/partition/md.py

@@ -5,6 +5,7 @@
 
 from unstructured.documents.elements import Element
 from unstructured.documents.xml import VALID_PARSERS
+from unstructured.partition.common import exactly_one
 from unstructured.partition.html import partition_html
 
 
@@ -23,20 +24,17 @@ def partition_md(
     include_metadata: bool = True,
     parser: VALID_PARSERS = None,
 ) -> List[Element]:
-    if not any([filename, file, text, url]):
-        raise ValueError("One of filename, file, or text must be specified.")
+    # Verify that only one of the arguments was provided
+    exactly_one(filename=filename, file=file, text=text, url=url)
 
-    if filename is not None and not file and not text and not url:
+    if filename is not None:
         with open(filename, encoding="utf8") as f:
             text = optional_decode(f.read())
 
-    elif file is not None and not filename and not text and not url:
+    elif file is not None:
         text = optional_decode(file.read())
 
-    elif text is not None and not filename and not file and not url:
-        pass
-
-    elif url is not None and not filename and not file and not text:
+    elif url is not None:
         response = requests.get(url)
         if not response.ok:
             raise ValueError(f"URL return an error: {response.status_code}")
@@ -47,8 +45,8 @@ def partition_md(
 
         text = response.text
 
-    else:
-        raise ValueError("Only one of filename, file, or text can be specified.")
+    elif text is None:
+        raise ValueError("Exactly one of filename, file, url or text must be specified.")
 
     html = markdown.markdown(text)
 

unstructured/partition/ppt.py

@@ -3,7 +3,7 @@
 from typing import IO, List, Optional
 
 from unstructured.documents.elements import Element
-from unstructured.partition.common import convert_office_doc
+from unstructured.partition.common import convert_office_doc, exactly_one
 from unstructured.partition.pptx import partition_pptx
 
 
@@ -23,20 +23,20 @@ def partition_ppt(
     include_page_breaks
         If True, includes a PageBreak element between slides
     """
-    if not any([filename, file]):
-        raise ValueError("One of filename or file must be specified.")
+    # Verify that only one of the arguments was provided
+    exactly_one(filename=filename, file=file)
 
-    if filename is not None and not file:
+    if filename is not None:
         _, filename_no_path = os.path.split(os.path.abspath(filename))
         base_filename, _ = os.path.splitext(filename_no_path)
-    elif file is not None and not filename:
+    elif file is not None:
         tmp = tempfile.NamedTemporaryFile(delete=False)
         tmp.write(file.read())
         tmp.close()
         filename = tmp.name
         _, filename_no_path = os.path.split(os.path.abspath(tmp.name))
     else:
-        raise ValueError("Only one of filename or file can be specified.")
+        raise ValueError("Exactly one of filename and file must be specified.")
 
     if not os.path.exists(filename):
         raise ValueError(f"The file {filename} does not exist.")

unstructured/partition/pptx.py

@@ -11,6 +11,7 @@
     Text,
     Title,
 )
+from unstructured.partition.common import exactly_one
 from unstructured.partition.text_type import (
     is_possible_narrative_text,
     is_possible_title,
@@ -41,15 +42,13 @@ def partition_pptx(
         metadata.
     """
 
-    if not any([filename, file]):
-        raise ValueError("One of filename or file must be specified.")
+    # Verify that only one of the arguments was provided
+    exactly_one(filename=filename, file=file)
 
-    if filename is not None and not file:
+    if filename is not None:
         presentation = pptx.Presentation(filename)
-    elif file is not None and not filename:
+    elif file is not None:
         presentation = pptx.Presentation(file)
-    else:
-        raise ValueError("Only one of filename or file can be specified.")
 
     elements: List[Element] = []
     metadata_filename = metadata_filename or filename