enhancement: add encoding to elements_to_json and elements_from_json (#694)

* add encoding to elements_to_json and elements_from_json

* version and changelog

* add new test

* fix version

* revert test file

* blank line to test

* no blank line

Author: MthwRobinson

CHANGELOG.md

@@ -1,7 +1,8 @@
-## 0.7.2-dev6
+## 0.7.2
 
 ### Enhancements
 
+* Adds an optional encoding kwarg to `elements_to_json` and `elements_from_json`
 * Bump version of base image to use new stable version of tesseract
 
 ### Features

test_unstructured/staging/test_base_staging.py

@@ -19,6 +19,7 @@
     Text,
     Title,
 )
+from unstructured.partition.text import partition_text
 from unstructured.staging import base
 
 
@@ -134,3 +135,10 @@ def test_serialized_deserialize_elements_to_json(tmpdir):
     elements_str = base.elements_to_json(elements)
     new_elements_text = base.elements_from_json(text=elements_str)
     assert elements == new_elements_text
+
+
+def test_read_and_write_json_with_encoding(filename="example-docs/fake-text-utf-16-be.txt"):
+    elements = partition_text(filename=filename)
+    base.elements_to_json(elements, filename=filename, encoding="utf-16")
+    new_elements_filename = base.elements_from_json(filename=filename, encoding="utf-16")
+    assert elements == new_elements_filename

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.7.2-dev6"  # pragma: no cover
+__version__ = "0.7.2"  # pragma: no cover

unstructured/staging/base.py

@@ -47,14 +47,15 @@ def elements_to_json(
     elements: List[Element],
     filename: Optional[str] = None,
     indent: int = 4,
+    encoding: str = "utf-8",
 ) -> Optional[str]:
     """
     Saves a list of elements to a JSON file if filename is specified.
     Otherwise, return the list of elements as a string.
     """
     element_dict = convert_to_dict(elements)
     if filename is not None:
-        with open(filename, "w") as f:
+        with open(filename, "w", encoding=encoding) as f:
             json.dump(element_dict, f, indent=indent)
             return None
     else:
@@ -105,12 +106,16 @@ def dict_to_elements(element_dict: List[Dict[str, Any]]) -> List[Element]:
     return isd_to_elements(element_dict)
 
 
-def elements_from_json(filename: str = "", text: str = "") -> List[Element]:
+def elements_from_json(
+    filename: str = "",
+    text: str = "",
+    encoding: str = "utf-8",
+) -> List[Element]:
     """Loads a list of elements from a JSON file or a string."""
     exactly_one(filename=filename, text=text)
 
     if filename:
-        with open(filename) as f:
+        with open(filename, encoding=encoding) as f:
             element_dict = json.load(f)
         return dict_to_elements(element_dict)
     else: