chore: Add --ocr-languages parameter to unstructured ingest (#793)

Author: six5532one

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Enhancements
 
+* Adds --partition-ocr-languages to unstructured-ingest
+
+
 ### Features
 
 * Adds `partition_org` for processed Org Mode documents.

test_unstructured_ingest/expected-structured-output/parameterized-ingest-output/example-docs/english-and-korean.png.json

@@ -0,0 +1,56 @@
+[
+  {
+    "type": "Title",
+    "element_id": "a726edcce6a7b116c43e1b5109c2f4c8",
+    "metadata": {
+      "data_source": {},
+      "filetype": "image/png"
+    },
+    "text": "RULES AND INSTRUCTIONS"
+  },
+  {
+    "type": "NarrativeText",
+    "element_id": "ba41648485acf4a8e7dd7d183b764811",
+    "metadata": {
+      "data_source": {},
+      "filetype": "image/png"
+    },
+    "text": "1. Template for day 1 (korean) , for day 2 (English) for day 3 both English and korean. 2. Use all your accounts. use different emails to send. Its better to have many email accounts."
+  },
+  {
+    "type": "NarrativeText",
+    "element_id": "30089b5886f3074973195439e5343f64",
+    "metadata": {
+      "data_source": {},
+      "filetype": "image/png"
+    },
+    "text": "Note: Remember to write your own \"OPENING MESSAGE\" before you copy and paste the template. please always include [TREASURE HARUTO] for example:"
+  },
+  {
+    "type": "NarrativeText",
+    "element_id": "577b0ea5cd4f62a19b9d14dd0bd7272e",
+    "metadata": {
+      "data_source": {},
+      "filetype": "image/png"
+    },
+    "text": "안녕하세요, 저 희 는 YGEAS 그룹 TREASUREWH HARUTOM|2] 팬 입니다. 팬 으 로서, HARUTO 씨 받 는 대 우 에 대해 의 구 심 과 불 공 평 함 을 LRU, 이 메 일 을 통해 저 희 의 의 혹 을 전 달 하여 귀 사 의 진지한 고 민 과 적극적인 답 변 을 받을 수 있 기 를 바랍니다."
+  },
+  {
+    "type": "NarrativeText",
+    "element_id": "95c5b9d0d081bc45be0e50a109924191",
+    "metadata": {
+      "data_source": {},
+      "filetype": "image/png"
+    },
+    "text": "3. CC [email protected] so we can keep track of how many emails were successfully sent"
+  },
+  {
+    "type": "NarrativeText",
+    "element_id": "da91cb5310386aafd5aaf6fd988b9616",
+    "metadata": {
+      "data_source": {},
+      "filetype": "image/png"
+    },
+    "text": "4. Use the hashtag of Haruto on your tweet to show that vou have sent vour email]"
+  }
+]

test_unstructured_ingest/test-ingest-local-single-file.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+cd "$SCRIPT_DIR"/.. || exit 1
+
+PYTHONPATH=. ./unstructured/ingest/main.py \
+    --metadata-exclude filename,file_directory,metadata.data_source.date_processed \
+    --local-input-path example-docs/english-and-korean.png \
+    --structured-output-dir parameterized-ingest-output \
+    --partition-ocr-languages eng+kor \
+    --verbose \
+    --reprocess
+
+set +e
+
+# to update ingest test fixtures, run scripts/ingest-test-fixtures-update.sh on x86_64
+if [[ "$OVERWRITE_FIXTURES" != "false" ]]; then
+
+    cp -r parameterized-ingest-output/ test_unstructured_ingest/expected-structured-output/
+
+elif ! diff -ru test_unstructured_ingest/expected-structured-output/parameterized-ingest-output parameterized-ingest-output ; then
+    echo
+    echo "There are differences from the previously checked-in structured outputs."
+    echo
+    echo "If these differences are acceptable, overwrite by the fixtures by setting the env var:"
+    echo
+    echo "  export OVERWRITE_FIXTURES=true"
+    echo
+    echo "and then rerun this script."
+    echo
+    echo "NOTE: You'll likely just want to run scripts/ingest-test-fixtures-update.sh on x86_64 hardware"
+    echo "to update fixtures for CI,"
+    echo
+    exit 1
+
+fi

test_unstructured_ingest/test-ingest.sh

@@ -21,5 +21,6 @@ export OMP_THREAD_LIMIT=1
 ./test_unstructured_ingest/test-ingest-slack.sh
 ./test_unstructured_ingest/test-ingest-against-api.sh
 ./test_unstructured_ingest/test-ingest-gcs.sh
+./test_unstructured_ingest/test-ingest-local-single-file.sh
 # NOTE(yuming): The following test should be put after any tests with --preserve-downloads option
 ./test_unstructured_ingest/test-ingest-pdf-fast-reprocess.sh

unstructured/ingest/doc_processor/generalized.py

@@ -15,7 +15,7 @@ def initialize():
 
 
 def process_document(doc: "IngestDoc", **partition_kwargs) -> Optional[List[Dict[str, Any]]]:
-    """Process any IngestDoc-like class of document with choosen Unstructured's partition logic.
+    """Process any IngestDoc-like class of document with chosen Unstructured's partition logic.
 
     Parameters
     ----------

unstructured/ingest/main.py

@@ -151,6 +151,14 @@ def run(self):
     help="The method that will be used to process the documents. "
     "Default: auto. Other strategies include `fast` and `hi_res`.",
 )
+@click.option(
+    "--partition-ocr-languages",
+    default="eng",
+    help="A list of language packs to specify which languages to use for OCR, separated by '+' "
+    "e.g. 'eng+deu' to use the English and German language packs. The appropriate Tesseract "
+    "language pack needs to be installed."
+    "Default: eng",
+)
 @click.option(
     "--api-key",
     default="",
@@ -470,6 +478,7 @@ def main(
     partition_by_api,
     partition_endpoint,
     partition_strategy,
+    partition_ocr_languages,
     api_key,
     local_input_path,
     local_file_glob,
@@ -781,14 +790,15 @@ def main(
         logger.error("No connector-specific option was specified!")
         sys.exit(1)
 
-    process_document_with_partition_strategy = partial(
+    process_document_with_partition_args = partial(
         process_document,
         strategy=partition_strategy,
+        ocr_languages=partition_ocr_languages,
     )
 
     MainProcess(
         doc_connector=doc_connector,
-        doc_processor_fn=process_document_with_partition_strategy,
+        doc_processor_fn=process_document_with_partition_args,
         num_processes=num_processes,
         reprocess=reprocess,
         verbose=verbose,