Chore: throw 400 error if a PDF is password protected (#147)

* pip compile expect don't bump to 0.8.0

* Revert "pip compile expect don't bump to 0.8.0"

This reverts commit 7a62df5.

* only do pycryptodome install

* disable test

* note

* stage for 400

* password protect  400 error

* add test file

* lint

* lint..

* put back coordinate test

* name nit:  pdf_page_splits -> pdf_pages

* new pip-compile

* changelog

Author: yuming-long

CHANGELOG.md

@@ -1,5 +1,6 @@
-## 0.0.32-dev0
+## 0.0.32-dev1
 
+* Throw 400 error if a PDF is password protected
 * Improve logging of params to single line json
 * Add support for `include_page_breaks` parameter
 

pipeline-notebooks/pipeline-general.ipynb

@@ -549,11 +549,10 @@
     "\n",
     "from concurrent.futures import ThreadPoolExecutor\n",
     "from functools import partial\n",
+    "import pypdf\n",
     "from pypdf import PdfReader, PdfWriter\n",
     "from unstructured.partition.auto import partition\n",
     "from unstructured.staging.base import convert_to_isd, convert_to_dataframe, elements_from_json\n",
-    "import tempfile\n",
-    "import pdfminer\n",
     "import requests\n",
     "import time"
    ]
@@ -598,7 +597,7 @@
    "source": [
     "# pipeline-api\n",
     "\n",
-    "def get_pdf_splits(pdf, split_size=1):\n",
+    "def get_pdf_splits(pdf_pages, split_size=1):\n",
     "    '''\n",
     "    Given a pdf (PdfReader) with n pages, split it into pdfs each with split_size # of pages\n",
     "    Return the files with their page offset in the form [( BytesIO, int)]\n",
@@ -607,12 +606,12 @@
     "\n",
     "    offset = 0\n",
     "\n",
-    "    while offset < len(pdf.pages):\n",
+    "    while offset < len(pdf_pages):\n",
     "        new_pdf = PdfWriter()\n",
     "        pdf_buffer = io.BytesIO()\n",
     "\n",
     "        end = offset+split_size\n",
-    "        for page in pdf.pages[offset : end]:\n",
+    "        for page in pdf_pages[offset : end]:\n",
     "            new_pdf.add_page(page)\n",
     "\n",
     "        new_pdf.write(pdf_buffer)\n",
@@ -678,7 +677,7 @@
     "\n",
     "    return elements\n",
     "\n",
-    "def partition_pdf_splits(request, file, file_filename, content_type, coordinates, **partition_kwargs):\n",
+    "def partition_pdf_splits(request, pdf_pages, file, file_filename, content_type, coordinates, **partition_kwargs):\n",
     "    '''\n",
     "    Split a pdf into chunks and process in parallel with more api calls, or partition\n",
     "    locally if the chunk is small enough. As soon as any remote call fails, bubble up\n",
@@ -691,10 +690,9 @@
     "    partition_kwargs holds any others parameters that will be forwarded, or passed to partition\n",
     "    '''    \n",
     "    pages_per_pdf = int(os.environ.get(\"UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE\", 1))\n",
-    "    pdf = PdfReader(file)\n",
     "\n",
     "    # If it's small enough, just process locally\n",
-    "    if len(pdf.pages) <= pages_per_pdf:\n",
+    "    if len(pdf_pages) <= pages_per_pdf:\n",
     "        return partition(\n",
     "            file=file,\n",
     "            file_filename=file_filename,\n",
@@ -703,7 +701,7 @@
     "       )\n",
     "\n",
     "    results = []\n",
-    "    page_tuples = get_pdf_splits(pdf, split_size=pages_per_pdf)\n",
+    "    page_tuples = get_pdf_splits(pdf_pages, split_size=pages_per_pdf)\n",
     "    \n",
     "    partition_func = partial(\n",
     "        partition_file_via_api,\n",
@@ -771,6 +769,20 @@
     "        # Note(yuming): convert file type for msg files\n",
     "        # since fast api might sent the wrong one.\n",
     "        file_content_type = \"application/x-ole-storage\"\n",
+    "        \n",
+    "    if filename.endswith(\".pdf\"):\n",
+    "        try: \n",
+    "            pdf = PdfReader(file)\n",
+    "        except pypdf.errors.EmptyFileError:\n",
+    "            raise HTTPException(\n",
+    "                status_code=400,\n",
+    "                detail=f\"{filename} does not appear to be a valid PDF\"\n",
+    "            )\n",
+    "        if  pdf.is_encrypted:\n",
+    "            raise HTTPException(\n",
+    "                status_code=400,\n",
+    "                detail=f\"File: {filename} is encrypted. Please decrypt it with password.\"\n",
+    "            )\n",
     "    \n",
     "    strategy = (m_strategy[0] if len(m_strategy) else 'auto').lower()\n",
     "    strategies = ['fast', 'hi_res', 'auto', 'ocr_only']\n",
@@ -828,6 +840,7 @@
     "        if file_content_type == \"application/pdf\" and pdf_parallel_mode_enabled:\n",
     "            elements = partition_pdf_splits(\n",
     "                request,\n",
+    "                pdf_pages = pdf.pages,\n",
     "                file=file,\n",
     "                file_filename=filename,\n",
     "                content_type=file_content_type,\n",
@@ -856,8 +869,6 @@
     "        if 'Invalid file' in e.args[0]:\n",
     "            raise HTTPException(status_code=400, detail=f\"{file_content_type} not currently supported\")\n",
     "        raise e\n",
-    "    except pdfminer.pdfparser.PDFSyntaxError:\n",
-    "        raise HTTPException(status_code=400, detail=f\"{filename} does not appear to be a valid PDF\")\n",
     "\n",
     "    if response_type == \"text/csv\":\n",
     "        df = convert_to_dataframe(elements)\n",

prepline_general/api/general.py

@@ -20,10 +20,10 @@
 import pandas as pd
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
+import pypdf
 from pypdf import PdfReader, PdfWriter
 from unstructured.partition.auto import partition
 from unstructured.staging.base import convert_to_isd, convert_to_dataframe, elements_from_json
-import pdfminer
 import requests
 import time
 from unstructured_inference.models.chipper import MODEL_TYPES as CHIPPER_MODEL_TYPES
@@ -70,7 +70,7 @@ def is_expected_response_type(media_type, response_type):
     os.environ["UNSTRUCTURED_ALLOWED_MIMETYPES"] = DEFAULT_MIMETYPES
 
 
-def get_pdf_splits(pdf, split_size=1):
+def get_pdf_splits(pdf_pages, split_size=1):
     """
     Given a pdf (PdfReader) with n pages, split it into pdfs each with split_size # of pages
     Return the files with their page offset in the form [( BytesIO, int)]
@@ -79,12 +79,12 @@ def get_pdf_splits(pdf, split_size=1):
 
     offset = 0
 
-    while offset < len(pdf.pages):
+    while offset < len(pdf_pages):
         new_pdf = PdfWriter()
         pdf_buffer = io.BytesIO()
 
         end = offset + split_size
-        for page in pdf.pages[offset:end]:
+        for page in pdf_pages[offset:end]:
             new_pdf.add_page(page)
 
         new_pdf.write(pdf_buffer)
@@ -150,7 +150,7 @@ def partition_file_via_api(file_tuple, request, filename, content_type, **partit
 
 
 def partition_pdf_splits(
-    request, file, file_filename, content_type, coordinates, **partition_kwargs
+    request, pdf_pages, file, file_filename, content_type, coordinates, **partition_kwargs
 ):
     """
     Split a pdf into chunks and process in parallel with more api calls, or partition
@@ -164,16 +164,15 @@ def partition_pdf_splits(
     partition_kwargs holds any others parameters that will be forwarded, or passed to partition
     """
     pages_per_pdf = int(os.environ.get("UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE", 1))
-    pdf = PdfReader(file)
 
     # If it's small enough, just process locally
-    if len(pdf.pages) <= pages_per_pdf:
+    if len(pdf_pages) <= pages_per_pdf:
         return partition(
             file=file, file_filename=file_filename, content_type=content_type, **partition_kwargs
         )
 
     results = []
-    page_tuples = get_pdf_splits(pdf, split_size=pages_per_pdf)
+    page_tuples = get_pdf_splits(pdf_pages, split_size=pages_per_pdf)
 
     partition_func = partial(
         partition_file_via_api,
@@ -236,6 +235,19 @@ def pipeline_api(
         # since fast api might sent the wrong one.
         file_content_type = "application/x-ole-storage"
 
+    if filename.endswith(".pdf"):
+        try:
+            pdf = PdfReader(file)
+        except pypdf.errors.EmptyFileError:
+            raise HTTPException(
+                status_code=400, detail=f"{filename} does not appear to be a valid PDF"
+            )
+        if pdf.is_encrypted:
+            raise HTTPException(
+                status_code=400,
+                detail=f"File: {filename} is encrypted. Please decrypt it with password.",
+            )
+
     strategy = (m_strategy[0] if len(m_strategy) else "auto").lower()
     strategies = ["fast", "hi_res", "auto", "ocr_only"]
     if strategy not in strategies:
@@ -301,6 +313,7 @@ def pipeline_api(
         if file_content_type == "application/pdf" and pdf_parallel_mode_enabled:
             elements = partition_pdf_splits(
                 request,
+                pdf_pages=pdf.pages,
                 file=file,
                 file_filename=filename,
                 content_type=file_content_type,
@@ -331,8 +344,6 @@ def pipeline_api(
                 status_code=400, detail=f"{file_content_type} not currently supported"
             )
         raise e
-    except pdfminer.pdfparser.PDFSyntaxError:
-        raise HTTPException(status_code=400, detail=f"{filename} does not appear to be a valid PDF")
 
     if response_type == "text/csv":
         df = convert_to_dataframe(elements)

requirements/base.in

@@ -8,5 +8,5 @@ click==8.1.3
 ratelimit
 requests
 pypdf
-
+pycryptodome
 

requirements/base.txt

@@ -20,7 +20,7 @@ beautifulsoup4==4.12.2
     # via nbconvert
 bleach==6.0.0
     # via nbconvert
-certifi==2023.5.7
+certifi==2023.7.22
     # via requests
 cffi==1.15.1
     # via cryptography
@@ -54,7 +54,7 @@ exceptiongroup==1.1.2
     # via anyio
 fastapi==0.100.0
     # via unstructured-api-tools
-fastjsonschema==2.17.1
+fastjsonschema==2.18.0
     # via nbformat
 filelock==3.12.2
     # via
@@ -65,7 +65,7 @@ filetype==1.2.0
     # via unstructured
 flatbuffers==23.5.26
     # via onnxruntime
-fonttools==4.41.0
+fonttools==4.41.1
     # via matplotlib
 fsspec==2023.6.0
     # via huggingface-hub
@@ -103,9 +103,9 @@ jinja2==3.1.2
     #   unstructured-api-tools
 joblib==1.3.1
     # via nltk
-jsonschema==4.18.3
+jsonschema==4.18.4
     # via nbformat
-jsonschema-specifications==2023.6.1
+jsonschema-specifications==2023.7.1
     # via jsonschema
 jupyter-client==8.3.0
     # via nbclient
@@ -146,7 +146,7 @@ mypy-extensions==1.0.0
     # via mypy
 nbclient==0.8.0
     # via nbconvert
-nbconvert==7.7.1
+nbconvert==7.7.2
     # via unstructured-api-tools
 nbformat==5.9.1
     # via
@@ -203,7 +203,7 @@ pdfminer-six==20221105
     # via
     #   pdfplumber
     #   unstructured
-pdfplumber==0.10.0
+pdfplumber==0.10.1
     # via layoutparser
 pillow==10.0.0
     # via
@@ -227,6 +227,8 @@ pycocotools==2.0.6
     # via effdet
 pycparser==2.21
     # via cffi
+pycryptodome==3.18.0
+    # via -r requirements/base.in
 pydantic==1.10.11
     # via
     #   -r requirements/base.in
@@ -239,7 +241,7 @@ pypandoc==1.11
     # via unstructured
 pyparsing==3.0.9
     # via matplotlib
-pypdf==3.12.2
+pypdf==3.13.0
     # via -r requirements/base.in
 pypdfium2==4.18.0
     # via pdfplumber
@@ -264,7 +266,7 @@ python-pptx==0.6.21
     # via unstructured
 pytz==2023.3
     # via pandas
-pyyaml==6.0
+pyyaml==6.0.1
     # via
     #   huggingface-hub
     #   layoutparser
@@ -276,7 +278,7 @@ pyzmq==25.1.0
     # via jupyter-client
 ratelimit==2.2.1
     # via -r requirements/base.in
-referencing==0.29.1
+referencing==0.30.0
     # via
     #   jsonschema
     #   jsonschema-specifications
@@ -291,7 +293,7 @@ requests==2.31.0
     #   torchvision
     #   transformers
     #   unstructured
-rpds-py==0.8.11
+rpds-py==0.9.2
     # via
     #   jsonschema
     #   referencing
@@ -353,13 +355,13 @@ traitlets==5.9.0
     #   nbclient
     #   nbconvert
     #   nbformat
-transformers==4.30.2
+transformers==4.31.0
     # via unstructured-inference
-types-requests==2.31.0.1
+types-requests==2.31.0.2
     # via unstructured-api-tools
-types-ujson==5.8.0.0
+types-ujson==5.8.0.1
     # via unstructured-api-tools
-types-urllib3==1.26.25.13
+types-urllib3==1.26.25.14
     # via types-requests
 typing-extensions==4.7.1
     # via
@@ -371,6 +373,7 @@ typing-extensions==4.7.1
     #   pypdf
     #   starlette
     #   torch
+    #   uvicorn
 tzdata==2023.3
     # via pandas
 unstructured[local-inference]==0.8.1
@@ -379,9 +382,9 @@ unstructured-api-tools==0.10.10
     # via -r requirements/base.in
 unstructured-inference==0.5.5
     # via unstructured
-urllib3==2.0.3
+urllib3==2.0.4
     # via requests
-uvicorn[standard]==0.23.0
+uvicorn[standard]==0.23.1
     # via unstructured-api-tools
 uvloop==0.17.0
     # via uvicorn