feat: deal with qpdf errors on a page by page basis (#11)

* adding load/unload from key

Signed-off-by: Peter Staar <[email protected]>

* updated tests

Signed-off-by: Peter Staar <[email protected]>

* all fixed, still need to clean all commented out code

Signed-off-by: Peter Staar <[email protected]>

* ran pre-commit hooks

Signed-off-by: Peter Staar <[email protected]>

* allow more tabulate versions

Signed-off-by: Michele Dolfi <[email protected]>

* renamed some key functions

Signed-off-by: Peter Staar <[email protected]>

* ran pre-commit hooks (2)

Signed-off-by: Peter Staar <[email protected]>

* added log-level to run

Signed-off-by: Peter Staar <[email protected]>

* Added try-catch to avoid poisonous pdf-pages (2)

Signed-off-by: Peter Staar <[email protected]>

* updated the readme

Signed-off-by: Peter Staar <[email protected]>

* removed multi-threading build

Signed-off-by: Peter Staar <[email protected]>

---------

Signed-off-by: Peter Staar <[email protected]>
Signed-off-by: Michele Dolfi <[email protected]>
Co-authored-by: Michele Dolfi <[email protected]>

Author: PeterStaar-IBM

README.md

@@ -58,6 +58,9 @@ for page in range(0, num_pages):
     # No need to unload a specifc page 
     json_doc = parser.parse_pdf_from_key_on_page(doc_key, page)
 
+    if "pages" not in json_doc:  # page could not get parsed
+       continue
+
     # parsed page is the first one!				  
     json_page = json_doc["pages"][0] 
 

build.py

@@ -39,7 +39,7 @@ def build_local(multi_threaded=True):
 
     cmd = f"cmake --build {BUILD_DIR} --target install"
     if multi_threaded:
-        cmd += " -j"
+        cmd += " -j 4"
     success = run(cmd, cwd=ROOT_DIR)
     if not success:
         raise RuntimeError("Error building.")

docling_parse/run.py

@@ -13,6 +13,16 @@ def main():
     # Create the argument parser
     parser = argparse.ArgumentParser(description="Process a PDF file.")
 
+    # Add an argument for the path to the PDF file
+    parser.add_argument(
+        "-l",
+        "--log-level",
+        type=int,
+        required=False,
+        default=2,
+        help="log-level 1,2,3,4",
+    )
+
     # Add an argument for the path to the PDF file
     parser.add_argument(
         "-p", "--pdf", type=str, help="Path to the PDF file", required=True
@@ -30,6 +40,7 @@ def main():
     # Print the path to the PDF file (or add your processing logic here)
 
     parser = docling_parse.pdf_parser()
+    parser.set_loglevel(args.log_level)
 
     doc_file = args.pdf  # filename
     doc_key = f"key={args.pdf}"  # unique document key (eg hash, UUID, etc)
@@ -43,6 +54,10 @@ def main():
     # Parse page by page to minimize memory footprint
     for page in range(0, num_pages):
         json_doc = parser.parse_pdf_from_key_on_page(doc_key, page)
+
+        if "pages" not in json_doc:  # page could not get parsed
+            continue
+
         json_page = json_doc["pages"][0]
 
         page_dimensions = [

src/proj_folders/pdf_interface/parser.h

@@ -691,7 +691,23 @@ namespace pdf_lib
     auto& parser = loaded_parsers.at(key);
 
     doc->resize_pages(0);
-    parser->process_all();
+    try
+      {
+	parser->process_all();
+      }
+    catch (const std::exception& e)  // Catch standard exceptions
+      {
+	logging_lib::error("pdf-parser") << __FILE__ << ":" << __LINE__
+					 << " error with process_page_from_document: "
+					 << e.what();
+	return false;
+      }
+    catch (...)  // Catch any other types of exceptions
+      {
+	logging_lib::error("pdf-parser") << __FILE__ << ":" << __LINE__
+					 << " unknown error with process_page_from_document";
+	return false;
+      }
 
     try
       {
@@ -725,8 +741,24 @@ namespace pdf_lib
 
     auto& doc = loaded_documents.at(key);
     auto& parser = loaded_parsers.at(key);
-    
-    parser->process_page_from_document(page);
+
+    try
+      {
+	parser->process_page_from_document(page);
+      }
+    catch (const std::exception& e)  // Catch standard exceptions
+      {
+	logging_lib::error("pdf-parser") << __FILE__ << ":" << __LINE__
+					 << " error with process_page_from_document: "
+					 << e.what();
+	return false;
+      }    
+    catch (...)
+      {
+        logging_lib::error("pdf-parser") << __FILE__ << ":" << __LINE__
+					 << " error with process_page_from_document";
+	return false;
+      }
 
     try
       {
@@ -739,7 +771,7 @@ namespace pdf_lib
     catch (...)
       {
         logging_lib::error("pdf-parser") << __FILE__ << ":" << __LINE__
-                                         << "\t ERROR in conversion pdf_lib::core::DOCUMENT --> container !!\n";
+                                         << " ERROR in conversion pdf_lib::core::DOCUMENT --> container !!\n";
         return false;
       }
 
@@ -755,10 +787,18 @@ namespace pdf_lib
 
     pdf_lib::core::object<pdf_lib::core::DOCUMENT> doc;
 
-    {
-      pdf_lib::qpdf::parser<pdf_lib::core::DOCUMENT> parser(doc);
-      parser.load_document(filename).process_all();
-    }
+    try
+      {
+	pdf_lib::qpdf::parser<pdf_lib::core::DOCUMENT> parser(doc);
+	parser.load_document(filename).process_all();
+      }
+    catch (const std::exception& e)  // Catch standard exceptions
+      {
+	logging_lib::error("pdf-parser") << __FILE__ << ":" << __LINE__
+					 << " error with process_page_from_document: "
+					 << e.what();
+	return false;
+      }
 
     try
       {