feat: reset to the old parameters in sanitation (#163)

Signed-off-by: Peter Staar <[email protected]>

Author: PeterStaar-IBM

app/pybind_parse.cpp

@@ -584,14 +584,14 @@ Sanitize table cells with specified parameters and return the processed JSON.
 	 [](docling::docling_sanitizer &self,
 	    double horizontal_cell_tolerance,
 	    bool enforce_same_font,
-	    double space_width_factor_for_merge = 1.0) -> nlohmann::json {
+	    double space_width_factor_for_merge/* = 1.0*/) -> nlohmann::json {
 	   return self.create_word_cells(horizontal_cell_tolerance,
 					 enforce_same_font,
 					 space_width_factor_for_merge);
 	 },
-	 pybind11::arg("horizontal_cell_tolerance")=1.0,
-	 pybind11::arg("enforce_same_font")=true,
-	 pybind11::arg("space_width_factor_for_merge")=0.33,
+	 pybind11::arg("horizontal_cell_tolerance"), // =1.0,
+	 pybind11::arg("enforce_same_font"), //=true,
+	 pybind11::arg("space_width_factor_for_merge"), //=0.33,
 	 R"(
     Create word cells
 
@@ -607,17 +607,17 @@ Sanitize table cells with specified parameters and return the processed JSON.
 	 [](docling::docling_sanitizer &self,
 	    double horizontal_cell_tolerance,
 	    bool enforce_same_font,
-	    double space_width_factor_for_merge = 1.0,
-	    double space_width_factor_for_merge_with_space = 0.33) -> nlohmann::json {
+	    double space_width_factor_for_merge /*= 1.0*/,
+	    double space_width_factor_for_merge_with_space /*= 0.33*/) -> nlohmann::json {
 	   return self.create_line_cells(horizontal_cell_tolerance,
 					 enforce_same_font,
 					 space_width_factor_for_merge,
 					 space_width_factor_for_merge_with_space);
 	 },
-	 pybind11::arg("horizontal_cell_tolerance")=1.0,
-	 pybind11::arg("enforce_same_font")=true,
-	 pybind11::arg("space_width_factor_for_merge")=1.0,
-	 pybind11::arg("space_width_factor_for_merge_with_space")=0.33,
+	 pybind11::arg("horizontal_cell_tolerance"), //=1.0,
+	 pybind11::arg("enforce_same_font"), //=true,
+	 pybind11::arg("space_width_factor_for_merge"), //=1.0,
+	 pybind11::arg("space_width_factor_for_merge_with_space"), //=0.33,
 	 R"(
     Create line cells
 

docling_parse/pdf_parser.py

@@ -275,47 +275,6 @@ def _to_page_geometry(self, dimension: dict) -> PdfPageGeometry:
             bleed_bbox=bleed_bbox,
         )
 
-    """
-    def _to_cells(self, cells: dict) -> List[Union[PdfTextCell, TextCell]]:
-
-        assert "data" in cells, '"data" in cells'
-        assert "header" in cells, '"header" in cells'
-
-        data = cells["data"]
-        header = cells["header"]
-
-        result: List[Union[PdfTextCell, TextCell]] = []
-        for ind, row in enumerate(data):
-            rect = BoundingRectangle(
-                r_x0=row[header.index(f"r_x0")],
-                r_y0=row[header.index(f"r_y0")],
-                r_x1=row[header.index(f"r_x1")],
-                r_y1=row[header.index(f"r_y1")],
-                r_x2=row[header.index(f"r_x2")],
-                r_y2=row[header.index(f"r_y2")],
-                r_x3=row[header.index(f"r_x3")],
-                r_y3=row[header.index(f"r_y3")],
-            )
-            cell = PdfTextCell(
-                rect=rect,
-                text=row[header.index(f"text")],
-                orig=row[header.index(f"text")],
-                font_key=row[header.index(f"font-key")],
-                font_name=row[header.index(f"font-name")],
-                widget=row[header.index(f"widget")],
-                text_direction=(
-                    TextDirection.LEFT_TO_RIGHT
-                    if row[header.index(f"left_to_right")]
-                    else TextDirection.RIGHT_TO_LEFT
-                ),
-                index=ind,
-                rendering_mode=row[header.index(f"rendering-mode")],
-            )
-            result.append(cell)
-
-        return result
-    """
-
     def _to_cells(self, cells: dict) -> List[Union[PdfTextCell, TextCell]]:
         assert "data" in cells, '"data" in cells'
         assert "header" in cells, '"header" in cells'
@@ -481,14 +440,22 @@ def _to_segmented_page(
 
         if create_words and ("word_cells" in page):
             segmented_page.word_cells = self._to_cells(page["word_cells"])
-        elif create_words:
+            segmented_page.has_words = len(segmented_page.word_cells) > 0
+        elif keep_chars:
+            logging.warning(
+                "`words` will be created for segmented_page in an inefficient way!"
+            )
             self._create_word_cells(segmented_page, enforce_same_font=enforce_same_font)
         else:
             logging.warning("No `words` will be created for segmented_page")
 
-        if create_textlines and ("word_cells" in page):
+        if create_textlines and ("line_cells" in page):
             segmented_page.textline_cells = self._to_cells(page["line_cells"])
-        elif create_textlines:
+            segmented_page.has_lines = len(segmented_page.textline_cells) > 0
+        elif keep_chars:
+            logging.warning(
+                "`text_lines` will be created for segmented_page in an inefficient way!"
+            )
             self._create_textline_cells(
                 segmented_page, enforce_same_font=enforce_same_font
             )
@@ -501,11 +468,11 @@ def _create_word_cells(
         self,
         segmented_page: SegmentedPdfPage,
         *,
+        horizontal_cell_tolerance: float = 1.0,
         space_width_factor_for_merge: float = 0.33,
         enforce_same_font: bool = True,
         _loglevel: str = "fatal",
     ):
-
         if len(segmented_page.word_cells) > 0:
             return
 
@@ -523,6 +490,7 @@ def _create_word_cells(
 
         # data = sanitizer.create_word_cells(space_width_factor_for_merge=0.33)
         data = sanitizer.create_word_cells(
+            horizontal_cell_tolerance=horizontal_cell_tolerance,
             space_width_factor_for_merge=space_width_factor_for_merge,
             enforce_same_font=enforce_same_font,
         )
@@ -538,6 +506,7 @@ def _create_textline_cells(
         self,
         segmented_page: SegmentedPdfPage,
         *,
+        horizontal_cell_tolerance: float = 1.0,
         space_width_factor_for_merge: float = 1.0,
         space_width_factor_for_merge_with_space: float = 0.33,
         enforce_same_font: bool = True,
@@ -564,6 +533,7 @@ def _create_textline_cells(
 
         # data = sanitizer.create_line_cells()
         data = sanitizer.create_line_cells(
+            horizontal_cell_tolerance=horizontal_cell_tolerance,
             space_width_factor_for_merge=space_width_factor_for_merge,
             space_width_factor_for_merge_with_space=space_width_factor_for_merge_with_space,
             enforce_same_font=enforce_same_font,

docling_parse/performance_timings.py

@@ -66,7 +66,7 @@ def test_performance_pdf_parse_py(
     create_textlines: bool = True,
     enforce_same_font: bool = True,
     lazy: bool = True,
-    loglevel: str = "fatal",
+    loglevel: str = "error",
 ) -> list[DocumentTiming]:
 
     pdf_docs = sorted(glob.glob(str(ifolder)))
@@ -110,10 +110,12 @@ def test_performance_pdf_parse_py(
             timing.num_pages += 1
 
             elapsed = datetime.now() - start_1_time
-            # timing.page_times.append(elapsed.total_seconds())
+            timing.page_times.append(elapsed.total_seconds())
 
             start_1_time = datetime.now()
 
+            break
+
         elapsed = datetime.now() - start_0_time
         timing.total_time = elapsed.total_seconds()
 
@@ -132,6 +134,7 @@ def main():
     for _ in timings:
         print(_)
 
+    # Optimized
     timings = test_performance_pdf_parse_py(
         ifolder=ifolder,
         keep_chars=False,
@@ -145,6 +148,7 @@ def main():
     for _ in timings:
         print(_)
 
+    # Optoimized for time, not memory
     timings = test_performance_pdf_parse_py(
         ifolder=ifolder,
         keep_chars=True,
@@ -158,6 +162,20 @@ def main():
     for _ in timings:
         print(_)
 
+    # Original ...
+    timings = test_performance_pdf_parse_py(
+        ifolder=ifolder,
+        keep_chars=True,
+        keep_lines=True,
+        keep_bitmaps=True,
+        create_words=False,
+        create_textlines=False,
+        enforce_same_font=True,
+    )
+
+    for _ in timings:
+        print(_)
+
 
 if __name__ == "__main__":
     main()

src/pybind/docling_sanitizer.h

@@ -20,18 +20,14 @@ namespace docling
 
     bool set_char_cells(nlohmann::json& data);
 
-    //bool set_char_cells(pdflib::pdf_resource<pdflib::PAGE_CELLS>& char_cells_) { char_cells = char_cells_; }
-
-    //nlohmann::json to_records(std::string label);
-
-    nlohmann::json create_word_cells(double horizontal_cell_tolerance=1.00,
-				     bool enforce_same_font=true,
-				     double space_width_factor_for_merge=0.05);
-
-    nlohmann::json create_line_cells(double horizontal_cell_tolerance=1.00,
-				     bool enforce_same_font=true,
-				     double space_width_factor_for_merge=1.00,
-				     double space_width_factor_for_merge_with_space=0.33);
+    nlohmann::json create_word_cells(double horizontal_cell_tolerance, //=1.00,
+				     bool enforce_same_font, //=true,
+				     double space_width_factor_for_merge); //=0.05);
+
+    nlohmann::json create_line_cells(double horizontal_cell_tolerance, //=1.00,
+				     bool enforce_same_font, //=true,
+				     double space_width_factor_for_merge, //=1.00,
+				     double space_width_factor_for_merge_with_space); //=0.33);
 
   private:
 
@@ -287,9 +283,9 @@ namespace docling
     */
 
     word_cells = cell_sanitizer.create_word_cells(char_cells,
-					    horizontal_cell_tolerance,
-					    enforce_same_font,
-					    space_width_factor_for_merge);
+						  horizontal_cell_tolerance,
+						  enforce_same_font,
+						  space_width_factor_for_merge);
 
     return cell_sanitizer.to_records(word_cells);
   }
@@ -326,8 +322,6 @@ namespace docling
 
     return cell_sanitizer.to_records(line_cells);
   }  
-
-  
 
 }
 

src/v2/pdf_decoders/document.h

@@ -310,16 +310,32 @@ namespace pdflib
 	    if(create_word_cells)
 	      {
 		LOG_S(INFO) << "creating word-cells in `original` (2)";        
+
+		double horizontal_cell_tolerance=1.00;
+		bool enforce_same_font=true;
+		double space_width_factor_for_merge=0.33;
 
-		pdf_resource<PAGE_CELLS> word_cells = sanitizer.create_word_cells(page_decoder.get_page_cells());
+		pdf_resource<PAGE_CELLS> word_cells = sanitizer.create_word_cells(page_decoder.get_page_cells(),
+										  horizontal_cell_tolerance,
+										  enforce_same_font,
+										  space_width_factor_for_merge);
 		page["original"]["word_cells"] = word_cells.get();
 	      }
 
 	    if(create_line_cells)
 	      {
 		LOG_S(INFO) << "creating line-cells in `original` (2)";        
+
+		double horizontal_cell_tolerance=1.00;
+		bool enforce_same_font=true;
+		double space_width_factor_for_merge=1.00;
+		double space_width_factor_for_merge_with_space=0.33;
 
-		pdf_resource<PAGE_CELLS> line_cells = sanitizer.create_line_cells(page_decoder.get_page_cells());
+		pdf_resource<PAGE_CELLS> line_cells = sanitizer.create_line_cells(page_decoder.get_page_cells(),
+										  horizontal_cell_tolerance,
+										  enforce_same_font,
+										  space_width_factor_for_merge,
+										  space_width_factor_for_merge_with_space);
 		page["original"]["line_cells"] = line_cells.get();
 	      }	    
 

src/v2/pdf_sanitators/cells.h

@@ -17,15 +17,15 @@ namespace pdflib
     nlohmann::json to_records(pdf_resource<PAGE_CELLS>& cells);
 
     pdf_resource<PAGE_CELLS> create_word_cells(pdf_resource<PAGE_CELLS>& cells,
-					       double horizontal_cell_tolerance=1.00,
-					       bool enforce_same_font=true,
-					       double space_width_factor_for_merge=0.05);
+					       double horizontal_cell_tolerance, //=1.00,
+					       bool enforce_same_font, //=true,
+					       double space_width_factor_for_merge); //=0.05);
 
     pdf_resource<PAGE_CELLS> create_line_cells(pdf_resource<PAGE_CELLS>& cells,
-					       double horizontal_cell_tolerance=1.00,
-					       bool enforce_same_font=true,
-					       double space_width_factor_for_merge=1.00,
-					       double space_width_factor_for_merge_with_space=0.33);
+					       double horizontal_cell_tolerance, //=1.00,
+					       bool enforce_same_font, //=true,
+					       double space_width_factor_for_merge, //=1.00,
+					       double space_width_factor_for_merge_with_space); //=0.33);
 
 
     void remove_duplicate_chars(pdf_resource<PAGE_CELLS>& cells, double eps=1.0e-1);
@@ -137,7 +137,8 @@ namespace pdflib
 									double space_width_factor_for_merge)
   {
     LOG_S(INFO) << __FUNCTION__;
-
+    LOG_S(INFO) << "space_width_factor_for_merge (create_word_cells): " << space_width_factor_for_merge;
+    
     // do a deep copy
     pdf_resource<PAGE_CELLS> word_cells;
     word_cells = char_cells;
@@ -182,7 +183,9 @@ namespace pdflib
 									double space_width_factor_for_merge_with_space)
   {
     LOG_S(INFO) << __FUNCTION__ << " -> char_cells: " << char_cells.size();
-
+    LOG_S(INFO) << "space_width_factor_for_merge (create_line_cells): " << space_width_factor_for_merge;
+    LOG_S(INFO) << "space_width_factor_for_merge_with_space (create_line_cells): " << space_width_factor_for_merge_with_space;
+    
     // do a deep copy
     pdf_resource<PAGE_CELLS> line_cells;
     line_cells = char_cells;