feat: add page unloading (#150)

Signed-off-by: Peter Staar <[email protected]>

Author: PeterStaar-IBM

README.md

@@ -99,7 +99,6 @@ options:
   -p PDF, --pdf PDF  Path to the PDF file
 ```
 
-
 ## Performance Benchmarks
 
 ### Characteristics of different parser versions
@@ -185,22 +184,26 @@ If you dont have an input file, then a template input file will be printed on th
 
 To build the package, simply run (make sure [uv](https://docs.astral.sh/uv/) is [installed](https://docs.astral.sh/uv/getting-started/installation)),
 
-```
+```sh
 uv sync
 ```
 
-To test the package, run:
+The latter will only work after a clean `git clone`. If you are developing and updating C++ code, please use,
 
+```sh
+uv pip install --force-reinstall --no-deps -e .
 ```
+
+To test the package, run:
+
+```sh
 uv run pytest ./tests -v -s
 ```
 
-
 ## Contributing
 
 Please read [Contributing to Docling Parse](https://github.com/docling-project/docling-parse/blob/main/CONTRIBUTING.md) for details.
 
-
 ## References
 
 If you use Docling in your projects, please consider citing the following:

app/pybind_parse.cpp

@@ -291,6 +291,36 @@ PYBIND11_MODULE(pdf_parsers, m) {
     Returns:
         bool: True if the document was successfully unloaded, False otherwise.)")
 
+    .def("unload_document_pages",
+	 [](docling::docling_parser_v2 &self, const std::string &key) -> bool {
+	   return self.unload_document_pages(key);
+	 },
+	 pybind11::arg("key"),
+	 R"(
+    Unload the only the cached pages of the document by its unique key.
+
+    Parameters:
+        key (str): The unique key of the document to unload.
+
+    Returns:
+        bool: True if the document was successfully unloaded, False otherwise.)")
+
+    .def("unload_document_page",
+	 [](docling::docling_parser_v2 &self, const std::string &key, int page) -> bool {
+	   return self.unload_document_page(key, page);
+	 },
+	 pybind11::arg("key"),
+	 pybind11::arg("page"),	 
+	 R"(
+    Unload a single page of the document by its unique key and page_number.
+
+    Parameters:
+        key (str): The unique key of the document to unload.
+        page (int): The page number of the document to unload.
+
+    Returns:
+        bool: True if the document was successfully unloaded, False otherwise.)")
+    
     .def("number_of_pages",
 	 [](docling::docling_parser_v2 &self, const std::string &key) -> int {
 	   return self.number_of_pages(key);

docling_parse/pdf_parser.py

@@ -58,6 +58,13 @@ def unload(self) -> bool:
         else:
             return False
 
+    def unload_pages(self, page_range: tuple[int, int]):
+        """unload page in range [page_range[0], page_range[1]["""
+        for page_no in range(page_range[0], page_range[1]):
+            if page_no in self._pages:
+                self._parser.unload_document_page(key=self._key, page=page_no)
+                del self._pages[page_no]
+
     def number_of_pages(self) -> int:
         if self.is_loaded():
             return self._parser.number_of_pages(key=self._key)

docling_parse/visualize.py

@@ -237,6 +237,8 @@ def visualise_py(
                 print(f"text-lines (sanitized, page_no: {page_no}):")
                 print("\n".join(lines))
 
+        pdf_doc.unload_pages(page_range=(page_no, page_no + 1))
+
 
 def main():
 

src/pybind/docling_parser_v2.h

@@ -35,6 +35,8 @@ namespace docling
     bool load_document_from_bytesio(std::string key, pybind11::object bytes_io);
 
     bool unload_document(std::string key);
+    bool unload_document_pages(std::string key);
+    bool unload_document_page(std::string key, int page_num);
 
     void unload_documents();
 
@@ -254,6 +256,40 @@ namespace docling
     return false;    
   }
 
+  bool docling_parser_v2::unload_document_page(std::string key, int page_num)
+  {
+    auto itr = key2doc.find(key);
+
+    if(itr!=key2doc.end())
+      {
+	decoder_ptr_type decoder_ptr = itr->second;
+	decoder_ptr->unload_page(page_num);
+      }
+    else
+      {
+	LOG_S(ERROR) << "key not found: " << key;	
+      }
+    
+    return false;    
+  }
+
+  bool docling_parser_v2::unload_document_pages(std::string key)
+  {
+    auto itr = key2doc.find(key);
+
+    if(itr!=key2doc.end())
+      {
+	decoder_ptr_type decoder_ptr = itr->second;
+	decoder_ptr->unload_pages();
+      }
+    else
+      {
+	LOG_S(ERROR) << "key not found: " << key;	
+      }
+    
+    return false;    
+  }
+  
   void docling_parser_v2::unload_documents()
   {
     key2doc.clear();

src/v1/proj_folders/pdf_library/qpdf/parser/font.h

@@ -911,10 +911,9 @@ namespace pdf_lib
       std::string name = _handle.getKey("/Subtype").getName();
       logging_lib::info("pdf-parser") << __FILE__ << ":" << __LINE__ << "\t detected /Subtype: " << name;
 
-      //std::cout << name << "\n";
-      
+      //if(bbox) // use the new `bool()`      
       if(bbox.isInitialized())
-        {
+	{
           if(not fm.ascent)
             {
               fm.ascent = bbox.getArrayItem(3).getNumericValue();

src/v2/pdf_decoders/document.h

@@ -29,11 +29,15 @@ namespace pdflib
 
     bool process_document_from_file(std::string& _filename);
     bool process_document_from_bytesio(std::string& _buffer);
-
+    
     void decode_document(std::string page_boundary, bool do_sanitization);
 
     void decode_document(std::vector<int>& page_numbers, std::string page_boundary, bool do_sanitization);
 
+    bool unload_pages();
+
+    bool unload_page(int page_number);
+    
   private:
 
     void update_qpdf_logger();
@@ -232,7 +236,7 @@ namespace pdflib
       {
 	utils::timer page_timer;
 
-        pdf_decoder<PAGE> page_decoder(page);
+        pdf_decoder<PAGE> page_decoder(page, page_number);
 
         auto timings_ = page_decoder.decode_page(page_boundary, do_sanitization);
 	update_timings(timings_, set_timer);
@@ -271,7 +275,7 @@ namespace pdflib
 	  {
 	    utils::timer page_timer;
 
-	    pdf_decoder<PAGE> page_decoder(pages.at(page_number));
+	    pdf_decoder<PAGE> page_decoder(pages.at(page_number), page_number);
 
 	    auto timings_ = page_decoder.decode_page(page_boundary, do_sanitization);
 
@@ -313,6 +317,45 @@ namespace pdflib
       }    
   }
 
+  bool pdf_decoder<DOCUMENT>::unload_page(int page_number)
+  {
+    if(not json_document.contains("pages"))
+      {
+	LOG_S(WARNING) << "json_document does not have `pages`";        
+	return false;
+      }
+
+    nlohmann::json& json_pages = json_document["pages"];
+    
+    for(int l=0; l<json_pages.size(); l++)
+      {
+	if((json_pages[l].is_object()) and
+	   (json_pages[l].contains("page_number")) and 
+	   (json_pages[l]["page_number"]==page_number))
+	  {
+	    json_pages[l].clear();
+
+	    nlohmann::json none;
+	    json_pages[l] = none;
+	  }
+      }
+
+    return true;
+  }
+
+  bool pdf_decoder<DOCUMENT>::unload_pages()
+  {
+    if(not json_document.contains("pages"))
+      {
+	LOG_S(WARNING) << "json_document does not have `pages`";        
+	return false;
+      }
+
+    json_document["pages"] = nlohmann::json::array({});
+
+    return true;
+  }
+    
 }
 
 #endif

src/v2/pdf_decoders/page.h

@@ -16,9 +16,11 @@ namespace pdflib
   {
   public:
 
-    pdf_decoder(QPDFObjectHandle page);
+    pdf_decoder(QPDFObjectHandle page, int page_num);
     ~pdf_decoder();
 
+    int get_page_number();
+    
     nlohmann::json get();
 
     std::map<std::string, double> decode_page(std::string page_boundary, bool do_sanitization);
@@ -49,6 +51,9 @@ namespace pdflib
   private:
 
     QPDFObjectHandle qpdf_page;
+
+    int page_number;
+    
     QPDFObjectHandle qpdf_parent_resources;
     QPDFObjectHandle qpdf_resources;
     QPDFObjectHandle qpdf_grphs;
@@ -81,19 +86,27 @@ namespace pdflib
     std::map<std::string, double> timings;
   };
 
-  pdf_decoder<PAGE>::pdf_decoder(QPDFObjectHandle page):
-    qpdf_page(page)
+  pdf_decoder<PAGE>::pdf_decoder(QPDFObjectHandle page, int page_num):
+    qpdf_page(page),
+    page_number(page_num)    
   {
   }
 
   pdf_decoder<PAGE>::~pdf_decoder()
   {
   }
+
+  int pdf_decoder<PAGE>::get_page_number()
+  {
+    return page_number;
+  }
 
   nlohmann::json pdf_decoder<PAGE>::get()
   {
     nlohmann::json result;
     {
+      result["page_number"] = page_number;
+      
       result["annotations"] = json_annots;
 
       nlohmann::json& timings_ = result["timings"];

tests/test_parse.py

@@ -288,6 +288,9 @@ def test_reference_documents_from_filenames():
             img = pred_page.render_as_image(cell_unit=TextCellUnit.LINE)
             # img.show()
 
+            print(f"unloading page: {page_no}")
+            pdf_doc.unload_pages(page_range=(page_no, page_no + 1))
+
         toc: PdfTableOfContents = pdf_doc.get_table_of_contents()
         """
         if toc is not None: