@@ -353,12 +353,11 @@ def __run(self,
353353 start_page : int = None ,
354354 end_page : int = None ,
355355 remove_frame : bool = False ,
356- gost_json_path : str = "" ,
357- jvm_args : Optional [List [str ]] = None
356+ gost_json_path : str = ""
358357 ) -> bytes :
359358 import subprocess
360359
361- args = ["java" ] + ( jvm_args or []) + ["-jar" , self .__jar_path (), "-i" , path , "-tmp" , f"{ tmp_dir } /" ]
360+ args = ["java" ] + ["-jar" , self .__jar_path (), "-i" , path , "-tmp" , f"{ tmp_dir } /" ]
362361 if remove_frame :
363362 args += ["-rf" , gost_json_path ]
364363 if start_page is not None and end_page is not None :
@@ -381,11 +380,11 @@ def __get_raw_pages(self, path: str, parameters: dict, tmp_dir: str, first_tabby
381380 (``__tabby_raw_pages_in``), those pages are reused and only the remaining ones are extracted -- the detection
382381 read produces a complete extraction of them anyway, so re-extracting was pure duplicate work.
383382
384- Page numbers are absolute and the ranges are contiguous & disjoint, so concatenating the ``pages`` lists
385- reproduces a single-call extraction exactly (the same invariant :meth:`__process_pdf_parallel` relies on).
386- The concatenation happens *before* the per-page processing in :meth:`__extract`, which is what keeps cross-page
387- merging (paragraphs/lines spanning the boundary) intact -- handing whole parsed documents over per page range
388- instead would break it.
383+ Tabby's per-page output is page-local and page numbers are absolute, and the ranges here are contiguous and
384+ disjoint, so concatenating the ``pages`` lists reproduces a single-call extraction exactly. The concatenation
385+ happens *before* the per-page processing in :meth:`__extract`, which is what keeps cross-page merging
386+ (paragraphs/lines spanning the boundary) intact -- handing whole parsed documents over per page range instead
387+ would break it.
389388 """
390389 cached = parameters .get ("__tabby_raw_pages_in" )
391390 cached_pages = cached .get ("pages" ) if cached else None
@@ -416,65 +415,10 @@ def __process_pdf(self,
416415 import json
417416 import os
418417
419- n_chunks = self .__parallel_chunk_count (start_page , end_page , remove_frame )
420- if n_chunks > 1 :
421- return self .__process_pdf_parallel (path , tmp_dir , start_page , end_page , gost_json_path , remove_frame , n_chunks )
422-
423418 self .__run (path = path , start_page = start_page , end_page = end_page , tmp_dir = tmp_dir , remove_frame = remove_frame , gost_json_path = gost_json_path )
424419 with open (os .path .join (tmp_dir , "data.json" ), "r" , encoding = "utf-8" ) as response : # encoding= : data.json is UTF-8, avoids cp1251 breakage on RU-locale Windows
425420 return json .load (response )
426421
427- def __parallel_chunk_count (self , start_page : Optional [int ], end_page : Optional [int ], remove_frame : bool ) -> int :
428- """How many parallel tabby subprocesses to split this page range into. The tabby JAR uses only ~2 of the
429- machine's cores, so a large document is faster split into contiguous page ranges run concurrently (validated
430- bit-identical: tabby's per-page output is page-local and page numbers are absolute, so merging = concatenating
431- the ``pages`` lists). Small documents keep a single call (N JVM startups would cost more than the saving);
432- the GOST-frame path and open-ended ranges also stay single. Tunables: tabby_parallel_chunks /
433- DEDOC_TABBY_CHUNKS (cap, default 4; set 1 to disable), tabby_parallel_min_pages_per_chunk (default 20)."""
434- import os
435- if remove_frame or start_page is None or end_page is None :
436- return 1
437- max_chunks = int (os .environ .get ("DEDOC_TABBY_CHUNKS" , self .config .get ("tabby_parallel_chunks" , 4 )))
438- min_per_chunk = int (os .environ .get ("DEDOC_TABBY_MIN_PAGES_PER_CHUNK" , self .config .get ("tabby_parallel_min_pages_per_chunk" , 20 )))
439- pages = end_page - start_page + 1
440- return max (1 , min (max_chunks , pages // max (min_per_chunk , 1 )))
441-
442- def __process_pdf_parallel (self , path : str , tmp_dir : str , start_page : int , end_page : int , gost_json_path : str ,
443- remove_frame : bool , n_chunks : int ) -> dict :
444- import json
445- import math
446- import os
447- from concurrent .futures import ThreadPoolExecutor
448-
449- per_chunk = math .ceil ((end_page - start_page + 1 ) / n_chunks )
450- ranges , page = [], start_page
451- while page <= end_page :
452- ranges .append ((page , min (page + per_chunk - 1 , end_page )))
453- page = ranges [- 1 ][1 ] + 1
454- self .logger .info (f"Reading PDF in { len (ranges )} parallel tabby chunks: { ranges } " )
455- # bound each JVM's GC (SerialGC + small heap) so N concurrent tabby processes do not saturate memory bandwidth
456- # with parallel GC threads -- measured to cut inter-JVM contention. Override via DEDOC_TABBY_JVM_ARGS.
457- jvm_args = os .environ .get ("DEDOC_TABBY_JVM_ARGS" , "-XX:+UseSerialGC -Xmx1024m" ).split ()
458-
459- def run_chunk (indexed_range : tuple ) -> dict :
460- index , (chunk_start , chunk_end ) = indexed_range
461- chunk_tmp = os .path .join (tmp_dir , f"chunk_{ index } " )
462- os .makedirs (chunk_tmp , exist_ok = True )
463- self .__run (path = path , start_page = chunk_start , end_page = chunk_end , tmp_dir = chunk_tmp ,
464- remove_frame = remove_frame , gost_json_path = gost_json_path , jvm_args = jvm_args )
465- with open (os .path .join (chunk_tmp , "data.json" ), "r" , encoding = "utf-8" ) as response :
466- return json .load (response )
467-
468- with ThreadPoolExecutor (max_workers = len (ranges )) as executor :
469- documents = list (executor .map (run_chunk , enumerate (ranges )))
470-
471- # merge: page numbers are absolute and the ranges are contiguous & disjoint, so concatenating the per-chunk
472- # ``pages`` lists in range order reproduces the single-call output exactly (multi-page tables spanning a chunk
473- # boundary are still assembled downstream by TableRecognizer.convert_to_multipages_tables from page fragments)
474- merged = documents [0 ]
475- merged ["pages" ] = [page for document in documents for page in document .get ("pages" , [])]
476- return merged
477-
478422 def _process_one_page (self ,
479423 image : ndarray ,
480424 parameters : ParametersForParseDoc ,
0 commit comments