More code cleanups

Author: pamelafox

app/backend/prepdocs.py

@@ -11,23 +11,19 @@
 from rich.logging import RichHandler
 
 from load_azd_env import load_azd_env
-from prepdocslib.csvparser import CsvParser
-from prepdocslib.fileprocessor import FileProcessor
 from prepdocslib.filestrategy import FileStrategy
 from prepdocslib.integratedvectorizerstrategy import (
     IntegratedVectorizerStrategy,
 )
-from prepdocslib.jsonparser import JsonParser
 from prepdocslib.listfilestrategy import (
     ADLSGen2ListFileStrategy,
     ListFileStrategy,
     LocalListFileStrategy,
 )
-from prepdocslib.parser import Parser
 from prepdocslib.servicesetup import (
     OpenAIHost,
+    build_file_processors,
     clean_key_if_exists,
-    select_parser,
     setup_blob_manager,
     setup_embeddings_service,
     setup_figure_processor,
@@ -36,8 +32,6 @@
     setup_search_info,
 )
 from prepdocslib.strategy import DocumentAction, Strategy
-from prepdocslib.textparser import TextParser
-from prepdocslib.textsplitter import SentenceTextSplitter, SimpleTextSplitter
 
 logger = logging.getLogger("scripts")
 
@@ -100,61 +94,20 @@ def setup_file_processors(
     openai_deployment: Optional[str] = None,
     content_understanding_endpoint: Optional[str] = None,
 ):
-    sentence_text_splitter = SentenceTextSplitter()
+    """Setup file processors and figure processor for document ingestion.
+
+    Uses build_file_processors from servicesetup to ensure consistent parser/splitter
+    selection logic with the Azure Functions cloud ingestion pipeline.
+    """
+    file_processors = build_file_processors(
+        azure_credential=azure_credential,
+        document_intelligence_service=document_intelligence_service,
+        document_intelligence_key=document_intelligence_key,
+        use_local_pdf_parser=local_pdf_parser,
+        use_local_html_parser=local_html_parser,
+        process_figures=use_multimodal,
+    )
 
-    # Build mapping of file extensions to parsers using shared select_parser helper.
-    # Each select attempt may instantiate a DI parser; duplication is acceptable at startup.
-    def _try_select(ext: str, content_type: str) -> Parser | None:
-        file_name = f"dummy{ext}"
-        try:
-            return select_parser(
-                file_name=file_name,
-                content_type=content_type,
-                azure_credential=azure_credential,
-                document_intelligence_service=document_intelligence_service,
-                document_intelligence_key=document_intelligence_key,
-                process_figures=use_multimodal,
-                use_local_pdf_parser=local_pdf_parser,
-                use_local_html_parser=local_html_parser,
-            )
-        except ValueError:
-            return None
-
-    pdf_parser: Parser | None = _try_select(".pdf", "application/pdf")
-    html_parser: Parser | None = _try_select(".html", "text/html")
-
-    # DI-only formats
-    di_exts = [
-        ".docx",
-        ".pptx",
-        ".xlsx",
-        ".png",
-        ".jpg",
-        ".jpeg",
-        ".tiff",
-        ".bmp",
-        ".heic",
-    ]
-    di_parsers: dict[str, Parser] = {}
-    for ext in di_exts:
-        parser = _try_select(ext, "application/octet-stream")
-        if parser is not None:
-            di_parsers[ext] = parser
-
-    # These file formats can always be parsed:
-    file_processors = {
-        ".json": FileProcessor(JsonParser(), SimpleTextSplitter()),
-        ".md": FileProcessor(TextParser(), sentence_text_splitter),
-        ".txt": FileProcessor(TextParser(), sentence_text_splitter),
-        ".csv": FileProcessor(CsvParser(), sentence_text_splitter),
-    }
-    # These require either a Python package or Document Intelligence
-    if pdf_parser is not None:
-        file_processors[".pdf"] = FileProcessor(pdf_parser, sentence_text_splitter)
-    if html_parser is not None:
-        file_processors[".html"] = FileProcessor(html_parser, sentence_text_splitter)
-    for ext, parser in di_parsers.items():
-        file_processors[ext] = FileProcessor(parser, sentence_text_splitter)
     figure_processor = setup_figure_processor(
         credential=azure_credential,
         use_multimodal=use_multimodal,

app/backend/prepdocslib/servicesetup.py

@@ -12,13 +12,17 @@
 from openai import AsyncOpenAI
 
 from .blobmanager import BlobManager
+from .csvparser import CsvParser
 from .embeddings import ImageEmbeddings, OpenAIEmbeddings
 from .figureprocessor import FigureProcessor, MediaDescriptionStrategy
+from .fileprocessor import FileProcessor
 from .htmlparser import LocalHTMLParser
+from .jsonparser import JsonParser
 from .parser import Parser
 from .pdfparser import DocumentAnalysisParser, LocalPdfParser
 from .strategy import SearchInfo
 from .textparser import TextParser
+from .textsplitter import SentenceTextSplitter, SimpleTextSplitter
 
 logger = logging.getLogger("scripts")
 
@@ -241,77 +245,92 @@ def setup_figure_processor(
     return None
 
 
-def select_parser(
+def build_file_processors(
     *,
-    file_name: str,
-    content_type: str,
     azure_credential: AsyncTokenCredential,
     document_intelligence_service: str | None,
     document_intelligence_key: str | None = None,
-    process_figures: bool = False,
     use_local_pdf_parser: bool = False,
     use_local_html_parser: bool = False,
-) -> Parser:
-    """Return a parser instance appropriate for the file type and configuration.
-
-    Args:
-        file_name: Source filename (used to derive extension)
-        content_type: MIME type (fallback for extension-based selection)
-        azure_credential: Token credential for DI service
-        document_intelligence_service: Name of DI service (None disables DI)
-        document_intelligence_key: Optional key credential (overrides token when provided)
-        process_figures: Whether figure extraction should be enabled in DI parser
-        use_local_pdf_parser: Force local PDF parsing instead of DI
-        use_local_html_parser: Force local HTML parsing instead of DI
-
-    Returns:
-        Parser capable of yielding Page objects for the document.
-
-    Raises:
-        ValueError: Unsupported file type or missing DI configuration for required formats.
-    """
-    extension = file_name.lower().rsplit(".", 1)[-1] if "." in file_name else ""
-    ext_with_dot = f".{extension}" if extension else ""
+    process_figures: bool = False,
+) -> dict[str, FileProcessor]:
+    sentence_text_splitter = SentenceTextSplitter()
 
-    # Build DI parser lazily only if needed
-    di_parser: DocumentAnalysisParser | None = None
+    doc_int_parser: Optional[DocumentAnalysisParser] = None
+    # check if Azure Document Intelligence credentials are provided
     if document_intelligence_service:
         credential: AsyncTokenCredential | AzureKeyCredential
         if document_intelligence_key:
             credential = AzureKeyCredential(document_intelligence_key)
         else:
             credential = azure_credential
-        di_parser = DocumentAnalysisParser(
+        doc_int_parser = DocumentAnalysisParser(
             endpoint=f"https://{document_intelligence_service}.cognitiveservices.azure.com/",
             credential=credential,
             process_figures=process_figures,
         )
 
-    # Plain text / structured text formats always local
-    if ext_with_dot in {".txt", ".md", ".csv", ".json"} or content_type.startswith("text/plain"):
-        return TextParser()
-
-    # HTML
-    if ext_with_dot in {".html", ".htm"} or content_type in {"text/html", "application/html"}:
-        if use_local_html_parser or not di_parser:
-            return LocalHTMLParser()
-        return di_parser
-
-    # PDF
-    if ext_with_dot == ".pdf":
-        if use_local_pdf_parser or not di_parser:
-            return LocalPdfParser()
-        return di_parser
-
-    # Formats requiring DI
-    di_required_exts = {".docx", ".pptx", ".xlsx", ".png", ".jpg", ".jpeg", ".tiff", ".bmp", ".heic"}
-    if ext_with_dot in di_required_exts:
-        if not di_parser:
-            raise ValueError("Document Intelligence service must be configured to process this file type")
-        return di_parser
-
-    # Fallback: if MIME suggests application/* and DI available, use DI
-    if content_type.startswith("application/") and di_parser:
-        return di_parser
-
-    raise ValueError(f"Unsupported file type: {file_name}")
+    pdf_parser: Optional[Parser] = None
+    if use_local_pdf_parser or document_intelligence_service is None:
+        pdf_parser = LocalPdfParser()
+    elif document_intelligence_service is not None:
+        pdf_parser = doc_int_parser
+    else:
+        logger.warning("No PDF parser available")
+
+    html_parser: Optional[Parser] = None
+    if use_local_html_parser or document_intelligence_service is None:
+        html_parser = LocalHTMLParser()
+    elif document_intelligence_service is not None:
+        html_parser = doc_int_parser
+    else:
+        logger.warning("No HTML parser available")
+
+    # These file formats can always be parsed:
+    file_processors = {
+        ".json": FileProcessor(JsonParser(), SimpleTextSplitter()),
+        ".md": FileProcessor(TextParser(), sentence_text_splitter),
+        ".txt": FileProcessor(TextParser(), sentence_text_splitter),
+        ".csv": FileProcessor(CsvParser(), sentence_text_splitter),
+    }
+    # These require either a Python package or Document Intelligence
+    if pdf_parser is not None:
+        file_processors.update({".pdf": FileProcessor(pdf_parser, sentence_text_splitter)})
+    if html_parser is not None:
+        file_processors.update({".html": FileProcessor(html_parser, sentence_text_splitter)})
+    # These file formats require Document Intelligence
+    if doc_int_parser is not None:
+        file_processors.update(
+            {
+                ".docx": FileProcessor(doc_int_parser, sentence_text_splitter),
+                ".pptx": FileProcessor(doc_int_parser, sentence_text_splitter),
+                ".xlsx": FileProcessor(doc_int_parser, sentence_text_splitter),
+                ".png": FileProcessor(doc_int_parser, sentence_text_splitter),
+                ".jpg": FileProcessor(doc_int_parser, sentence_text_splitter),
+                ".jpeg": FileProcessor(doc_int_parser, sentence_text_splitter),
+                ".tiff": FileProcessor(doc_int_parser, sentence_text_splitter),
+                ".bmp": FileProcessor(doc_int_parser, sentence_text_splitter),
+                ".heic": FileProcessor(doc_int_parser, sentence_text_splitter),
+            }
+        )
+    return file_processors
+
+
+def select_processor_for_filename(file_name: str, file_processors: dict[str, FileProcessor]) -> FileProcessor:
+    """Select the appropriate file processor for a given filename.
+
+    Args:
+        file_name: Name of the file to process
+        file_processors: Dictionary mapping file extensions to FileProcessor instances
+
+    Returns:
+        FileProcessor instance for the file
+
+    Raises:
+        ValueError: If the file extension is not supported
+    """
+    file_ext = os.path.splitext(file_name)[1].lower()
+    file_processor = file_processors.get(file_ext)
+    if not file_processor:
+        raise ValueError(f"Unsupported file type: {file_name}")
+    return file_processor

app/functions/document_extractor/function_app.py

@@ -15,8 +15,12 @@
 from azure.core.exceptions import HttpResponseError
 from azure.identity.aio import ManagedIdentityCredential
 
+from prepdocslib.fileprocessor import FileProcessor
 from prepdocslib.page import Page
-from prepdocslib.servicesetup import select_parser
+from prepdocslib.servicesetup import (
+    build_file_processors,
+    select_processor_for_filename,
+)
 
 app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS)
 
@@ -25,10 +29,7 @@
 
 @dataclass
 class GlobalSettings:
-    use_local_pdf_parser: bool
-    use_local_html_parser: bool
-    use_multimodal: bool
-    document_intelligence_service: str | None
+    file_processors: dict[str, FileProcessor]
     azure_credential: ManagedIdentityCredential
 
 
@@ -52,11 +53,18 @@ def configure_global_settings():
         logger.info("Using default Managed Identity without client ID")
         azure_credential = ManagedIdentityCredential()
 
-    settings = GlobalSettings(
+    # Build file processors dict for parser selection
+    file_processors = build_file_processors(
+        azure_credential=azure_credential,
+        document_intelligence_service=document_intelligence_service,
+        document_intelligence_key=None,
         use_local_pdf_parser=use_local_pdf_parser,
         use_local_html_parser=use_local_html_parser,
-        use_multimodal=use_multimodal,
-        document_intelligence_service=document_intelligence_service,
+        process_figures=use_multimodal,
+    )
+
+    settings = GlobalSettings(
+        file_processors=file_processors,
         azure_credential=azure_credential,
     )
 
@@ -176,16 +184,9 @@ async def process_document(data: dict[str, Any]) -> dict[str, Any]:
     document_stream, file_name, content_type = get_document_stream_filedata(data)
     logger.info("Processing document: %s", file_name)
 
-    parser = select_parser(
-        file_name=file_name,
-        content_type=content_type,
-        azure_credential=settings.azure_credential,
-        document_intelligence_service=settings.document_intelligence_service,
-        document_intelligence_key=None,
-        process_figures=settings.use_multimodal,
-        use_local_pdf_parser=settings.use_local_pdf_parser,
-        use_local_html_parser=settings.use_local_html_parser,
-    )
+    # Get parser from file_processors dict based on file extension
+    file_processor = select_processor_for_filename(file_name, settings.file_processors)
+    parser = file_processor.parser
 
     pages: list[Page] = []
     try: