cleanup configs and run_ocr

Author: marwan37

omni-reader/configs/batch_pipeline.yaml

@@ -16,18 +16,17 @@
 #
 
 # pipeline configuration
-build: "batch-ocr-pipeline-20254007"
-run_name: "run_ocr"
+build: batch-ocr-pipeline-20254007
+run_name: run_ocr
 
 # environment configuration
 settings:
   docker:
-    requirements:
-      - requirements.txt
+    requirements: requirements.txt
     required_integrations:
       - aws
       - s3
-    python_package_installer: "uv"
+    python_package_installer: uv
     environment:
       OPENAI_API_KEY: ${{ env.OPENAI_API_KEY }}
       MISTRAL_API_KEY: ${{ env.MISTRAL_API_KEY }}
@@ -42,44 +41,44 @@ enable_step_logs: True
 steps:
   load_images:
     parameters:
-      image_folder: "./assets/samples_for_ocr"
+      image_folder: ./assets/samples_for_ocr
       image_paths: []
     enable_cache: False
 
   run_ocr:
     parameters:
       custom_prompt: null
       models: # can be model names or shorthands
-        - "pixtral"
-        - "llava-phi3"
-        - "gpt4o"
-        - "granite3.2-vision"
-        - "gemma3"
+        - pixtral
+        - gemma3
+        - llava-phi3
+        - gpt4o
+        - granite
     enable_cache: False
 
 # vision models configuration
 models_registry:
-  - name: "mistral/pixtral-12b-2409"
-    shorthand: "pixtral"
-    ocr_processor: "litellm"
-    provider: "mistral"
+  - name: mistral/pixtral-12b-2409
+    shorthand: pixtral
+    ocr_processor: litellm
+    provider: mistral
 
-  - name: "gpt-4o-mini"
-    shorthand: "gpt4o"
-    ocr_processor: "openai"
+  - name: gpt-4o-mini
+    shorthand: gpt4o
+    ocr_processor: openai
 
-  - name: "gemma3:27b"
-    shorthand: "gemma3"
-    ocr_processor: "ollama"
+  - name: gemma3:27b
+    shorthand: gemma3
+    ocr_processor: ollama
 
-  - name: "llava:34b"
-    shorthand: "llava34b"
-    ocr_processor: "ollama"
+  - name: llava:34b
+    shorthand: llava34b
+    ocr_processor: ollama
 
-  - name: "llava-phi3"
-    shorthand: "llava-phi3"
-    ocr_processor: "ollama"
+  - name: llava-phi3
+    shorthand: llava-phi3
+    ocr_processor: ollama
 
-  - name: "granite3.2-vision"
-    shorthand: "granite"
-    ocr_processor: "ollama"
+  - name: granite3.2-vision
+    shorthand: granite
+    ocr_processor: ollama

omni-reader/configs/evaluation_pipeline.yaml

@@ -16,18 +16,17 @@
 #
 
 # pipeline configuration
-build: "ocr-evaluation-pipeline-20254007"
-run_name: "ocr_evaluation_run"
+build: ocr-evaluation-pipeline-20254007
+run_name: ocr_evaluation_run
 
 # environment configuration
 settings:
   docker:
-    requirements:
-      - requirements.txt
+    requirements: requirements.txt
     required_integrations:
       - aws
       - s3
-    python_package_installer: "uv"
+    python_package_installer: uv
     environment:
       OPENAI_API_KEY: ${{ env.OPENAI_API_KEY }}
       MISTRAL_API_KEY: ${{ env.MISTRAL_API_KEY }}
@@ -43,11 +42,14 @@ steps:
   load_ocr_results: # loads OCR results from batch pipeline runs
     enable_cache: False
     parameters:
-      artifact_name: "ocr_results"
+      artifact_name: ocr_results
       version: null
 
   load_ground_truth_texts:
     enable_cache: False
     parameters:
-      ground_truth_folder: "ground_truth_texts"
+      ground_truth_folder: ground_truth_texts
       ground_truth_files: []
+
+  evaluate_models:
+    enable_cache: False

omni-reader/pipelines/batch_pipeline.py

@@ -35,6 +35,7 @@
 docker_settings = DockerSettings(
     required_integrations=["s3", "aws"],
     python_package_installer="uv",
+    requirements="requirements.txt",
     environment={
         "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY"),
         "MISTRAL_API_KEY": os.getenv("MISTRAL_API_KEY"),

omni-reader/steps/run_ocr.py

@@ -114,17 +114,15 @@ def run_ocr(
     combined_results = pl.concat(list(model_dfs.values()), how="diagonal")
 
     # Generate HTML visualization
-    try:
-        html_visualization = create_ocr_batch_visualization(combined_results)
-        log_metadata(
-            metadata={
-                "ocr_results_artifact_name": "ocr_results",
-                "ocr_results_artifact_type": "polars.DataFrame",
-                "ocr_batch_visualization_artifact_name": "ocr_batch_visualization",
-                "ocr_batch_visualization_artifact_type": "zenml.types.HTMLString",
-            },
-        )
-    except Exception as e:
-        logger.error(f"Error generating visualization: {e}")
+    html_visualization = create_ocr_batch_visualization(combined_results)
+
+    log_metadata(
+        metadata={
+            "ocr_results_artifact_name": "ocr_results",
+            "ocr_results_artifact_type": "polars.DataFrame",
+            "ocr_batch_visualization_artifact_name": "ocr_batch_visualization",
+            "ocr_batch_visualization_artifact_type": "zenml.types.HTMLString",
+        },
+    )
 
     return combined_results, html_visualization