fix(paddle-ocr): complete multi-language OCR support with docs and CI

- Fix map_language_code gap: add "ka" (Kannada) identity mapping
- Update setup-paddle-ocr-models CI action for 12 script families
- Update download_paddle_models example for per-family rec layout
- Update docs (OCR guide, CLI usage, features, env vars)
- Fix backend.rs TOCTOU double-check comment clarity

Author: Goldziher

.github/actions/setup-paddle-ocr-models/action.yml

@@ -41,7 +41,7 @@ runs:
       id: cache-models
       with:
         path: ~/.cache/kreuzberg/paddle-ocr
-        key: ${{ inputs.cache-key-suffix }}-${{ runner.os }}-${{ runner.arch }}-v2
+        key: ${{ inputs.cache-key-suffix }}-${{ runner.os }}-${{ runner.arch }}-v3
         restore-keys: |
           ${{ inputs.cache-key-suffix }}-${{ runner.os }}-${{ runner.arch }}-
           ${{ inputs.cache-key-suffix }}-${{ runner.os }}-
@@ -62,7 +62,7 @@ runs:
         for attempt in 1 2 3; do
           if [ $attempt -gt 1 ]; then
             backoff=$((5 * 3 ** (attempt - 2)))
-            echo "Retry attempt $attempt/$3 after ${backoff}s backoff..."
+            echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
             sleep $backoff
           fi
 
@@ -92,7 +92,7 @@ runs:
         for attempt in 1 2 3; do
           if [ $attempt -gt 1 ]; then
             backoff=$((5 * 3 ** (attempt - 2)))
-            echo "Retry attempt $attempt/$3 after ${backoff}s backoff..."
+            echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
             sleep $backoff
           fi
 
@@ -107,22 +107,22 @@ runs:
         rm -f "$MODEL_FILE"
         exit 1
 
-    - name: Download recognition model (rec)
+    - name: Download recognition model (rec/english)
       if: contains(inputs.models, 'rec') && steps.cache-models.outputs.cache-hit != 'true'
       shell: bash
       run: |
-        MODEL_URL="https://huggingface.co/Kreuzberg/paddleocr-onnx-models/resolve/main/en_PP-OCRv4_rec_infer.onnx"
+        MODEL_URL="https://huggingface.co/Kreuzberg/paddleocr-onnx-models/resolve/main/rec/english/model.onnx"
         CACHE_DIR="$HOME/.cache/kreuzberg/paddle-ocr"
-        MODEL_DIR="$CACHE_DIR/rec"
+        MODEL_DIR="$CACHE_DIR/rec/english"
         MODEL_FILE="$MODEL_DIR/model.onnx"
 
-        echo "Downloading recognition model from $MODEL_URL"
+        echo "Downloading English recognition model from $MODEL_URL"
         mkdir -p "$MODEL_DIR"
 
         for attempt in 1 2 3; do
           if [ $attempt -gt 1 ]; then
             backoff=$((5 * 3 ** (attempt - 2)))
-            echo "Retry attempt $attempt/$3 after ${backoff}s backoff..."
+            echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
             sleep $backoff
           fi
 
@@ -137,6 +137,36 @@ runs:
         rm -f "$MODEL_FILE"
         exit 1
 
+    - name: Download recognition dictionary (rec/english/dict.txt)
+      if: contains(inputs.models, 'rec') && steps.cache-models.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        DICT_URL="https://huggingface.co/Kreuzberg/paddleocr-onnx-models/resolve/main/rec/english/dict.txt"
+        CACHE_DIR="$HOME/.cache/kreuzberg/paddle-ocr"
+        MODEL_DIR="$CACHE_DIR/rec/english"
+        DICT_FILE="$MODEL_DIR/dict.txt"
+
+        echo "Downloading English recognition dictionary from $DICT_URL"
+        mkdir -p "$MODEL_DIR"
+
+        for attempt in 1 2 3; do
+          if [ $attempt -gt 1 ]; then
+            backoff=$((5 * 3 ** (attempt - 2)))
+            echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
+            sleep $backoff
+          fi
+
+          if curl -f -L --progress-bar --connect-timeout 30 --max-time 600 \
+            -o "$DICT_FILE" "$DICT_URL"; then
+            echo "Dictionary downloaded successfully ($(du -h "$DICT_FILE" | cut -f1))"
+            exit 0
+          fi
+        done
+
+        echo "ERROR: Failed to download dictionary after 3 attempts"
+        rm -f "$DICT_FILE"
+        exit 1
+
     - name: Verify downloaded models
       id: verify-models
       shell: bash
@@ -161,11 +191,17 @@ runs:
           echo "  ✓ Classification model: $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
         fi
 
-        if [ -f "$CACHE_DIR/rec/model.onnx" ]; then
-          SIZE=$(wc -c < "$CACHE_DIR/rec/model.onnx" | tr -d ' ')
+        if [ -f "$CACHE_DIR/rec/english/model.onnx" ]; then
+          SIZE=$(wc -c < "$CACHE_DIR/rec/english/model.onnx" | tr -d ' ')
           AVAILABLE_MODELS+=("rec")
           TOTAL_SIZE=$((TOTAL_SIZE + SIZE))
-          echo "  ✓ Recognition model: $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
+          echo "  ✓ Recognition model (English): $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
+        fi
+
+        if [ -f "$CACHE_DIR/rec/english/dict.txt" ]; then
+          SIZE=$(wc -c < "$CACHE_DIR/rec/english/dict.txt" | tr -d ' ')
+          TOTAL_SIZE=$((TOTAL_SIZE + SIZE))
+          echo "  ✓ Recognition dictionary (English): $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
         fi
 
         if [ ${#AVAILABLE_MODELS[@]} -eq 0 ]; then

crates/kreuzberg/examples/download_paddle_models.rs

@@ -4,17 +4,6 @@
 //! and cache ONNX models locally. This is useful for offline applications or
 //! pre-warming the model cache before starting document extraction.
 //!
-//! # Security Notice
-//!
-//! **IMPORTANT**: The PaddleOCR models are currently downloaded without SHA256
-//! checksum verification. The model definitions in `paddle_ocr/model_manager.rs`
-//! contain empty checksum strings (lines 59, 66, 73) with a note stating:
-//! "Skip checksum for now - will be updated with actual checksums".
-//!
-//! This is a security concern for production use. Models should be verified
-//! against their known cryptographic signatures before use. See the model manager
-//! module for implementation details and to track when checksums are added.
-//!
 //! # Usage
 //!
 //! ```sh
@@ -36,14 +25,23 @@
 //!
 //! # Language Support
 //!
-//! The current implementation downloads fixed model sets optimized for:
-//! - Detection (PP-OCRv4 English)
-//! - Classification (MobileNet v2.0 Chinese/Universal)
-//! - Recognition (PP-OCRv4 English)
+//! This implementation supports 12 script families covering 106+ languages:
+//! - **English**: English-optimized recognition models
+//! - **Chinese**: Simplified and Traditional Chinese
+//! - **Latin**: European languages using Latin script
+//! - **Korean**: Hangul script
+//! - **Eslav**: Cyrillic-based languages (Russian, Ukrainian, etc.)
+//! - **Thai**: Thai script
+//! - **Greek**: Greek script
+//! - **Arabic**: Arabic and Persian scripts
+//! - **Devanagari**: Hindi and related scripts
+//! - **Tamil**: Tamil script
+//! - **Telugu**: Telugu script
+//! - **Kannada**: Kannada script
 //!
-//! Language-specific model selection is not yet implemented in the ModelManager.
-//! To use models for other languages, you would need to manually download from
-//! the PaddleOCR model repository and configure custom model paths.
+//! Models are downloaded on-demand per script family. The English recognition model
+//! and dictionary are downloaded by default. Other language families are automatically
+//! downloaded when needed during document processing.
 //!
 //! # Examples
 //!
@@ -63,6 +61,7 @@
 //!     println!("Detection model:       {:?}", models.det_model);
 //!     println!("Classification model: {:?}", models.cls_model);
 //!     println!("Recognition model:    {:?}", models.rec_model);
+//!     println!("Dictionary file:      {:?}", models.dict_file);
 //!
 //!     // Show cache statistics
 //!     let stats = manager.cache_stats()?;
@@ -173,20 +172,22 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             println!("Models being downloaded:");
             println!("  - Detection model (PP-OCRv4 det)");
             println!("  - Classification model (Mobile v2.0 cls)");
-            println!("  - Recognition model (PP-OCRv4 rec)");
-            println!("\nWARNING: Models are downloaded without checksum verification.");
-            println!("For production use, verify model integrity independently.\n");
-
-            // SECURITY: Download and ensure models exist
-            // NOTE: SHA256 checksums are currently empty in model_manager.rs
-            // This should be updated with actual checksums before production deployment
+            println!("  - Recognition model (PP-OCRv4 rec, English)");
+            println!("  - Dictionary file (for text recognition)\n");
+            println!("Additional language family models are downloaded on-demand.");
+            println!("Supported families: English, Chinese, Latin, Korean, Eslav, Thai,");
+            println!("Greek, Arabic, Devanagari, Tamil, Telugu, Kannada.\n");
+
+            // Download and ensure models exist
+            // SHA256 checksums are now embedded and verified automatically
             match manager.ensure_models_exist() {
                 Ok(paths) => {
                     println!("\nModels downloaded successfully!\n");
                     println!("Model locations:");
                     println!("  Detection:       {}", paths.det_model.display());
                     println!("  Classification:  {}", paths.cls_model.display());
                     println!("  Recognition:     {}", paths.rec_model.display());
+                    println!("  Dictionary:      {}", paths.dict_file.display());
                 }
                 Err(e) => {
                     eprintln!("Error downloading models: {}", e);
@@ -285,9 +286,10 @@ fn print_usage(program_name: &str) {
     println!("    --help, -h             Print this help message");
     println!();
     println!("NOTES:");
-    println!("    Language-specific model selection is not yet supported.");
-    println!("    Models downloaded are optimized for English/Chinese OCR.");
-    println!("    See example documentation for security considerations.");
+    println!("    Language-specific models are supported and downloaded on-demand.");
+    println!("    Supported script families: English, Chinese, Latin, Korean, Eslav,");
+    println!("    Thai, Greek, Arabic, Devanagari, Tamil, Telugu, Kannada.");
+    println!("    See example documentation for language support details.");
     println!();
     println!("EXAMPLES:");
     println!("    {} --cache-dir /tmp/models", program_name);
@@ -310,7 +312,7 @@ fn list_cache_contents(cache_dir: &PathBuf) -> Result<(), Box<dyn std::error::Er
         if path.is_dir() {
             println!("  [DIR] {}/", file_name.to_string_lossy());
 
-            // List files in subdirectory
+            // List files in subdirectory (2 levels deep for most, 3 for rec/)
             for sub_entry in fs::read_dir(&path)? {
                 let sub_entry = sub_entry?;
                 let sub_path = sub_entry.path();
@@ -321,7 +323,23 @@ fn list_cache_contents(cache_dir: &PathBuf) -> Result<(), Box<dyn std::error::Er
                     let size_kb = metadata.len() as f64 / 1000.0;
                     println!("      - {} ({:.1} KB)", sub_name.to_string_lossy(), size_kb);
                 } else if sub_path.is_dir() {
+                    // For rec/ directory, we have 3 levels: rec/{family}/{files}
                     println!("      [DIR] {}/", sub_name.to_string_lossy());
+
+                    // List files in the third level
+                    for third_entry in fs::read_dir(&sub_path)? {
+                        let third_entry = third_entry?;
+                        let third_path = third_entry.path();
+                        let third_name = third_entry.file_name();
+
+                        if third_path.is_file() {
+                            let metadata = fs::metadata(&third_path)?;
+                            let size_kb = metadata.len() as f64 / 1000.0;
+                            println!("          - {} ({:.1} KB)", third_name.to_string_lossy(), size_kb);
+                        } else if third_path.is_dir() {
+                            println!("          [DIR] {}/", third_name.to_string_lossy());
+                        }
+                    }
                 }
             }
         } else if path.is_file() {

crates/kreuzberg/src/paddle_ocr/backend.rs

@@ -144,11 +144,19 @@ impl PaddleOcrBackend {
 
         let engine = Arc::new(Mutex::new(ocr_lite));
 
-        // Insert into pool
+        // Insert into pool (with double-check for concurrent initialization)
         let mut pool = self.engine_pool.lock().map_err(|e| crate::KreuzbergError::Plugin {
             message: format!("Failed to acquire engine pool lock: {e}"),
             plugin_name: "paddle-ocr".to_string(),
         })?;
+
+        // Re-check if another thread already inserted an engine while we were creating ours
+        if let Some(existing_engine) = pool.get(family) {
+            // Another thread beat us; use their engine instead
+            return Ok(Arc::clone(existing_engine));
+        }
+
+        // We're first; insert our engine
         pool.insert(family.to_string(), Arc::clone(&engine));
 
         Ok(engine)

crates/kreuzberg/src/paddle_ocr/mod.rs

@@ -121,22 +121,26 @@ pub fn map_language_code(kreuzberg_code: &str) -> Option<&'static str> {
         "fr" | "fra" | "french" => Some("french"),
         "de" | "deu" | "german" => Some("german"),
         "ko" | "kor" | "korean" => Some("korean"),
-        "ja" | "jpn" | "japanese" => Some("japan"),
-        "chi_tra" | "zh_tw" | "zh_hant" => Some("chinese_cht"),
+        "ja" | "jpn" | "japanese" | "japan" => Some("japan"),
+        "chi_tra" | "zh_tw" | "zh_hant" | "chinese_cht" => Some("chinese_cht"),
         "ta" | "tam" | "tamil" => Some("ta"),
         "te" | "tel" | "telugu" => Some("te"),
-        "ka" | "kan" | "kannada" => Some("ka"),
+        // Kannada: "kn" is ISO 639-1 (correct), "ka" is PaddleOCR-native code for Kannada
+        // Note: "ka" is ISO 639-1 for Georgian, but PaddleOCR uses "ka" for Kannada
+        "ka" | "kn" | "kan" | "kannada" => Some("ka"),
         "ar" | "ara" | "arabic" => Some("arabic"),
-        "ru" | "rus" | "russian" | "uk" | "ukr" | "ukrainian" | "be" | "bel" | "belarusian" => Some("cyrillic"),
-        "hi" | "hin" | "hindi" => Some("devanagari"),
+        "ru" | "rus" | "russian" | "uk" | "ukr" | "ukrainian" | "be" | "bel" | "belarusian" | "cyrillic" => {
+            Some("cyrillic")
+        }
+        "hi" | "hin" | "hindi" | "devanagari" => Some("devanagari"),
         "th" | "tha" | "thai" => Some("thai"),
         "el" | "ell" | "greek" => Some("greek"),
         // Latin script fallback for European languages
-        "es" | "spa" | "spanish" | "it" | "ita" | "italian" | "pt" | "por" | "portuguese" | "nl" | "nld" | "dutch"
-        | "pl" | "pol" | "polish" | "sv" | "swe" | "swedish" | "da" | "dan" | "danish" | "no" | "nor" | "norwegian"
-        | "fi" | "fin" | "finnish" | "cs" | "ces" | "czech" | "sk" | "slk" | "slovak" | "hr" | "hrv" | "croatian"
-        | "hu" | "hun" | "hungarian" | "ro" | "ron" | "romanian" | "tr" | "tur" | "turkish" | "id" | "ind"
-        | "indonesian" | "ms" | "msa" | "malay" | "vi" | "vie" | "vietnamese" => Some("latin"),
+        "latin" | "es" | "spa" | "spanish" | "it" | "ita" | "italian" | "pt" | "por" | "portuguese" | "nl" | "nld"
+        | "dutch" | "pl" | "pol" | "polish" | "sv" | "swe" | "swedish" | "da" | "dan" | "danish" | "no" | "nor"
+        | "norwegian" | "fi" | "fin" | "finnish" | "cs" | "ces" | "czech" | "sk" | "slk" | "slovak" | "hr" | "hrv"
+        | "croatian" | "hu" | "hun" | "hungarian" | "ro" | "ron" | "romanian" | "tr" | "tur" | "turkish" | "id"
+        | "ind" | "indonesian" | "ms" | "msa" | "malay" | "vi" | "vie" | "vietnamese" => Some("latin"),
         _ => None,
     }
 }

docs/cli/usage.md

@@ -119,6 +119,37 @@ Force OCR even for PDFs with text layer:
 kreuzberg extract document.pdf --force-ocr true
 ```
 
+### OCR Language Selection
+
+Set the OCR language using the `--ocr-language` flag. This flag is backend-agnostic and works with all supported OCR backends (Tesseract, PaddleOCR, EasyOCR).
+
+**Language Code Formats:**
+
+- **Tesseract**: Uses ISO 639-3 codes (three-letter codes)
+  - Examples: `eng` (English), `fra` (French), `deu` (German), `spa` (Spanish), `jpn` (Japanese)
+- **PaddleOCR**: Accepts flexible language codes and full language names
+  - Examples: `en`, `ch`, `french`, `korean`, `thai`, `greek`, `arabic`, `cyrillic`, etc.
+- **EasyOCR**: Similar flexible format to PaddleOCR
+
+When used with `--ocr true`, the language flag overrides the default language. When used without `--ocr`, it overrides the language specified in your config file.
+
+```bash title="Terminal"
+# French OCR with Tesseract (default backend)
+kreuzberg extract --ocr true --ocr-language fra document.pdf
+
+# Chinese OCR with PaddleOCR
+kreuzberg extract --ocr true --ocr-backend paddle-ocr --ocr-language ch document.pdf
+
+# Thai OCR with PaddleOCR
+kreuzberg extract --ocr true --ocr-backend paddle-ocr --ocr-language thai document.pdf
+
+# German OCR with Tesseract
+kreuzberg extract --ocr true --ocr-language deu document.pdf
+
+# Override config file language with Spanish
+kreuzberg extract document.pdf --config kreuzberg.toml --ocr-language spa
+```
+
 ### OCR Configuration
 
 OCR options are configured via config file. CLI flags override config settings:

docs/features.md

@@ -99,7 +99,9 @@ PaddleOCR is available as a native Rust backend in all non-WASM bindings via the
 
 - Production-ready OCR using ONNX Runtime
 - Ultra-lightweight models (~25MB total)
-- 80+ language support
+- 106+ language support across 12 script families: English, Chinese, Latin, Korean, Cyrillic, Thai, Greek, Arabic, Devanagari, Tamil, Telugu, Kannada
+- Per-family models downloaded on demand
+- Concurrent multi-language OCR via engine pool
 - Excellent CJK (Chinese, Japanese, Korean) accuracy
 - No Python dependency required
 - Also available as a Python package (`pip install kreuzberg[paddleocr]`, requires Python <3.14)