fix(paddle-ocr): correct dict index offset, default angle cls off, configurable padding (#395)

- Prepend CTC blank token and append space token in read_keys_from_file()
  to match get_keys() layout, fixing off-by-one character mapping errors
- Default use_angle_cls to false (misfires on short text regions)
- Replace hardcoded padding=50 with configurable padding (default 10)
- Add unit tests for key loading and CTC decoding

Author: Goldziher

CHANGELOG.md

@@ -14,9 +14,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **PDF markdown output format**: Native PDF text extraction now supports `output_format: Markdown`, producing structured markdown with headings (via font-size clustering), paragraphs, inline bold/italic markup, and list detection — instead of flat text with visual line breaks.
 - **Multi-column PDF layout detection**: Histogram-based column gutter detection identifies 2+ column layouts (academic papers, magazines) and processes each column independently, preventing text interleaving across columns.
 - **Bold/italic detection via font name fallback**: When PDF font descriptor flags don't indicate bold/italic, the extractor checks font names for "Bold"/"Italic"/"Oblique" substrings and font weight >= 700 as secondary signals.
+- **musl/Alpine Linux native builds for Elixir, Java, and C#**: New Docker-based CI jobs build native libraries (`libkreuzberg_rustler.so`, `libkreuzberg_ffi.so`) targeting `x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl`. Enables instant install on Alpine Linux and musl-based distributions without compiling from source.
+- **Pre-compiled platform-specific Ruby gems**: The publish workflow now ships pre-compiled native gems for `x86_64-linux`, `aarch64-linux`, `arm64-darwin`, and `x64-mingw-ucrt`, eliminating the 30+ minute compile-from-source on `gem install kreuzberg`. A fallback source gem is still published for unsupported platforms.
 
 ### Fixed
 
+- **PaddleOCR dict index offset causing wrong character recognition (#395)**: `read_keys_from_file()` was missing the CTC blank token (`#`) at index 0 and the space token at the end, causing off-by-one character mapping errors. Now matches the `get_keys()` layout used for embedded models.
+- **PaddleOCR angle classifier misfiring on short text (#395)**: Changed `use_angle_cls` default from `true` to `false`. The angle classifier can misfire on short text regions (e.g., 2-3 character table cells), rotating crops incorrectly before recognition. Users can re-enable via `PaddleOcrConfig::with_angle_cls(true)` for rotated documents.
+- **PaddleOCR excessive padding including table gridlines (#395)**: Reduced default detection padding from 50px to 10px and made it configurable via `PaddleOcrConfig::with_padding()`. Large padding on small images caused table gridlines to be included in text crops.
+- **Ruby CI Bundler gems destroyed by vendoring script**: The `vendor-kreuzberg-core.py` script was deleting the entire `vendor/` directory including `vendor/bundle/` (Bundler's gem installation). Now only cleans crate subdirectories, preserving Bundler state.
 - **PDF document loaded twice for markdown rendering**: Eliminated redundant Pdfium initialization and document parsing by rendering markdown speculatively during the first document load, saving 25-40ms per PDF.
 - **NaN panics in PDF text clustering and block merging**: Replaced `expect()` calls on `partial_cmp` with `unwrap_or(Ordering::Equal)` across clustering, extraction, and markdown modules to handle corrupt PDF coordinates gracefully.
 - **PDF heading detection false positives**: Added distance threshold to font-size centroid matching — decorative elements with extreme font sizes no longer receive heading levels.

crates/kreuzberg-paddle-ocr/src/crnn_net.rs

@@ -107,7 +107,12 @@ impl CrnnNet {
         let content = std::fs::read_to_string(path)?;
         let mut keys = Vec::new();
 
+        // Prepend CTC blank token and append space token to match get_keys() layout.
+        // The ONNX model's output index 0 = blank, last index = space.
+        keys.push("#".to_string());
         keys.extend(content.split('\n').map(|s| s.to_string()));
+        keys.push(" ".to_string());
+
         self.keys = keys;
         Ok(())
     }
@@ -223,3 +228,56 @@ impl CrnnNet {
         Ok(text_line)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_score_to_text_line_skips_blank_index() {
+        // keys[0] = "#" (CTC blank), keys[1] = "a", keys[2] = "b"
+        let keys = vec!["#".to_string(), "a".to_string(), "b".to_string()];
+        // 3 timesteps, 3 classes each. Simulate: blank, "a", "b"
+        let output = vec![
+            1.0, 0.0, 0.0, // timestep 0: max at index 0 (blank) -> skip
+            0.0, 0.9, 0.1, // timestep 1: max at index 1 ("a")
+            0.0, 0.1, 0.8, // timestep 2: max at index 2 ("b")
+        ];
+        let result = CrnnNet::score_to_text_line(&output, 3, 3, &keys).unwrap();
+        assert_eq!(result.text, "ab");
+    }
+
+    #[test]
+    fn test_score_to_text_line_deduplicates_consecutive() {
+        let keys = vec!["#".to_string(), "h".to_string(), "i".to_string()];
+        // 4 timesteps: "h", "h", "i", "i" -> should deduplicate to "hi"
+        let output = vec![
+            0.0, 0.9, 0.0, // "h"
+            0.0, 0.8, 0.0, // "h" again (same index, skip)
+            0.0, 0.0, 0.9, // "i"
+            0.0, 0.0, 0.8, // "i" again (same index, skip)
+        ];
+        let result = CrnnNet::score_to_text_line(&output, 4, 3, &keys).unwrap();
+        assert_eq!(result.text, "hi");
+    }
+
+    #[test]
+    fn test_read_keys_from_file_has_blank_and_space() {
+        let dir = std::env::temp_dir().join("kreuzberg_test_dict");
+        std::fs::create_dir_all(&dir).unwrap();
+        let dict_path = dir.join("test_dict.txt");
+        std::fs::write(&dict_path, "a\nb\nc").unwrap();
+
+        let mut net = CrnnNet::new();
+        net.read_keys_from_file(dict_path.to_str().unwrap()).unwrap();
+
+        // Should be: ["#", "a", "b", "c", " "]
+        assert_eq!(net.keys[0], "#");
+        assert_eq!(net.keys[1], "a");
+        assert_eq!(net.keys[2], "b");
+        assert_eq!(net.keys[3], "c");
+        assert_eq!(net.keys[net.keys.len() - 1], " ");
+
+        std::fs::remove_dir_all(&dir).ok();
+    }
+}

crates/kreuzberg/src/paddle_ocr/backend.rs

@@ -260,7 +260,7 @@ impl PaddleOcrBackend {
             plugin_name: "paddle-ocr".to_string(),
         })?;
 
-        let padding = 50u32;
+        let padding = config.padding;
         let max_side_len = config.det_limit_side_len;
         let box_score_thresh = config.det_db_thresh;
         let box_thresh = config.det_db_box_thresh;

crates/kreuzberg/src/paddle_ocr/config.rs

@@ -37,7 +37,8 @@ pub struct PaddleOcrConfig {
     /// Optional custom cache directory for model files
     pub cache_dir: Option<PathBuf>,
 
-    /// Enable angle classification for rotated text (default: true)
+    /// Enable angle classification for rotated text (default: false).
+    /// Can misfire on short text regions, rotating crops incorrectly before recognition.
     pub use_angle_cls: bool,
 
     /// Enable table structure detection (default: false)
@@ -62,6 +63,10 @@ pub struct PaddleOcrConfig {
     /// Batch size for recognition inference (default: 6)
     /// Number of text regions to process simultaneously
     pub rec_batch_num: u32,
+
+    /// Padding in pixels added around the image before detection (default: 10).
+    /// Large values can include surrounding content like table gridlines.
+    pub padding: u32,
 }
 
 impl PaddleOcrConfig {
@@ -82,13 +87,14 @@ impl PaddleOcrConfig {
         Self {
             language: language.into(),
             cache_dir: None,
-            use_angle_cls: true,
+            use_angle_cls: false,
             enable_table_detection: false,
             det_db_thresh: 0.3,
             det_db_box_thresh: 0.5,
             det_db_unclip_ratio: 1.6,
             det_limit_side_len: 960,
             rec_batch_num: 6,
+            padding: 10,
         }
     }
 
@@ -191,6 +197,16 @@ impl PaddleOcrConfig {
         self
     }
 
+    /// Sets padding in pixels added around images before detection.
+    ///
+    /// # Arguments
+    ///
+    /// * `padding` - Padding in pixels (0-100)
+    pub fn with_padding(mut self, padding: u32) -> Self {
+        self.padding = padding.clamp(0, 100);
+        self
+    }
+
     /// Resolves the cache directory, checking in order:
     /// 1. Configured `cache_dir` if set
     /// 2. `KREUZBERG_CACHE_DIR` environment variable + `/paddle-ocr`
@@ -383,8 +399,9 @@ mod tests {
     fn test_new_config() {
         let config = PaddleOcrConfig::new("en");
         assert_eq!(config.language, "en");
-        assert!(config.use_angle_cls);
+        assert!(!config.use_angle_cls);
         assert!(!config.enable_table_detection);
+        assert_eq!(config.padding, 10);
     }
 
     #[test]
@@ -396,21 +413,24 @@ mod tests {
         assert_eq!(config.det_db_unclip_ratio, 1.6);
         assert_eq!(config.det_limit_side_len, 960);
         assert_eq!(config.rec_batch_num, 6);
+        assert_eq!(config.padding, 10);
     }
 
     #[test]
     fn test_builder_pattern() {
         let config = PaddleOcrConfig::new("ch")
-            .with_angle_cls(false)
+            .with_angle_cls(true)
             .with_table_detection(true)
             .with_det_db_thresh(0.4)
-            .with_rec_batch_num(12);
+            .with_rec_batch_num(12)
+            .with_padding(25);
 
         assert_eq!(config.language, "ch");
-        assert!(!config.use_angle_cls);
+        assert!(config.use_angle_cls);
         assert!(config.enable_table_detection);
         assert_eq!(config.det_db_thresh, 0.4);
         assert_eq!(config.rec_batch_num, 12);
+        assert_eq!(config.padding, 25);
     }
 
     #[test]