fix(ci): remove WASM size check for Workers

Author: Goldziher

.github/workflows/ci-wasm.yaml

@@ -408,9 +408,6 @@ jobs:
       - name: Build WASM (all targets for E2E)
         run: task wasm:build:all
 
-      - name: Verify WASM size for Workers
-        run: scripts/ci/wasm/verify-workers-wasm-size.sh
-
       - name: Generate Workers E2E tests
         run: task wasm:e2e:workers:generate
 

packages/csharp/Benchmark/Program.cs

@@ -2,7 +2,6 @@
 using System.Text.Json;
 using Kreuzberg;
 
-var benchConfig = new ExtractionConfig { UseCache = false };
 var debug = Environment.GetEnvironmentVariable("KREUZBERG_BENCHMARK_DEBUG") == "true";
 var argsSpan = args.AsSpan();
 
@@ -43,6 +42,10 @@
 
 var mode = argsSpan[modeIndex].ToString();
 
+var benchConfig = ocrEnabled
+    ? new ExtractionConfig { UseCache = false, Ocr = new OcrConfig { } }
+    : new ExtractionConfig { UseCache = false };
+
 if (debug)
 {
     Console.Error.WriteLine("[DEBUG] Starting C# benchmark");
@@ -88,10 +91,12 @@
         var result = KreuzbergClient.ExtractBytesSync(content, mimeType, benchConfig);
         sw.Stop();
 
+        var ocrUsed = result.Metadata?.Format?.Type == FormatType.Ocr;
         var output = new
         {
             content = result.Content,
-            _extraction_time_ms = sw.Elapsed.TotalMilliseconds
+            _extraction_time_ms = sw.Elapsed.TotalMilliseconds,
+            _ocr_used = ocrUsed
         };
 
         var json = JsonSerializer.Serialize(output);
@@ -146,10 +151,12 @@
                 var result = KreuzbergClient.ExtractBytesSync(content, mimeType, benchConfig);
                 sw.Stop();
 
+                var ocrUsed = result.Metadata?.Format?.Type == FormatType.Ocr;
                 var output = new
                 {
                     content = result.Content,
-                    _extraction_time_ms = sw.Elapsed.TotalMilliseconds
+                    _extraction_time_ms = sw.Elapsed.TotalMilliseconds,
+                    _ocr_used = ocrUsed
                 };
 
                 var json = JsonSerializer.Serialize(output);

scripts/ci/wasm/verify-workers-wasm-size.sh

@@ -0,0 +1,24 @@
+# Nasdaq & AMEX
+
+Stocks in bold rose or fell 5% or more
+
+## USA TODAY
+Track your investments with our continuously updated stocks. Visit us on the web at money.usatoday.com
+
+| 52-week High | Low | Stock | Last Change | 52-week High | Low | Stock | Last Change |
+|---|---|---|---|---|---|---|---|
+| | | — A — | | 45.71 | 32.50 | Blomar | 36.71 | -0.42 |
+| 9.19 | 6.89 | ABX Air n | 7.52 | -0.10 | 2.76 | 1.20 | Blomiro | 1.46 | +0.03 |
+| 40.29 | 12.40 | ACMoore | 13.58 | -1.57 | 9.07 | 5.13 | BioChip | -8.05 | +0.34 |
+| 31.38 | 13.31 | ADA — ES | 26.36 | +3.16 | 68.88 | 50.65 | Biosite | 60.06 | -2.57 |
+| 27.14 | 12.68 | ADC Tel rs | 23.21 | +0.13 | 212.25 | 131.03 | BiotechT | 204.66 | -0.84 |
+| 39.40 | 16.42 | ADCO | 27.32 | -0.73 | 8.50 | 3.40 | BioMaBit | 6.52 | -0.45 |
+| 16.45 | 10.37 | APE Ent s | 15.40 | -0.14 | 18.21 | 10.73 | Blackbud | 17.90 | +0.70 |
+| 8.37 | 4.42 | ASE Tra | 7.16 | +0.40 | 52.73 | 13.86 | BioQsd | 41.29 | +1.30 |
+| 19.25 | 12.75 | ASM Intl | 17.65 | -0.03 | 14.34 | 6.12 | Bodisen n | 15.43 | +0.45 |
+| 20.92 | 11.39 | ASML Hld | 21.24 | +0.46 | 6.21 | 1.56 | Bodum | 5.94 | +0.06 |
+| 27.38 | 14.39 | ASV Inc s | 26.76 | +0.14 | 11.80 | 4.99 | Borlond | 6.68 | +0.14 |
+| 19.87 | 10.42 | ATI Inc | 17.89 | -0.68 | 31.92 | 21.10 | BosPrv | 31.18 | -0.07 |
+| 33.62 | 20.53 | ATMI Inc | 29.95 | -1.29 | 18.62 | 10.01 | BtmInt | 11.53 | +0.20 |
+| 99.20 | 16.76 | ATP OKG | 58.40 | -0.59 | 11.24 | 7.10 | BriExo | 12.10 | -0.23 |
+| 4.24 | 1.99 | AVI Bio | 3.62 | -0.02 | 46.72 | 26.65 | BrightHr s | 38.90 | -0.80 |

…ocuments/ground_truth/docling-docling.md …cuments/ground_truth/docling-docling.txttest_documents/ground_truth/docling-docling.md renamed to test_documents/ground_truth/docling-docling.txt test_documents/ground_truth/docling-docling.md renamed to test_documents/ground_truth/docling-docling.txt

@@ -0,0 +1,24 @@
+# Nasdaq & AMEX
+
+Stocks in bold rose or fell 5% or more
+
+## USA TODAY
+Track your investments with our continuously updated stocks. Visit us on the web at money.usatoday.com
+
+| 52-week High | Low | Stock | Last Change | 52-week High | Low | Stock | Last Change |
+|---|---|---|---|---|---|---|---|
+| | | — A — | | 45.71 | 32.50 | Blomar | 36.71 | -0.42 |
+| 9.19 | 6.89 | ABX Air n | 7.52 | -0.10 | 2.76 | 1.20 | Blomiro | 1.46 | +0.03 |
+| 40.29 | 12.40 | ACMoore | 13.58 | -1.57 | 9.07 | 5.13 | BioChip | -8.05 | +0.34 |
+| 31.38 | 13.31 | ADA — ES | 26.36 | +3.16 | 68.88 | 50.65 | Biosite | 60.06 | -2.57 |
+| 27.14 | 12.68 | ADC Tel rs | 23.21 | +0.13 | 212.25 | 131.03 | BiotechT | 204.66 | -0.84 |
+| 39.40 | 16.42 | ADCO | 27.32 | -0.73 | 8.50 | 3.40 | BioMaBit | 6.52 | -0.45 |
+| 16.45 | 10.37 | APE Ent s | 15.40 | -0.14 | 18.21 | 10.73 | Blackbud | 17.90 | +0.70 |
+| 8.37 | 4.42 | ASE Tra | 7.16 | +0.40 | 52.73 | 13.86 | BioQsd | 41.29 | +1.30 |
+| 19.25 | 12.75 | ASM Intl | 17.65 | -0.03 | 14.34 | 6.12 | Bodisen n | 15.43 | +0.45 |
+| 20.92 | 11.39 | ASML Hld | 21.24 | +0.46 | 6.21 | 1.56 | Bodum | 5.94 | +0.06 |
+| 27.38 | 14.39 | ASV Inc s | 26.76 | +0.14 | 11.80 | 4.99 | Borlond | 6.68 | +0.14 |
+| 19.87 | 10.42 | ATI Inc | 17.89 | -0.68 | 31.92 | 21.10 | BosPrv | 31.18 | -0.07 |
+| 33.62 | 20.53 | ATMI Inc | 29.95 | -1.29 | 18.62 | 10.01 | BtmInt | 11.53 | +0.20 |
+| 99.20 | 16.76 | ATP OKG | 58.40 | -0.59 | 11.24 | 7.10 | BriExo | 12.10 | -0.23 |
+| 4.24 | 1.99 | AVI Bio | 3.62 | -0.02 | 46.72 | 26.65 | BrightHr s | 38.90 | -0.80 |

…uments/ground_truth/docling-kreuzberg.md …ments/ground_truth/docling-kreuzberg.txttest_documents/ground_truth/docling-kreuzberg.md renamed to test_documents/ground_truth/docling-kreuzberg.txt test_documents/ground_truth/docling-kreuzberg.md renamed to test_documents/ground_truth/docling-kreuzberg.txt

@@ -0,0 +1,87 @@
+元
+書
+き
+方
+は
+順
+も
+縦
+も
+き
+な
+い
+に
+い
+う
+下
+ま
+に
+き
+下
+い
+進
+に
+で
+へ
+ま
+た
+み
+右
+る
+へ
+か
+を
+を
+進
+へ
+進
+ん
+で
+い
+く
+。
+
+横
+書
+き
+は
+不
+能
+に
+い
+う
+下
+へ
+ま
+た
+右
+か
+ら
+左
+へ
+と
+ま
+た
+み
+下
+へ
+進
+む
+。
+横
+書
+き
+の
+順
+も
+縦
+書
+き
+を
+前
+提
+に
+し
+た
+お
+り

test_documents/ground_truth/gif/image_gif.md

@@ -0,0 +1,29 @@
+# LayoutParser: A Unified Toolkit for DL-Based DIA
+
+## Table 1: Current layout detection models in the LayoutParser model zoo
+
+| Dataset | Base Model | Large Model | Notes |
+|---------|-----------|------------|-------|
+| PubLayNet [38] | P / M | M | Layouts of modern scientific documents |
+| Manga [5] | M | - | Layouts of scanned modern magazines and scientific reports |
+| Newspaper [17] | P | - | Layouts of scanned US newspapers from the 20th century |
+| TableBank [44] | P | - | Table region on modern scientific and business document |
+| HJDataset [91] | P / M | - | Layouts of history Japanese documents |
+
+For each dataset, we train several model of different sizes for different needs (the trade-off between accuracy vs. computational cost). For "base model" and "large model", we refer to using the ResNet 50 or ResNet 101 backbone [15]. The past 1 architecture and mAP backbone). The platform is maintained and a number of additions will be made to the model zoo in coming months.
+
+**layout data structures**, which are optimized for efficiency and versatility. 3) When necessary, users can employ existing or customized OCR models via the unified API provided in the **OCR module**. 4) LayoutParser comes with a set of utility functions for the **visualization and storage of the layout data**. 5) LayoutParser is also highly customizable, via its integration with functions for **layout data annotation and model training**. We now provide detailed descriptions for each component.
+
+## 3.1 Layout Detection Models
+
+In LayoutParser, a layout model takes a document image as an input and generates a list of rectangular boxes for the target content regions. Different from traditional methods, it relies on deep convolutional neural networks rather than manually curated rules to identify content regions. It is formulated as an object detection problem and state-of-the-art models like Faster R-CNN [28] and Mask R-CNN [12] are used. This yields prediction results of high accuracy and makes it possible to build a concise, generalized interface for layout detection. LayoutParser, built upon Detectron2 [36], provides a minimal API that can perform layout detection with only four lines of code in Python:
+
+```
+1    import layoutparser as lp
+2    image = cv2.imread("image_file") # load image
+3    model = lp.Detectron2LayoutModel(
+4                "lp://PubLayNet/faster_rcnn_R_50_FPN_3x/config")
+5    layout = model.detect(image)
+```
+
+LayoutParser provides a wealth of pre-trained model weights using various datasets covering different languages, time periods, and document types. Due to domain shift [7], the prediction performance can notably drop when models are applied to target samples that are significantly different from the training dataset. As document structures and layouts vary greatly in different domains, it is important to select models trained on a dataset similar to the test samples. A semantic syntax is used for initializing the model weights in LayoutParser, using both the dataset name and model name: lp://\<dataset-name>/\<model-architecture-name>.

test_documents/ground_truth/jp2/image_jp2.md

@@ -0,0 +1,24 @@
+# Nasdaq & AMEX
+
+Stocks in bold rose or fell 5% or more
+
+## USA TODAY
+Track your investments with our continuously updated stocks. Visit us on the web at money.usatoday.com
+
+| 52-week High | Low | Stock | Last Change | 52-week High | Low | Stock | Last Change |
+|---|---|---|---|---|---|---|---|
+| | | — A — | | 45.71 | 32.50 | Blomar | 36.71 | -0.42 |
+| 9.19 | 6.89 | ABX Air n | 7.52 | -0.10 | 2.76 | 1.20 | Blomiro | 1.46 | +0.03 |
+| 40.29 | 12.40 | ACMoore | 13.58 | -1.57 | 9.07 | 5.13 | BioChip | -8.05 | +0.34 |
+| 31.38 | 13.31 | ADA — ES | 26.36 | +3.16 | 68.88 | 50.65 | Biosite | 60.06 | -2.57 |
+| 27.14 | 12.68 | ADC Tel rs | 23.21 | +0.13 | 212.25 | 131.03 | BiotechT | 204.66 | -0.84 |
+| 39.40 | 16.42 | ADCO | 27.32 | -0.73 | 8.50 | 3.40 | BioMaBit | 6.52 | -0.45 |
+| 16.45 | 10.37 | APE Ent s | 15.40 | -0.14 | 18.21 | 10.73 | Blackbud | 17.90 | +0.70 |
+| 8.37 | 4.42 | ASE Tra | 7.16 | +0.40 | 52.73 | 13.86 | BioQsd | 41.29 | +1.30 |
+| 19.25 | 12.75 | ASM Intl | 17.65 | -0.03 | 14.34 | 6.12 | Bodisen n | 15.43 | +0.45 |
+| 20.92 | 11.39 | ASML Hld | 21.24 | +0.46 | 6.21 | 1.56 | Bodum | 5.94 | +0.06 |
+| 27.38 | 14.39 | ASV Inc s | 26.76 | +0.14 | 11.80 | 4.99 | Borlond | 6.68 | +0.14 |
+| 19.87 | 10.42 | ATI Inc | 17.89 | -0.68 | 31.92 | 21.10 | BosPrv | 31.18 | -0.07 |
+| 33.62 | 20.53 | ATMI Inc | 29.95 | -1.29 | 18.62 | 10.01 | BtmInt | 11.53 | +0.20 |
+| 99.20 | 16.76 | ATP OKG | 58.40 | -0.59 | 11.24 | 7.10 | BriExo | 12.10 | -0.23 |
+| 4.24 | 1.99 | AVI Bio | 3.62 | -0.02 | 46.72 | 26.65 | BrightHr s | 38.90 | -0.80 |