Fix jina clip processor

Author: xenova

src/models/jina_clip/image_processing_jina_clip.js

@@ -1,5 +1,26 @@
-import { 
+import {
     ImageProcessor,
 } from "../../base/image_processors_utils.js";
 
-export class JinaCLIPImageProcessor extends ImageProcessor {}
+export class JinaCLIPImageProcessor extends ImageProcessor {
+    constructor(config) {
+        // JinaCLIPImageProcessor uses a custom preprocessor_config.json, so we configure it here
+        const { resize_mode, fill_color, interpolation, size, ...other } = config;
+
+        const new_size = resize_mode === 'squash'
+            ? { width: size, height: size }
+            : resize_mode === 'shortest'
+                ? { shortest_edge: size }
+                : { longest_edge: size };
+
+        const resample = interpolation === 'bicubic' ? 3 : 2;
+        super({
+            ...other,
+            size: new_size,
+            resample,
+            do_center_crop: true,
+            crop_size: size,
+            do_normalize: true,
+        });
+    }
+}

src/models/jina_clip/processing_jina_clip.js

@@ -0,0 +1,24 @@
+
+import { Processor } from "../../base/processing_utils.js";
+import { AutoImageProcessor } from "../auto/image_processing_auto.js";
+import { AutoTokenizer } from "../../tokenizers.js";
+
+export class JinaCLIPProcessor extends Processor {
+    static tokenizer_class = AutoTokenizer
+    static image_processor_class = AutoImageProcessor
+
+    async _call(text=null, images=null, kwargs = {}) {
+
+        if (!text && images){
+            throw new Error('Either text or images must be provided');
+        }
+
+        const text_inputs = text ? this.tokenizer(text, kwargs) : {};
+        const image_inputs = images ? await this.image_processor(images, kwargs) : {};
+
+        return {
+            ...text_inputs,
+            ...image_inputs,
+        }
+    }
+}

src/models/processors.js

@@ -1,6 +1,7 @@
 export * from './florence2/processing_florence2.js';
 export * from './mgp_str/processing_mgp_str.js';
 export * from './janus/processing_janus.js';
+export * from './jina_clip/processing_jina_clip.js';
 export * from './owlvit/processing_owlvit.js';
 export * from './pyannote/processing_pyannote.js';
 export * from './qwen2_vl/processing_qwen2_vl.js';

tests/processors.test.js

@@ -10,6 +10,7 @@ env.useFSCache = false;
 const sum = (array) => Number(array.reduce((a, b) => a + b, array instanceof BigInt64Array ? 0n : 0));
 const avg = (array) => sum(array) / array.length;
 
+/** @type {Map<string, RawImage>} */
 const IMAGE_CACHE = new Map();
 const load_image = async (url) => {
   const cached = IMAGE_CACHE.get(url);
@@ -40,6 +41,7 @@ const MODELS = {
   nougat: "Xenova/nougat-small",
   owlvit: "Xenova/owlvit-base-patch32",
   clip: "Xenova/clip-vit-base-patch16",
+  jina_clip: "jinaai/jina-clip-v2",
   vitmatte: "Xenova/vitmatte-small-distinctions-646",
   dinov2: "Xenova/dinov2-small-imagenet1k-1-layer",
   // efficientnet: 'Xenova/efficientnet-b0',
@@ -490,6 +492,27 @@ describe("Processors", () => {
       MAX_TEST_EXECUTION_TIME,
     );
 
+    // JinaCLIPImageProcessor
+    // - custom config overrides
+    it(
+      MODELS.jina_clip,
+      async () => {
+        const processor = await AutoImageProcessor.from_pretrained(MODELS.jina_clip);
+
+        {
+          const image = await load_image(TEST_IMAGES.tiger);
+          const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image);
+
+          compare(pixel_values.dims, [1, 3, 512, 512]);
+          compare(avg(pixel_values.data), -0.06637834757566452);
+
+          compare(original_sizes, [[408, 612]]);
+          compare(reshaped_input_sizes, [[512, 512]]);
+        }
+      },
+      MAX_TEST_EXECUTION_TIME,
+    );
+
     // VitMatteImageProcessor
     //  - tests custom overrides
     //  - tests multiple inputs