Raghav/image embedding

demos/python_demos/image_embeddings/model_conversion/clip_conversion.py

@@ -0,0 +1,29 @@
+from transformers import CLIPProcessor, CLIPModel
+from PIL import Image
+import openvino as ov
+import os
+
+model_id="openai/clip-vit-base-patch32"
+print(f"Downloading pretrained model {model_id}...")
+
+full_model=CLIPModel.from_pretrained(model_id)
+model=full_model.vision_model
+processor=CLIPProcessor.from_pretrained(model_id)
+
+image=Image.new("RGB",(224,224))
+inputs=processor(images=image,return_tensors="pt")["pixel_values"]
+
+print("Converting model...")
+ov_model=ov.convert_model(model,example_input=inputs)
+ov.save_model(ov_model,"clip_image_encoder.xml")
+print("Model saved!")
+
+mod_path="saved_mod/1"
+os.makedirs(mod_path,exist_ok=True)
+os.replace("clip_image_encoder.xml", f"{mod_path}/clip_image_encoder.xml")
+os.replace("clip_image_encoder.bin", f"{mod_path}/clip_image_encoder.bin")
+print("Model ready for OVMS")
+
+
+
+

demos/python_demos/image_embeddings/servable/config_model.json

@@ -0,0 +1,15 @@
+{
+  "model_config_list": [
+    {
+      "config": {
+        "name": "clip",
+        "base_path": "/saved_mod/"
+      }
+    }],
+    "mediapipe_config_list": [
+      {
+        "name": "python_model",
+        "graph_path": "/workspace/grph_pipeline.pbtxt"
+      }
+    ]
+}

demos/python_demos/image_embeddings/servable/graph_pipeline.pbtxt

@@ -0,0 +1,72 @@
+input_stream: "OVMS_PY_TENSOR:image"
+output_stream: "OVMS_PY_TENSOR:embedding"
+
+node{
+    name: "ImagePreprocessor"
+    calculator: "PythonExecutorCalculator"
+    input_stream: "OVMS_PY_TENSOR:image"
+    output_stream: "OVMS_PY_TENSOR:pixel_values"
+    node_options:{
+        [type.googleapis.com/mediapipe.PythonExecutorCalculatorOptions]:{
+            handler_path: "/workspace/pre.py"
+        }
+    }
+}
+
+node{
+    name: "PixelValuesConverter"
+    calculator: "PyTensorOvTensorConverterCalculator"
+    input_stream: "OVMS_PY_TENSOR:pixel_values"
+    output_stream: "OVTENSOR:40"
+}
+
+node {
+  calculator: "OpenVINOModelServerSessionCalculator"
+  output_side_packet: "SESSION:session"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOModelServerSessionCalculatorOptions]: {
+      servable_name: "clip"
+      servable_version: "1"
+    }
+  }
+}
+
+node {
+  calculator: "OpenVINOInferenceCalculator"
+  input_side_packet: "SESSION:session"
+  input_stream: "OVTENSOR:40"
+  output_stream: "OVTENSOR:pooler_output"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOInferenceCalculatorOptions]: {
+      tag_to_input_tensor_names {
+        key: "OVTENSOR"
+        value: "40"
+      }
+      tag_to_output_tensor_names {
+        key: "OVTENSOR"
+        value: "pooler_output"
+      }
+    }
+  }
+}
+
+
+node {
+  name: "EmbeddingConverter"
+  calculator: "OvTensorPyTensorConverterCalculator"
+  input_stream: "OVTENSOR:pooler_output"
+  output_stream: "OVMS_PY_TENSOR:pooler_output"
+}
+
+
+node {
+  name: "Postprocessor"
+  calculator: "PythonExecutorCalculator"
+  input_stream: "OVMS_PY_TENSOR:pooler_output"
+  output_stream: "OVMS_PY_TENSOR:embedding"
+  node_options: {
+    [type.googleapis.com/mediapipe.PythonExecutorCalculatorOptions]: {
+      handler_path: "/workspace/post.py"
+    }
+  }
+}

demos/python_demos/image_embeddings/servable/post.py

@@ -0,0 +1,16 @@
+from pyovms import Tensor
+import numpy as np
+from scipy.special import softmax
+from tritonclient.utils import deserialize_bytes_tensor
+
+class OvmsPythonModel:
+
+    def initialize(self, kwargs:dict):
+        pass
+
+    def execute(self, inputs:list):
+        embedding=inputs[0].as_numpy()
+        norm=np.linalg.norm(embedding, axis=1, keepdims=True)
+        normalized_embedding=embedding/norm
+
+        return[Tensor(name="embedding", data=normalized_embedding.astype(np.float32))]

demos/python_demos/image_embeddings/servable/pre.py

@@ -0,0 +1,20 @@
+from pyovms import Tensor
+from transformers import CLIPProcessor
+from PIL import Image
+import numpy as np
+from io import BytesIO
+from tritonclient.utils import deserialize_bytes_tensor
+
+class OvmsPythonModel:
+    def initialize(self, kwargs:dict):
+        model_id="openai/clip-vit-base-patch32"
+        self.processor=CLIPProcessor.from_pretrained(model_id)
+
+    def execute(self,inputs: list):
+        image_bytes = deserialize_bytes_tensor(bytes(inputs[0]))[0]
+
+        image=Image.open(BytesIO(image_bytes)).convert("RGB")
+        processed=self.processor(images=image,return_tensors="np")
+        pixel_values=processed["pixel_values"].astype(np.float32)
+        return[Tensor(name="40",data=pixel_values)]
+