Raghav/image embedding

Dockerfile.ubuntu

@@ -396,8 +396,8 @@ RUN apt-get update ; \
             curl -L -O https://github.com/intel/linux-npu-driver/releases/download/v1.16.0/intel-driver-compiler-npu_1.16.0.20250328-14132024782_ubuntu24.04_amd64.deb ; \
             curl -L -O https://github.com/intel/linux-npu-driver/releases/download/v1.16.0/intel-fw-npu_1.16.0.20250328-14132024782_ubuntu24.04_amd64.deb   ; \
             curl -L -O https://github.com/intel/linux-npu-driver/releases/download/v1.16.0/intel-level-zero-npu_1.16.0.20250328-14132024782_ubuntu24.04_amd64.deb ; \
-            curl -L -O https://github.com/oneapi-src/level-zero/releases/download/v1.20.2/level-zero_1.20.2+u24.04_amd64.deb ; \ 
-        fi ; \      
+            curl -L -O https://github.com/oneapi-src/level-zero/releases/download/v1.20.2/level-zero_1.20.2+u24.04_amd64.deb ; \
+        fi ; \
         dpkg -i ./*.deb && rm -Rf /tmp/npu_deps ; \
     fi ; \
     apt-get clean ; \

demos/python_demos/image_embeddings/grpc_cli.py

@@ -0,0 +1,82 @@
+import sys
+sys.path.append("../../common/python")
+import tritonclient.grpc as grpcclient
+from tritonclient.grpc import service_pb2
+from tritonclient.grpc import service_pb2_grpc
+from tritonclient.utils import serialize_byte_tensor
+
+import argparse
+import datetime
+import numpy as np
+from client_utils import print_statistics
+from urllib.request import urlretrieve
+from pathlib import Path
+import os
+import grpc
+import time
+
+parser = argparse.ArgumentParser(description='GRPC client for clip example')
+
+parser.add_argument('--timeout', required=False, default='15',
+                    help='Specify timeout to wait for models readiness on the server in seconds. default 15 seconds.')
+parser.add_argument('--url', required=False, default='localhost:9000',
+                    help='Specify url to grpc service. default:localhost:9000')
+
+parser.add_argument('--image_url', required=False, default='https://images.unsplash.com/photo-1716467891152-1b43a96de578?q=80&w=1481&auto=format&fit=crop&ixlib=rb-4.1.0&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D',
+                    help='Specify image_url to send to the CLIP model. default:https://images.unsplash.com/photo-1716467891152-1b43a96de578?q=80&w=1481&auto=format&fit=crop&ixlib=rb-4.1.0&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D')
+
+args = vars(parser.parse_args())
+
+iterations = args.get('iterations')
+iteration = 0
+
+timeout = int(args.get('timeout'))
+# Check models ready
+client = grpcclient.InferenceServerClient(args['url'])
+channel = grpc.insecure_channel(args['url'])
+grpc_stub = service_pb2_grpc.GRPCInferenceServiceStub(channel)
+
+while(timeout):
+    request = service_pb2.ServerReadyRequest()
+    response = grpc_stub.ServerReady(request)
+    print("Server Ready: {}".format(response.ready))
+    if response.ready:
+        break
+    time.sleep(1)
+    timeout-=1
+
+if not response.ready:
+    print("Models are not ready. Increase timeout or check server setup and errors.")
+    exit(-1)
+
+image_url = args['image_url']
+print(f"Using image_url:\n{image_url}\n")
+
+input_name = image_url.split("/")[-1]
+sample_path = Path(os.path.join("data", input_name))
+if not os.path.exists(sample_path):
+    sample_path.parent.mkdir(parents=True, exist_ok=True)
+    urlretrieve(
+        image_url,
+        sample_path,
+    )
+
+with open(sample_path, "rb") as f:
+    image_data = f.read()
+
+image_np = np.array([image_data], dtype=np.object_)
+
+# Set up inference input correctly
+image_input = grpcclient.InferInput("image", [1], "BYTES")
+image_input.set_data_from_numpy(image_np)
+
+processing_times = []
+start_time = datetime.datetime.now()
+results = client.infer("python_model", [image_input])
+end_time = datetime.datetime.now()
+duration = (end_time - start_time).total_seconds() * 1000
+processing_times.append(int(duration))
+print(f"Detection:\n{results.as_numpy('embedding')}\n")
+
+print_statistics(np.array(processing_times,int), batch_size = 1)
+

demos/python_demos/image_embeddings/model_conversion/clip_conversion.py

@@ -0,0 +1,29 @@
+from transformers import CLIPProcessor, CLIPModel
+from PIL import Image
+import openvino as ov
+import os
+
+model_id="openai/clip-vit-base-patch32"
+print(f"Downloading pretrained model {model_id}...")
+
+full_model=CLIPModel.from_pretrained(model_id)
+model=full_model.vision_model
+processor=CLIPProcessor.from_pretrained(model_id)
+
+image=Image.new("RGB",(224,224))
+inputs=processor(images=image,return_tensors="pt")["pixel_values"]
+
+print("Converting model...")
+ov_model=ov.convert_model(model,example_input=inputs)
+ov.save_model(ov_model,"clip_image_encoder.xml")
+print("Model saved!")
+
+mod_path="saved_mod/1"
+os.makedirs(mod_path,exist_ok=True)
+os.replace("clip_image_encoder.xml", f"{mod_path}/clip_image_encoder.xml")
+os.replace("clip_image_encoder.bin", f"{mod_path}/clip_image_encoder.bin")
+print("Model ready for OVMS")
+
+
+
+

demos/python_demos/image_embeddings/model_conversion/dino_conversion.py

@@ -0,0 +1,27 @@
+from transformers import AutoImageProcessor, AutoModel
+from PIL import Image
+import openvino as ov
+import os
+
+model_id="facebook/dinov2-base"
+print(f"Downloading pretrained model {model_id}...")
+
+model=AutoModel.from_pretrained(model_id)
+processor=AutoImageProcessor.from_pretrained(model_id)
+
+image=Image.new("RGB",(224,224))
+inputs=processor(images=image,return_tensors="pt")["pixel_values"]
+
+print("Converting models...")
+ov_model=ov.convert_model(model,example_input=inputs)
+ov.save_model(ov_model,"dino_image_encoder.xml")
+print("Model saved!")
+
+mod_path="saved_mod/dino/1"
+os.makedirs(mod_path,exist_ok=True)
+os.replace("dino_image_encoder.xml", f"{mod_path}/dino_image_encoder.xml")
+os.replace("dino_image_encoder.bin", f"{mod_path}/dino_image_encoder.bin")
+print("Model ready for OVMS")
+
+
+

demos/python_demos/image_embeddings/model_conversion/laion_conversion.py

@@ -0,0 +1,32 @@
+from transformers import CLIPProcessor, CLIPModel
+from PIL import Image
+import openvino as ov
+import torch
+import os
+
+# Replace this with your LAION model
+model_id = "laion/CLIP-ViT-B-32-laion2B-s34B-b79K"
+print(f"Downloading pretrained model: {model_id}")
+
+# Load processor and model
+processor = CLIPProcessor.from_pretrained(model_id)
+full_model = CLIPModel.from_pretrained(model_id)
+image_encoder = full_model.vision_model
+image_encoder.eval()
+
+# Dummy image input for tracing
+image = Image.new("RGB", (224, 224))
+inputs = processor(images=image, return_tensors="pt")["pixel_values"]
+
+# Convert to OpenVINO IR
+print("Converting image encoder to OpenVINO IR...")
+ov_model = ov.convert_model(image_encoder, example_input=inputs)
+ov.save_model(ov_model, "clip_image_encoder.xml")
+print("Model saved!")
+
+# Move to proper OVMS path
+mod_path = "saved_mod/laion/1"
+os.makedirs(mod_path, exist_ok=True)
+os.replace("clip_image_encoder.xml", f"{mod_path}/clip_image_encoder.xml")
+os.replace("clip_image_encoder.bin", f"{mod_path}/clip_image_encoder.bin")
+print(f"Model ready at {mod_path} for OpenVINO Model Server")

demos/python_demos/image_embeddings/requirements.txt

@@ -0,0 +1,2 @@
+tritonclient[all]==2.51.0
+numpy<2.0.0

demos/python_demos/image_embeddings/servable/config_model.json

@@ -0,0 +1,36 @@
+{
+  "model_config_list": [
+    {
+      "config": {
+        "name": "clip",
+        "base_path": "/saved_mod/clip"
+      }
+    },
+    {
+      "config": {
+        "name": "dino",
+        "base_path": "/saved_mod/dino"
+      }
+    },
+    {
+      "config": {
+        "name": "laion",
+        "base_path": "/saved_mod/laion"
+      }
+    }
+  ],
+  "mediapipe_config_list": [
+    {
+      "name": "clip_graph",
+      "graph_path": "/workspace/graph_clip.pbtxt"
+    },
+    {
+      "name": "dino_graph",
+      "graph_path": "/workspace/graph_dino.pbtxt"
+    },
+    {
+      "name": "laion_graph",
+      "graph_path": "/workspace/graph_laion.pbtxt"
+    }
+  ]
+}

demos/python_demos/image_embeddings/servable/graph_pipeline.pbtxt

@@ -0,0 +1,81 @@
+input_stream: "OVMS_PY_TENSOR:image"
+output_stream: "OVMS_PY_TENSOR:embedding"
+
+node{
+    name: "ImagePreprocessor"
+    calculator: "PythonExecutorCalculator"
+    input_side_packet: "PYTHON_NODE_RESOURCES:py"
+    input_stream: "OVMS_PY_TENSOR:image"
+    output_stream: "OVMS_PY_TENSOR:pixel_values"
+    node_options:{
+        [type.googleapis.com/mediapipe.PythonExecutorCalculatorOptions]:{
+            handler_path: "/workspace/pre.py"
+        }
+    }
+}
+
+node{
+    name: "PixelValuesConverter"
+    calculator: "PyTensorOvTensorConverterCalculator"
+    input_stream: "OVMS_PY_TENSOR:pixel_values"
+    output_stream: "OVTENSOR:image_em"
+}
+
+node {
+  calculator: "OpenVINOModelServerSessionCalculator"
+  output_side_packet: "SESSION:session"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOModelServerSessionCalculatorOptions]: {
+      servable_name: "clip"
+      servable_version: "1"
+    }
+  }
+}
+
+node {
+  calculator: "OpenVINOInferenceCalculator"
+  input_side_packet: "SESSION:session"
+  input_stream: "OVTENSOR:image_em"
+  output_stream: "OVTENSOR:pooler_output"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOInferenceCalculatorOptions]: {
+      tag_to_input_tensor_names {
+        key: "OVTENSOR"
+        value: "40"
+      }
+      tag_to_output_tensor_names {
+        key: "OVTENSOR"
+        value: "pooler_output"
+      }
+    }
+  }
+}
+
+
+node {
+  name: "EmbeddingConverter"
+  calculator: "PyTensorOvTensorConverterCalculator"
+  input_stream: "OVTENSOR:pooler_output"
+  output_stream: "OVMS_PY_TENSOR:pooler_output_em"
+  node_options: {
+        [type.googleapis.com/mediapipe.PyTensorOvTensorConverterCalculatorOptions]: {
+          tag_to_output_tensor_names{
+            key: "OVMS_PY_TENSOR"
+            value: "pooler_output_em"
+          }
+        }
+}
+}
+
+node {
+  name: "Postprocessor"
+  calculator: "PythonExecutorCalculator"
+  input_side_packet: "PYTHON_NODE_RESOURCES:py"
+  input_stream: "OVMS_PY_TENSOR:pooler_output_em"
+  output_stream: "OVMS_PY_TENSOR:embedding"
+  node_options: {
+    [type.googleapis.com/mediapipe.PythonExecutorCalculatorOptions]: {
+      handler_path: "/workspace/post.py"
+    }
+  }
+}

demos/python_demos/image_embeddings/servable/post.py

@@ -0,0 +1,25 @@
+from pyovms import Tensor
+import numpy as np
+
+class OvmsPythonModel:
+    def initialize(self, kwargs: dict):
+        self.node_name = kwargs.get("node_name", "")
+        if "clip" in self.node_name.lower():
+            self.mode = "clip"
+        elif "dino" in self.node_name.lower():
+            self.mode = "dino"
+        elif "laion" in self.node_name.lower():
+            self.mode = "laion"
+        else:
+            raise ValueError(f"Unsupported model type in node name: {self.node_name}")
+
+    def execute(self, inputs: list) -> list:
+        try:
+            tensor = inputs[0]
+            embedding = np.frombuffer(tensor.data, dtype=np.float32).reshape(tensor.shape)
+            norm = np.linalg.norm(embedding, axis=1, keepdims=True) + 1e-10
+            normalized = embedding / norm
+            return [Tensor(name="embedding", buffer=normalized.astype(np.float32))]
+        except Exception as e:
+            print(">>> ERROR in Postprocessor:", str(e))
+            raise

demos/python_demos/image_embeddings/servable/pre.py

@@ -0,0 +1,33 @@
+from pyovms import Tensor
+from transformers import CLIPProcessor,AutoImageProcessor
+from PIL import Image
+import numpy as np
+from io import BytesIO
+from tritonclient.utils import deserialize_bytes_tensor
+
+class OvmsPythonModel:
+    def initialize(self, kwargs:dict):
+        self.node_name=kwargs.get("node_name","")
+
+        if "clip" in self.node_name.lower():
+            self.processor=CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+            self.mode="clip"
+
+        elif "dino" in self.node_name.lower():
+            self.processor=AutoImageProcessor.from_pretrained("facebook/dinov2-base")
+            self.mode="dino"
+
+        elif "laion" in self.node_name.lower():
+            self.processor=CLIPProcessor.from_pretrained("laion/CLIP-ViT-B-32-laion2B-s34B-b79K")
+            self.mode="laion"
+        else:
+            raise ValueError(f"Unsupported model type in node name: {self.node_name}")
+
+    def execute(self,inputs: list):
+        image_bytes = deserialize_bytes_tensor(bytes(inputs[0]))[0]
+
+        image=Image.open(BytesIO(image_bytes)).convert("RGB")
+        processed=self.processor(images=image,return_tensors="np")
+        pixel_values=processed["pixel_values"].astype(np.float32)
+        return[Tensor(name="pixel_values",buffer=pixel_values)]
+