feat: add OpenVINO inference provider (#2848)

* feat: add OpenVINO inference provider

Fixes #2847

Signed-off-by: Jeff MAURY <[email protected]>

Author: jeffmaury

packages/backend/src/assets/ai.json

@@ -517,6 +517,15 @@
       },
       "memory": 4372811936,
       "backend": "llama-cpp"
+    },
+    {
+      "id": "OpenVINO/mistral-7B-instruct-v0.2-int4-ov",
+      "name": "OpenVINO/mistral-7B-instruct-v0.2-int4-ov",
+      "description": "# Mistral-7B-Instruct-v0.2-int4-ov\n* Model creator: [Mistral AI](https://huggingface.co/mistralai)\n * Original model: [Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2)\n\n## Description\n\nThis is [Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) model converted to the [OpenVINO™ IR](https://docs.openvino.ai/2024/documentation/openvino-ir-format.html) (Intermediate Representation) format.\n\n## Compatibility\n\nThe provided OpenVINO™ IR model is compatible with:\n\n* OpenVINO version 2024.2.0 and higher\n* Optimum Intel 1.19.0 and higher\n\n## Running Model Inference with [Optimum Intel](https://huggingface.co/docs/optimum/intel/index)\n\n\n1. Install packages required for using [Optimum Intel](https://huggingface.co/docs/optimum/intel/index) integration with the OpenVINO backend:\n\n```\npip install optimum[openvino]\n```\n\n2. Run model inference:\n\n```\nfrom transformers import AutoTokenizer\nfrom optimum.intel.openvino import OVModelForCausalLM\n\nmodel_id = \"OpenVINO/<model_name>\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\nmodel = OVModelForCausalLM.from_pretrained(model_id)\n\ninputs = tokenizer(\"What is OpenVINO?\", return_tensors=\"pt\")\n\noutputs = model.generate(**inputs, max_length=200)\ntext = tokenizer.batch_decode(outputs)[0]\nprint(text)\n```\n\nFor more examples and possible optimizations, refer to the [OpenVINO Large Language Model Inference Guide](https://docs.openvino.ai/2024/learn-openvino/llm_inference_guide.html).\n\n## Running Model Inference with [OpenVINO GenAI](https://github.com/openvinotoolkit/openvino.genai)\n\n1. Install packages required for using OpenVINO GenAI.\n```\npip install openvino-genai huggingface_hub\n```\n\n2. Download model from HuggingFace Hub\n   \n```\nimport huggingface_hub as hf_hub\n\nmodel_id = \"OpenVINO/Mistral-7B-Instruct-v0.2-int4-ov\"\nmodel_path = \"Mistral-7B-Instruct-v0.2-int4-ov\"\n\nhf_hub.snapshot_download(model_id, local_dir=model_path)\n\n```\n\n3. Run model inference:\n\n```\nimport openvino_genai as ov_genai\n\ndevice = \"CPU\"\npipe = ov_genai.LLMPipeline(model_path, device)\nprint(pipe.generate(\"What is OpenVINO?\", max_length=200))\n```\n\nMore GenAI usage examples can be found in OpenVINO GenAI library [docs](https://github.com/openvinotoolkit/openvino.genai/blob/master/src/README.md) and [samples](https://github.com/openvinotoolkit/openvino.genai?tab=readme-ov-file#openvino-genai-samples)\n\n## Limitations\n\nCheck the original model card for [limitations](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2#limitations).\n\n## Legal information\n\nThe original model is distributed under [apache-2.0](https://choosealicense.com/licenses/apache-2.0/) license. More details can be found in [original model card](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2).\n\n## Disclaimer\n\nIntel is committed to respecting human rights and avoiding causing or contributing to adverse impacts on human rights. See [Intel’s Global Human Rights Principles](https://www.intel.com/content/dam/www/central-libraries/us/en/documents/policy-human-rights.pdf). Intel’s products and software are intended only to be used in applications that do not cause or contribute to adverse impacts on human rights.",
+      "registry": "Hugging Face",
+      "license": "Apache-2.0",
+      "url": "huggingface:/OpenVINO/mistral-7B-instruct-v0.2-int4-ov",
+      "backend": "openvino"
     }
   ],
   "categories": [

packages/backend/src/assets/inference-images.json

@@ -5,5 +5,8 @@
   "llamacpp": {
     "default": "quay.io/ramalama/ramalama-llama-server@sha256:4e56101073e0bd6f2f2e15839b64315656d0dbfc1331a3385f2ae722e13f2279",
     "cuda": "quay.io/ramalama/cuda-llama-server@sha256:56efc824e5b3ae6a6a11e9537ed9e2ac05f9f9fc6f2e81a55eb67b662c94fe95"
+  },
+  "openvino": {
+    "default": "quay.io/ramalama/openvino@sha256:670d91cc322933cc4263606459317cd4ca3fcfb16d59a46b11dcd498c2cd7cb5"
   }
 }

packages/backend/src/managers/inference/inferenceManager.spec.ts

@@ -288,6 +288,7 @@ describe('Create Inference Server', () => {
       enabled: () => true,
       name: 'dummy-inference-provider',
       dispose: () => {},
+      prePerform: vi.fn().mockReturnValue(Promise.resolve()),
       perform: vi.fn<() => InferenceServer>().mockResolvedValue({
         container: {
           containerId: 'dummy-container-id',

packages/backend/src/managers/inference/inferenceManager.ts

@@ -193,6 +193,8 @@ export class InferenceManager extends Publisher<InferenceServer[]> implements Di
 
     if (!connection) throw new Error('cannot find running container provider connection');
 
+    await provider.prePerform(config);
+
     // upload models to podman machine if user system is supported
     config.modelsInfo = await Promise.all(
       config.modelsInfo.map(modelInfo =>

packages/backend/src/managers/modelsManager.spec.ts

@@ -658,7 +658,7 @@ describe('deleting models', () => {
       'machine-2',
       'rm',
       '-f',
-      '/home/user/ai-lab/models/dummyFile',
+      '/home/user/ai-lab/models/model-id-1',
     ]);
   });
 });

packages/backend/src/managers/playgroundV2Manager.spec.ts

@@ -206,13 +206,15 @@ test('valid submit should create IPlaygroundMessage and notify the webview', asy
         {
           id: 'dummyModelId',
           file: {
+            path: '.',
             file: 'dummyModelFile',
           },
         },
       ],
       connection: {
         port: 8888,
       },
+      labels: [],
     } as unknown as InferenceServer,
   ]);
   // @ts-expect-error the mocked return value is just a partial of the real OpenAI provider
@@ -284,13 +286,15 @@ test('error', async () => {
         {
           id: 'dummyModelId',
           file: {
+            path: '.',
             file: 'dummyModelFile',
           },
         },
       ],
       connection: {
         port: 8888,
       },
+      labels: [],
     } as unknown as InferenceServer,
   ]);
   const doStream: LanguageModelV1['doStream'] = async () => {
@@ -685,13 +689,15 @@ describe('system prompt', () => {
           {
             id: 'dummyModelId',
             file: {
+              path: '.',
               file: 'dummyModelFile',
             },
           },
         ],
         connection: {
           port: 8888,
         },
+        labels: [],
       } as unknown as InferenceServer,
     ]);
     // @ts-expect-error the mocked return value is just a partial of the real OpenAI provider

packages/backend/src/managers/playgroundV2Manager.ts

@@ -218,7 +218,7 @@ export class PlaygroundV2Manager implements Disposable {
       timestamp: Date.now(),
     } as UserChat);
 
-    if (!modelInfo.file?.file) throw new Error('model info has undefined file.');
+    if (!modelInfo.file?.path) throw new Error('model info has undefined file.');
 
     const telemetry: Record<string, unknown> = {
       conversationId: conversationId,
@@ -243,7 +243,7 @@ export class PlaygroundV2Manager implements Disposable {
 
     const openAiClient = createOpenAICompatible({
       name: modelInfo.name,
-      baseURL: `http://localhost:${server.connection.port}/v1`,
+      baseURL: server.labels['api'] ?? `http://localhost:${server.connection.port}/v1`,
     });
     let model = openAiClient(modelInfo.name);
     // Tool calling in OpenAI doesn't support streaming yet

packages/backend/src/models/HuggingFaceModelHandler.ts

@@ -129,7 +129,7 @@ export class HuggingFaceModelHandler extends ModelHandler {
               if (model) {
                 model.model.file = {
                   path: revision.path,
-                  file: revision.path,
+                  file: '',
                   creation: revision.lastModifiedAt,
                   size: revision.size,
                 };

packages/backend/src/studio.ts

@@ -62,6 +62,8 @@ import { HuggingFaceModelHandler } from './models/HuggingFaceModelHandler';
 import { LlamaStackApiImpl } from './llama-stack-api-impl';
 import { LLAMA_STACK_API_CHANNEL, type LlamaStackAPI } from '@shared/LlamaStackAPI';
 import { LlamaStackManager } from './managers/llama-stack/llamaStackManager';
+import { OpenVINO } from './workers/provider/OpenVINO';
+import os from 'node:os';
 
 export class Studio {
   readonly #extensionContext: ExtensionContext;
@@ -280,6 +282,13 @@ export class Studio {
     this.#extensionContext.subscriptions.push(
       this.#inferenceProviderRegistry.register(new WhisperCpp(this.#taskRegistry, this.#podmanConnection)),
     );
+    if (os.arch() === 'x64') {
+      this.#extensionContext.subscriptions.push(
+        this.#inferenceProviderRegistry.register(
+          new OpenVINO(this.#taskRegistry, this.#podmanConnection, this.#modelsManager, this.#configurationRegistry),
+        ),
+      );
+    }
 
     /**
      * The inference manager create, stop, manage Inference servers

packages/backend/src/utils/modelsUtils.spec.ts

@@ -135,7 +135,7 @@ describe('getRemoteModelFile', () => {
       },
     } as unknown as ModelInfo);
 
-    expect(path).toBe(posix.join(MACHINE_BASE_FOLDER, 'dummy.guff'));
+    expect(path).toBe(posix.join(MACHINE_BASE_FOLDER, 'dummyModelId'));
   });
 });