feat: added option to select default interferencing runtime to preferences

Signed-off-by: Evzen Gasta <[email protected]>

Author: gastoner

packages/backend/package.json

@@ -51,6 +51,16 @@
           "maximum": 65535,
           "description": "Port on which the API is listening (requires restart of extension)"
         },
+        "ai-lab.inferenceRuntime": {
+          "type": "string",
+          "default": "llama-cpp",
+          "enum": [
+            "llama-cpp",
+            "whisper-cpp",
+            "none"
+          ],
+          "description": "Choose the default inferencing runtime for AI Lab"
+        },
         "ai-lab.experimentalTuning": {
           "type": "boolean",
           "default": false,

packages/backend/src/registries/ConfigurationRegistry.ts

@@ -26,14 +26,15 @@ const CONFIGURATION_SECTIONS: string[] = [
   'models.path',
   'experimentalGPU',
   'apiPort',
+  'inferenceRuntime',
   'experimentalTuning',
   'modelUploadDisabled',
   'showGPUPromotion',
   'appearance',
 ];
 
 const API_PORT_DEFAULT = 10434;
-
+const INFERENCE_RUNTIME_DEFAULT = 'llama-cpp';
 export class ConfigurationRegistry extends Publisher<ExtensionConfiguration> implements Disposable {
   #configuration: Configuration;
   #configurationPodmanDesktop: Configuration;
@@ -54,6 +55,7 @@ export class ConfigurationRegistry extends Publisher<ExtensionConfiguration> imp
       modelsPath: this.getModelsPath(),
       experimentalGPU: this.#configuration.get<boolean>('experimentalGPU') ?? false,
       apiPort: this.#configuration.get<number>('apiPort') ?? API_PORT_DEFAULT,
+      inferenceRuntime: this.#configuration.get<string>('inferenceRuntime') ?? INFERENCE_RUNTIME_DEFAULT,
       experimentalTuning: this.#configuration.get<boolean>('experimentalTuning') ?? false,
       modelUploadDisabled: this.#configuration.get<boolean>('modelUploadDisabled') ?? false,
       showGPUPromotion: this.#configuration.get<boolean>('showGPUPromotion') ?? true,

packages/frontend/src/lib/select/ModelSelect.svelte

@@ -3,6 +3,8 @@ import { faCheckCircle, faDownload } from '@fortawesome/free-solid-svg-icons';
 import Select from './Select.svelte';
 import Fa from 'svelte-fa';
 import type { ModelInfo } from '@shared/models/IModelInfo';
+import { onMount } from 'svelte';
+import { configuration } from '/@/stores/extensionConfiguration';
 
 interface Props {
   disabled?: boolean;
@@ -44,6 +46,13 @@ let selected: (ModelInfo & { label: string; value: string }) | undefined = $deri
 function handleOnChange(nValue: (ModelInfo & { label: string; value: string }) | undefined): void {
   value = nValue;
 }
+
+let defaultRuntime: string = 'llama-cpp';
+
+onMount(() => {
+  const inferenceRuntime = $configuration?.inferenceRuntime;
+  if (inferenceRuntime) defaultRuntime = inferenceRuntime;
+});
 </script>
 
 <Select
@@ -54,6 +63,7 @@ function handleOnChange(nValue: (ModelInfo & { label: string; value: string }) |
   onchange={handleOnChange}
   placeholder="Select model to use"
   items={models
+    .filter(model => model.backend === defaultRuntime)
     .toSorted((a, b) => getModelSortingScore(a) - getModelSortingScore(b))
     .map(model => ({ ...model, value: model.id, label: model.name }))}>
   <div slot="item" let:item>

packages/shared/src/models/IExtensionConfiguration.ts

@@ -20,6 +20,7 @@ export interface ExtensionConfiguration {
   experimentalGPU: boolean;
   modelsPath: string;
   apiPort: number;
+  inferenceRuntime: string;
   experimentalTuning: boolean;
   modelUploadDisabled: boolean;
   showGPUPromotion: boolean;