From b7d38cf79c462a8cccaa143bf344edb0dc397f4c Mon Sep 17 00:00:00 2001 From: Evzen Gasta Date: Wed, 16 Apr 2025 16:55:37 +0200 Subject: [PATCH 1/7] feat: added option to select default interferencing runtime to preferences Signed-off-by: Evzen Gasta --- packages/backend/package.json | 10 ++++++++++ .../backend/src/registries/ConfigurationRegistry.ts | 4 +++- packages/frontend/src/lib/select/ModelSelect.svelte | 10 ++++++++++ packages/shared/src/models/IExtensionConfiguration.ts | 1 + 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 4b763bd8f..6eabc88f6 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -51,6 +51,16 @@ "maximum": 65535, "description": "Port on which the API is listening (requires restart of extension)" }, + "ai-lab.inferenceRuntime": { + "type": "string", + "default": "llama-cpp", + "enum": [ + "llama-cpp", + "whisper-cpp", + "none" + ], + "description": "Choose the default inferencing runtime for AI Lab" + }, "ai-lab.experimentalTuning": { "type": "boolean", "default": false, diff --git a/packages/backend/src/registries/ConfigurationRegistry.ts b/packages/backend/src/registries/ConfigurationRegistry.ts index 25d4ac057..1008e9ea1 100644 --- a/packages/backend/src/registries/ConfigurationRegistry.ts +++ b/packages/backend/src/registries/ConfigurationRegistry.ts @@ -26,6 +26,7 @@ const CONFIGURATION_SECTIONS: string[] = [ 'models.path', 'experimentalGPU', 'apiPort', + 'inferenceRuntime', 'experimentalTuning', 'modelUploadDisabled', 'showGPUPromotion', @@ -33,7 +34,7 @@ const CONFIGURATION_SECTIONS: string[] = [ ]; const API_PORT_DEFAULT = 10434; - +const INFERENCE_RUNTIME_DEFAULT = 'llama-cpp'; export class ConfigurationRegistry extends Publisher implements Disposable { #configuration: Configuration; #configurationPodmanDesktop: Configuration; @@ -54,6 +55,7 @@ export class ConfigurationRegistry extends Publisher imp modelsPath: this.getModelsPath(), experimentalGPU: this.#configuration.get('experimentalGPU') ?? false, apiPort: this.#configuration.get('apiPort') ?? API_PORT_DEFAULT, + inferenceRuntime: this.#configuration.get('inferenceRuntime') ?? INFERENCE_RUNTIME_DEFAULT, experimentalTuning: this.#configuration.get('experimentalTuning') ?? false, modelUploadDisabled: this.#configuration.get('modelUploadDisabled') ?? false, showGPUPromotion: this.#configuration.get('showGPUPromotion') ?? true, diff --git a/packages/frontend/src/lib/select/ModelSelect.svelte b/packages/frontend/src/lib/select/ModelSelect.svelte index 97483f3a3..9af62d1e7 100644 --- a/packages/frontend/src/lib/select/ModelSelect.svelte +++ b/packages/frontend/src/lib/select/ModelSelect.svelte @@ -3,6 +3,8 @@ import { faCheckCircle, faDownload } from '@fortawesome/free-solid-svg-icons'; import Select from './Select.svelte'; import Fa from 'svelte-fa'; import type { ModelInfo } from '@shared/models/IModelInfo'; +import { onMount } from 'svelte'; +import { configuration } from '/@/stores/extensionConfiguration'; interface Props { disabled?: boolean; @@ -44,6 +46,13 @@ let selected: (ModelInfo & { label: string; value: string }) | undefined = $deri function handleOnChange(nValue: (ModelInfo & { label: string; value: string }) | undefined): void { value = nValue; } + +let defaultRuntime: string = 'llama-cpp'; + +onMount(() => { + const inferenceRuntime = $configuration?.inferenceRuntime; + if (inferenceRuntime) defaultRuntime = inferenceRuntime; +}); { onchange={handleOnChange} placeholder="Select model to use" items={models - .filter(model => model.backend === defaultRuntime) + .filter(filterModel) .toSorted((a, b) => getModelSortingScore(a) - getModelSortingScore(b)) .map(model => ({ ...model, value: model.id, label: model.name }))}>
diff --git a/packages/frontend/src/pages/Recipes.svelte b/packages/frontend/src/pages/Recipes.svelte index dac45b0a0..61339e189 100644 --- a/packages/frontend/src/pages/Recipes.svelte +++ b/packages/frontend/src/pages/Recipes.svelte @@ -103,12 +103,12 @@ function openContribution(): void { studioClient.openURL('https://github.com/containers/ai-lab-recipes/blob/main/CONTRIBUTING.md').catch(console.error); } -let defaultRuntime: string = $state('llama-cpp'); +let defaultRuntime: string | undefined = $state(); onMount(() => { const inferenceRuntime = $configuration?.inferenceRuntime; if (inferenceRuntime) defaultRuntime = inferenceRuntime; - onFilterChange('tools', defaultRuntime); + onFilterChange('tools', defaultRuntime ?? 'all'); }); From f701a08950ae3e2d76bdcbe9fa7853819c6997d7 Mon Sep 17 00:00:00 2001 From: Evzen Gasta Date: Mon, 12 May 2025 15:50:30 +0200 Subject: [PATCH 4/7] chore: fixed tests Signed-off-by: Evzen Gasta --- packages/backend/src/workers/provider/OpenVINO.spec.ts | 1 + .../frontend/src/pages/instructlab/AboutInstructLab.spec.ts | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/packages/backend/src/workers/provider/OpenVINO.spec.ts b/packages/backend/src/workers/provider/OpenVINO.spec.ts index e6a56393c..eac1b1a49 100644 --- a/packages/backend/src/workers/provider/OpenVINO.spec.ts +++ b/packages/backend/src/workers/provider/OpenVINO.spec.ts @@ -95,6 +95,7 @@ beforeEach(() => { experimentalGPU: false, modelsPath: 'model-path', apiPort: 10434, + inferenceRuntime: 'llama-cpp', experimentalTuning: false, modelUploadDisabled: false, showGPUPromotion: false, diff --git a/packages/frontend/src/pages/instructlab/AboutInstructLab.spec.ts b/packages/frontend/src/pages/instructlab/AboutInstructLab.spec.ts index ba4d72008..d8363cc3c 100644 --- a/packages/frontend/src/pages/instructlab/AboutInstructLab.spec.ts +++ b/packages/frontend/src/pages/instructlab/AboutInstructLab.spec.ts @@ -59,6 +59,7 @@ const mockConfiguration: Writable = writable({ modelUploadDisabled: false, experimentalTuning: false, showGPUPromotion: false, + inferenceRuntime: 'llama-cpp', appearance: 'dark', }); @@ -75,6 +76,7 @@ test('renders Start Fine Tuning button if experimentalTuning is true', async () modelUploadDisabled: false, modelsPath: '', experimentalTuning: true, + inferenceRuntime: 'llama-cpp', apiPort: -1, appearance: 'dark', }); @@ -89,6 +91,7 @@ test('does not render Start Fine Tuning button if experimentalTuning is false', modelUploadDisabled: false, modelsPath: '', experimentalTuning: false, + inferenceRuntime: 'llama-cpp', apiPort: -1, appearance: 'dark', }); @@ -103,6 +106,7 @@ test('navigates to /tune/start when Start Fine Tuning is clicked', async () => { modelUploadDisabled: false, modelsPath: '', experimentalTuning: true, + inferenceRuntime: 'llama-cpp', apiPort: -1, appearance: 'dark', }); From 989601d343a3bc8730cf9fbfc4d8fb1088165a90 Mon Sep 17 00:00:00 2001 From: Evzen Gasta Date: Mon, 12 May 2025 15:51:55 +0200 Subject: [PATCH 5/7] chore: updated runtimes in preferences Signed-off-by: Evzen Gasta --- packages/backend/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index a870608c2..4edde8908 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -55,8 +55,8 @@ "type": "string", "enum": [ "llama-cpp", - "openvino", - "vllm" + "whisper-cpp", + "none" ], "description": "Choose the default inferencing runtime for AI Lab" }, From 15616c18f7bcc29056fd1223e85bf2c888dac209 Mon Sep 17 00:00:00 2001 From: Evzen Gasta Date: Wed, 14 May 2025 10:24:09 +0200 Subject: [PATCH 6/7] chore: added all option Signed-off-by: Evzen Gasta --- packages/backend/package.json | 1 + packages/backend/src/registries/ConfigurationRegistry.ts | 2 +- packages/frontend/src/lib/select/ModelSelect.svelte | 2 +- packages/frontend/src/pages/Recipes.svelte | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 4edde8908..e61604567 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -54,6 +54,7 @@ "ai-lab.inferenceRuntime": { "type": "string", "enum": [ + "all", "llama-cpp", "whisper-cpp", "none" diff --git a/packages/backend/src/registries/ConfigurationRegistry.ts b/packages/backend/src/registries/ConfigurationRegistry.ts index 534f3fdbc..e33ee60de 100644 --- a/packages/backend/src/registries/ConfigurationRegistry.ts +++ b/packages/backend/src/registries/ConfigurationRegistry.ts @@ -55,7 +55,7 @@ export class ConfigurationRegistry extends Publisher imp modelsPath: this.getModelsPath(), experimentalGPU: this.#configuration.get('experimentalGPU') ?? false, apiPort: this.#configuration.get('apiPort') ?? API_PORT_DEFAULT, - inferenceRuntime: this.#configuration.get('inferenceRuntime') ?? 'none', + inferenceRuntime: this.#configuration.get('inferenceRuntime') ?? 'all', experimentalTuning: this.#configuration.get('experimentalTuning') ?? false, modelUploadDisabled: this.#configuration.get('modelUploadDisabled') ?? false, showGPUPromotion: this.#configuration.get('showGPUPromotion') ?? true, diff --git a/packages/frontend/src/lib/select/ModelSelect.svelte b/packages/frontend/src/lib/select/ModelSelect.svelte index e98f66542..1e9eb8764 100644 --- a/packages/frontend/src/lib/select/ModelSelect.svelte +++ b/packages/frontend/src/lib/select/ModelSelect.svelte @@ -59,7 +59,7 @@ onMount(() => { function filterModel(model: ModelInfo): boolean { // If the defaultRuntime is undefined we should not filter any model - if (!defaultRuntime) return true; + if (!defaultRuntime || defaultRuntime === 'all') return true; return model.backend === defaultRuntime; } diff --git a/packages/frontend/src/pages/Recipes.svelte b/packages/frontend/src/pages/Recipes.svelte index 61339e189..2fb9da95e 100644 --- a/packages/frontend/src/pages/Recipes.svelte +++ b/packages/frontend/src/pages/Recipes.svelte @@ -108,7 +108,7 @@ let defaultRuntime: string | undefined = $state(); onMount(() => { const inferenceRuntime = $configuration?.inferenceRuntime; if (inferenceRuntime) defaultRuntime = inferenceRuntime; - onFilterChange('tools', defaultRuntime ?? 'all'); + if (inferenceRuntime !== 'all') onFilterChange('tools', defaultRuntime ?? ''); }); @@ -144,7 +144,7 @@ onMount(() => { onFilterChange(filterComponent.key, v)}>
From aa62f2913fe4bd4b62673f9f163ee929831d24ee Mon Sep 17 00:00:00 2001 From: Evzen Gasta Date: Thu, 15 May 2025 15:59:24 +0200 Subject: [PATCH 7/7] chore: removed redundant code Signed-off-by: Evzen Gasta --- .../src/lib/select/ModelSelect.spec.ts | 98 ------------------- .../src/lib/select/ModelSelect.svelte | 20 ---- 2 files changed, 118 deletions(-) diff --git a/packages/frontend/src/lib/select/ModelSelect.spec.ts b/packages/frontend/src/lib/select/ModelSelect.spec.ts index 22794afd5..5068af1bc 100644 --- a/packages/frontend/src/lib/select/ModelSelect.spec.ts +++ b/packages/frontend/src/lib/select/ModelSelect.spec.ts @@ -22,27 +22,6 @@ import { render, fireEvent, within } from '@testing-library/svelte'; import ModelSelect from '/@/lib/select/ModelSelect.svelte'; import type { ModelInfo } from '@shared/models/IModelInfo'; import { InferenceType } from '@shared/models/IInference'; -import { writable, type Writable } from 'svelte/store'; -import type { ExtensionConfiguration } from '@shared/models/IExtensionConfiguration'; -import { configuration } from '/@/stores/extensionConfiguration'; - -vi.mock('../../stores/extensionConfiguration', () => ({ - configuration: { - subscribe: vi.fn(), - unsubscribe: vi.fn(), - }, -})); - -const mockConfiguration: Writable = writable({ - inferenceRuntime: 'llama-cpp', - experimentalGPU: false, - showGPUPromotion: false, - modelUploadDisabled: false, - modelsPath: '', - experimentalTuning: false, - apiPort: -1, - appearance: 'dark', -}); const fakeRecommendedModel: ModelInfo = { id: 'dummy-model-1', @@ -66,39 +45,9 @@ const fakeRecommendedRemoteModel: ModelInfo = { name: 'Dummy Model 3', } as unknown as ModelInfo; -const fakeRemoteModelWhisper: ModelInfo = { - id: 'dummy-model-4', - backend: InferenceType.WHISPER_CPP, - name: 'Dummy Model 4', -} as unknown as ModelInfo; - -const fakeRemoteModelNone: ModelInfo = { - id: 'dummy-model-5', - backend: InferenceType.NONE, - name: 'Dummy Model 5', -} as unknown as ModelInfo; - -vi.mock('/@/utils/client', async () => { - return { - studioClient: { - updateExtensionConfiguration: vi.fn(), - telemetryLogUsage: vi.fn(), - }, - }; -}); - -vi.mock('../../stores/extensionConfiguration', () => ({ - configuration: { - subscribe: vi.fn(), - unsubscribe: vi.fn(), - }, -})); - beforeEach(() => { - vi.resetAllMocks(); // mock scrollIntoView window.HTMLElement.prototype.scrollIntoView = vi.fn(); - vi.mocked(configuration).subscribe.mockImplementation(run => mockConfiguration.subscribe(run)); }); test('ModelSelect should list all models provided', async () => { @@ -121,26 +70,6 @@ test('ModelSelect should list all models provided', async () => { expect(items[1]).toHaveTextContent(fakeRemoteModel.name); }); -test('ModelSelect should list all models based on selected runtime', async () => { - const { container } = render(ModelSelect, { - value: undefined, - disabled: undefined, - models: [fakeRecommendedModel, fakeRemoteModelWhisper, fakeRemoteModel, fakeRemoteModelNone], - recommended: [], - }); - - // first get the select input - const input = within(container).getByLabelText('Select Model'); - await fireEvent.pointerUp(input); // they are using the pointer up event instead of click. - - // get all options available - const items = container.querySelectorAll('div[class~="list-item"]'); - // ensure we have two options - expect(items.length).toBe(2); - expect(items[0]).toHaveTextContent(fakeRecommendedModel.name); - expect(items[1]).toHaveTextContent(fakeRemoteModel.name); -}); - test('ModelSelect should set star icon next to recommended model', async () => { const { container } = render(ModelSelect, { value: undefined, @@ -181,30 +110,3 @@ test('models should be sorted', async () => { expect(items[1]).toHaveTextContent(fakeRecommendedRemoteModel.name); expect(items[2]).toHaveTextContent(fakeRemoteModel.name); }); - -test('ModelSelect should filter out models based on selected default runtime', async () => { - const { container } = render(ModelSelect, { - value: undefined, - disabled: undefined, - models: [ - fakeRecommendedModel, - fakeRemoteModel, - fakeRemoteModelNone, - fakeRemoteModelWhisper, - fakeRecommendedRemoteModel, - ], - recommended: [], - }); - - // first get the select input - const input = within(container).getByLabelText('Select Model'); - await fireEvent.pointerUp(input); // they are using the pointer up event instead of click. - - // get all options available - const items = container.querySelectorAll('div[class~="list-item"]'); - // ensure we have two options - expect(items.length).toBe(3); - expect(items[0]).toHaveTextContent(fakeRecommendedModel.name); - expect(items[1]).toHaveTextContent(fakeRemoteModel.name); - expect(items[2]).toHaveTextContent(fakeRecommendedRemoteModel.name); -}); diff --git a/packages/frontend/src/lib/select/ModelSelect.svelte b/packages/frontend/src/lib/select/ModelSelect.svelte index 1e9eb8764..97483f3a3 100644 --- a/packages/frontend/src/lib/select/ModelSelect.svelte +++ b/packages/frontend/src/lib/select/ModelSelect.svelte @@ -3,8 +3,6 @@ import { faCheckCircle, faDownload } from '@fortawesome/free-solid-svg-icons'; import Select from './Select.svelte'; import Fa from 'svelte-fa'; import type { ModelInfo } from '@shared/models/IModelInfo'; -import { onMount } from 'svelte'; -import { configuration } from '/@/stores/extensionConfiguration'; interface Props { disabled?: boolean; @@ -46,23 +44,6 @@ let selected: (ModelInfo & { label: string; value: string }) | undefined = $deri function handleOnChange(nValue: (ModelInfo & { label: string; value: string }) | undefined): void { value = nValue; } - -let defaultRuntime: string | undefined = $state(); - -onMount(() => { - return configuration.subscribe(values => { - if (values?.inferenceRuntime) { - defaultRuntime = values.inferenceRuntime; - } - }); -}); - -function filterModel(model: ModelInfo): boolean { - // If the defaultRuntime is undefined we should not filter any model - if (!defaultRuntime || defaultRuntime === 'all') return true; - - return model.backend === defaultRuntime; -}