From 3194a76dad917c88fdb72240ba926f1709a8703e Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 10 Jun 2025 01:03:25 -0400 Subject: [PATCH] feat: added ability to select model service for playground Signed-off-by: Brian --- .../src/managers/playgroundV2Manager.ts | 65 +++++++++++++++---- .../src/registries/ConversationRegistry.ts | 3 +- packages/backend/src/studio-api-impl.ts | 4 +- .../lib/select/InferenceRuntimeSelect.svelte | 30 +++++++++ .../src/lib/select/ModelServiceSelect.svelte | 49 ++++++++++++++ .../src/pages/PlaygroundCreate.svelte | 62 +++++++++++++----- packages/shared/src/StudioAPI.ts | 2 +- .../shared/src/models/IPlaygroundMessage.ts | 1 + 8 files changed, 181 insertions(+), 35 deletions(-) create mode 100644 packages/frontend/src/lib/select/InferenceRuntimeSelect.svelte create mode 100644 packages/frontend/src/lib/select/ModelServiceSelect.svelte diff --git a/packages/backend/src/managers/playgroundV2Manager.ts b/packages/backend/src/managers/playgroundV2Manager.ts index fc20fad02..1cb2c18ba 100644 --- a/packages/backend/src/managers/playgroundV2Manager.ts +++ b/packages/backend/src/managers/playgroundV2Manager.ts @@ -33,6 +33,7 @@ import { AiStreamProcessor } from './playground/aiSdk'; import { type McpServerManager } from './playground/McpServerManager'; import type { ToolSet } from 'ai'; import { simulateStreamingMiddleware, wrapLanguageModel } from 'ai'; +import type { InferenceServer } from '@shared/models/IInference'; export class PlaygroundV2Manager implements Disposable { readonly #conversationRegistry: ConversationRegistry; @@ -57,7 +58,7 @@ export class PlaygroundV2Manager implements Disposable { this.#conversationRegistry.deleteConversation(conversationId); } - async requestCreatePlayground(name: string, model: ModelInfo): Promise { + async requestCreatePlayground(name: string, model: ModelInfo, selectedService?: InferenceServer): Promise { const trackingId: string = getRandomString(); const task = this.taskRegistry.createTask('Creating Playground environment', 'loading', { trackingId: trackingId, @@ -67,7 +68,7 @@ export class PlaygroundV2Manager implements Disposable { hasName: !!name, modelId: getHash(model.id), }; - this.createPlayground(name, model, trackingId) + this.createPlayground(name, model, trackingId, selectedService) .then((playgroundId: string) => { this.taskRegistry.updateTask({ ...task, @@ -106,7 +107,12 @@ export class PlaygroundV2Manager implements Disposable { return trackingId; } - async createPlayground(name: string, model: ModelInfo, trackingId: string): Promise { + async createPlayground( + name: string, + model: ModelInfo, + trackingId: string, + selectedService?: InferenceServer, + ): Promise { if (!name) { name = this.getFreeName(); } @@ -114,14 +120,24 @@ export class PlaygroundV2Manager implements Disposable { throw new Error(`a playground with the name ${name} already exists`); } - // Create conversation - const conversationId = this.#conversationRegistry.createConversation(name, model.id); + let server: InferenceServer | undefined; + let containerId: string; - // create/start inference server if necessary - const servers = this.inferenceManager.getServers(); - const server = servers.find(s => s.models.map(mi => mi.id).includes(model.id)); - if (!server) { - await this.inferenceManager.createInferenceServer( + if (selectedService) { + // Use selected running service + server = this.inferenceManager + .getServers() + .find(s => s.container.containerId === selectedService.container.containerId); + if (!server) { + throw new Error(`Selected inference server ${selectedService.container.containerId} not found`); + } + if (server.status === 'stopped') { + await this.inferenceManager.startInferenceServer(server.container.containerId); + } + containerId = server.container.containerId; + } else { + // Create new inference server and get its containerId + containerId = await this.inferenceManager.createInferenceServer( await withDefaultConfiguration({ modelsInfo: [model], labels: { @@ -129,10 +145,13 @@ export class PlaygroundV2Manager implements Disposable { }, }), ); - } else if (server.status === 'stopped') { - await this.inferenceManager.startInferenceServer(server.container.containerId); + // Retrieve the server after creation + server = this.inferenceManager.getServers().find(s => s.container.containerId === containerId); + if (!server) { + throw new Error(`Inference server with containerId ${containerId} was not registered correctly.`); + } } - + const conversationId = this.#conversationRegistry.createConversation(name, model.id, containerId); return conversationId; } @@ -192,8 +211,26 @@ export class PlaygroundV2Manager implements Disposable { async submit(conversationId: string, userInput: string, options?: ModelOptions): Promise { const conversation = this.#conversationRegistry.get(conversationId); + console.log('conversation model id', conversation.modelId); + const servers = this.inferenceManager.getServers(); - const server = servers.find(s => s.models.map(mi => mi.id).includes(conversation.modelId)); + + console.log('Available Inference Services:'); + servers.forEach((server, index) => { + console.log(`--- Service ${index + 1} ---`); + console.log('Container ID:', server.container?.containerId); + console.log('Status:', server.status || 'unknown'); + console.log('Health:', server.health?.Status ?? 'unknown'); + console.log('Models:', server.models?.map(m => m.id).join(', ') || 'No models'); + console.log('Inference Type:', server.type || 'N/A'); + }); + + const server = this.inferenceManager.getServers().find(s => s.container.containerId === conversation.containerId); + + console.log('picked service', server?.container.containerId); + console.log('picked service status', server?.status); + console.log('picked service health', server?.health?.Status); + if (server === undefined) throw new Error('Inference server not found.'); if (server.status !== 'running') throw new Error('Inference server is not running.'); diff --git a/packages/backend/src/registries/ConversationRegistry.ts b/packages/backend/src/registries/ConversationRegistry.ts index 048643bd0..a754e5b81 100644 --- a/packages/backend/src/registries/ConversationRegistry.ts +++ b/packages/backend/src/registries/ConversationRegistry.ts @@ -82,11 +82,12 @@ export class ConversationRegistry extends Publisher implements D this.notify(); } - createConversation(name: string, modelId: string): string { + createConversation(name: string, modelId: string, containerId?: string): string { const conversationId = this.getUniqueId(); this.#conversations.set(conversationId, { name: name, modelId: modelId, + containerId: containerId, messages: [], id: conversationId, usage: { diff --git a/packages/backend/src/studio-api-impl.ts b/packages/backend/src/studio-api-impl.ts index 9616400ad..f0f0c246f 100644 --- a/packages/backend/src/studio-api-impl.ts +++ b/packages/backend/src/studio-api-impl.ts @@ -98,9 +98,9 @@ export class StudioApiImpl implements StudioAPI { }); } - async requestCreatePlayground(name: string, model: ModelInfo): Promise { + async requestCreatePlayground(name: string, model: ModelInfo, selectedService?: InferenceServer): Promise { try { - return await this.playgroundV2.requestCreatePlayground(name, model); + return await this.playgroundV2.requestCreatePlayground(name, model, selectedService); } catch (err: unknown) { console.error('Something went wrong while trying to create playground environment', err); throw err; diff --git a/packages/frontend/src/lib/select/InferenceRuntimeSelect.svelte b/packages/frontend/src/lib/select/InferenceRuntimeSelect.svelte new file mode 100644 index 000000000..79f0e14f2 --- /dev/null +++ b/packages/frontend/src/lib/select/InferenceRuntimeSelect.svelte @@ -0,0 +1,30 @@ + + + diff --git a/packages/frontend/src/pages/PlaygroundCreate.svelte b/packages/frontend/src/pages/PlaygroundCreate.svelte index 0c90d7263..6d1ad2818 100644 --- a/packages/frontend/src/pages/PlaygroundCreate.svelte +++ b/packages/frontend/src/pages/PlaygroundCreate.svelte @@ -13,15 +13,20 @@ import { filterByLabel } from '../utils/taskUtils'; import type { Unsubscriber } from 'svelte/store'; import { Button, ErrorMessage, FormPage, Input } from '@podman-desktop/ui-svelte'; import ModelSelect from '/@/lib/select/ModelSelect.svelte'; +import ModelServiceSelect from '/@/lib/select/ModelServiceSelect.svelte'; import { InferenceType } from '@shared/models/IInference'; +import type { InferenceServer } from '@shared/models/IInference'; +import { inferenceServers } from '/@/stores/inferenceServers'; let localModels: ModelInfo[]; +let ModelServices: InferenceServer[]; $: localModels = $modelsInfo.filter(model => model.file && model.backend !== InferenceType.WHISPER_CPP); $: availModels = $modelsInfo.filter(model => !model.file); let model: ModelInfo | undefined = undefined; let submitted: boolean = false; let playgroundName: string; let errorMsg: string | undefined = undefined; +let selectedService: InferenceServer | undefined; // The tracking id is a unique identifier provided by the // backend when calling requestCreateInferenceServer @@ -34,12 +39,22 @@ $: { if (!model && localModels.length > 0) { model = localModels[0]; } + ModelServices = model ? $inferenceServers.filter(server => server.models.some(m => m.id === model?.id)) : []; + if (ModelServices.length > 0) { + selectedService = ModelServices[0]; + } else { + selectedService = undefined; + } } function openModelsPage(): void { router.goto(`/models`); } +function openModelServicePage(): void { + router.goto(`/services`); +} + // Navigate to the new created playground environment const openPlaygroundPage = (playgroundId: string): void => { router.goto(`/playground/${playgroundId}`); @@ -56,10 +71,10 @@ async function submit(): Promise { submitted = true; try { // Using || and not && as we want to have the empty string systemPrompt passed as undefined - trackingId = await studioClient.requestCreatePlayground(playgroundName, model); + trackingId = await studioClient.requestCreatePlayground(playgroundName, model, selectedService); } catch (err: unknown) { trackingId = undefined; - console.error('Something wrong while trying to create the playground.', err); + console.error('Something wrong while tryinfghfghgfhfghgg to create the playground.', err); errorMsg = String(err); submitted = false; } @@ -122,19 +137,17 @@ export function goToUpPage(): void {
- {#if trackedTasks.length > 0}
{/if} -
- - + - - + + {#if localModels.length === 0}
- +
+ {:else if availModels.length > 0} +
+ +
+ Other models are available on the models page.
- {:else if availModels.length > 0} + {/if} + {#if ModelServices.length > 0} + + {#key ModelServices} + + {/key}
-
{/if}
+ {#if errorMsg !== undefined} {/if} +