feat: start a recipe with Llama Stack backend (#3180)

* feat: start a recipe with Llama Stack backend
Signed-off-by: Philippe Martin <[email protected]>

* fix: add a llamaStack dependency to pullRecipe and related
Signed-off-by: Philippe Martin <[email protected]>

Author: feloy

packages/backend/src/managers/application/applicationManager.spec.ts

@@ -31,6 +31,7 @@ import { VMType } from '@shared/models/IPodman';
 import { POD_LABEL_MODEL_ID, POD_LABEL_RECIPE_ID } from '../../utils/RecipeConstants';
 import type { InferenceServer } from '@shared/models/IInference';
 import type { RpcExtension } from '@shared/messages/MessageProxy';
+import type { LlamaStackManager } from '../llama-stack/llamaStackManager';
 import type { ApplicationOptions } from '../../models/ApplicationOptions';
 
 const taskRegistryMock = {
@@ -76,6 +77,10 @@ const recipeManager = {
   buildRecipe: vi.fn(),
 } as unknown as RecipeManager;
 
+const llamaStackManager = {
+  getLlamaStackContainer: vi.fn(),
+} as unknown as LlamaStackManager;
+
 vi.mock('@podman-desktop/api', () => ({
   window: {
     withProgress: vi.fn(),
@@ -140,6 +145,11 @@ beforeEach(() => {
     id: 'fake-task',
   }));
   vi.mocked(modelsManagerMock.uploadModelToPodmanMachine).mockResolvedValue('downloaded-model-path');
+  vi.mocked(llamaStackManager.getLlamaStackContainer).mockResolvedValue({
+    containerId: 'container1',
+    port: 10001,
+    playgroundPort: 10002,
+  });
 });
 
 function getInitializedApplicationManager(): ApplicationManager {
@@ -152,6 +162,7 @@ function getInitializedApplicationManager(): ApplicationManager {
     telemetryMock,
     podManager,
     recipeManager,
+    llamaStackManager,
   );
 
   manager.init();
@@ -303,6 +314,9 @@ describe.each([true, false])('pullApplication, with model is %o', withModel => {
       : {
           connection: connectionMock,
           recipe: recipeMock,
+          dependencies: {
+            llamaStack: true,
+          },
         };
   });
 
@@ -336,6 +350,7 @@ describe.each([true, false])('pullApplication, with model is %o', withModel => {
         connection: connectionMock,
         recipe: recipeMock,
         model: withModel ? remoteModelMock : undefined,
+        dependencies: applicationOptions.dependencies,
       },
       {
         'test-label': 'test-value',
@@ -364,7 +379,7 @@ describe.each([true, false])('pullApplication, with model is %o', withModel => {
     expect(containerEngine.createContainer).toHaveBeenCalledWith('test-engine-id', {
       Image: recipeImageInfoMock.id,
       name: expect.any(String),
-      Env: [],
+      Env: withModel ? [] : ['MODEL_ENDPOINT=http://host.containers.internal:10001'],
       HealthCheck: undefined,
       HostConfig: undefined,
       Detach: true,
@@ -411,6 +426,7 @@ describe.each([true, false])('pullApplication, with model is %o', withModel => {
         connection: connectionMock,
         recipe: recipeMock,
         model: withModel ? remoteModelMock : undefined,
+        dependencies: applicationOptions.dependencies,
       },
       {
         'test-label': 'test-value',
@@ -439,7 +455,9 @@ describe.each([true, false])('pullApplication, with model is %o', withModel => {
     expect(containerEngine.createContainer).toHaveBeenCalledWith('test-engine-id', {
       Image: recipeImageInfoMock.id,
       name: expect.any(String),
-      Env: withModel ? ['MODEL_ENDPOINT=http://host.containers.internal:56001'] : [],
+      Env: withModel
+        ? ['MODEL_ENDPOINT=http://host.containers.internal:56001']
+        : ['MODEL_ENDPOINT=http://host.containers.internal:10001'],
       HealthCheck: undefined,
       HostConfig: undefined,
       Detach: true,

packages/backend/src/managers/application/applicationManager.ts

@@ -54,6 +54,7 @@ import { RECIPE_START_ROUTE } from '../../registries/NavigationRegistry';
 import type { RpcExtension } from '@shared/messages/MessageProxy';
 import { TaskRunner } from '../TaskRunner';
 import { getInferenceType } from '../../utils/inferenceUtils';
+import type { LlamaStackManager } from '../llama-stack/llamaStackManager';
 import { isApplicationOptionsWithModelInference, type ApplicationOptions } from '../../models/ApplicationOptions';
 
 export class ApplicationManager extends Publisher<ApplicationState[]> implements Disposable {
@@ -71,6 +72,7 @@ export class ApplicationManager extends Publisher<ApplicationState[]> implements
     private telemetry: TelemetryLogger,
     private podManager: PodManager,
     private recipeManager: RecipeManager,
+    private llamaStackManager: LlamaStackManager,
   ) {
     super(rpcExtension, MSG_APPLICATIONS_STATE_UPDATE, () => this.getApplicationsState());
     this.#applications = new ApplicationRegistry<ApplicationState>();
@@ -282,7 +284,7 @@ export class ApplicationManager extends Publisher<ApplicationState[]> implements
           ...labels,
           'pod-id': podInfo.Id,
         }));
-        await this.createContainerAndAttachToPod(options, podInfo, components, modelPath);
+        await this.createContainerAndAttachToPod(options, podInfo, components, modelPath, labels);
         return podInfo;
       },
     );
@@ -293,6 +295,7 @@ export class ApplicationManager extends Publisher<ApplicationState[]> implements
     podInfo: PodInfo,
     components: RecipeComponents,
     modelPath: string | undefined,
+    labels?: { [key: string]: string },
   ): Promise<void> {
     const vmType = options.connection.vmType ?? VMType.UNKNOWN;
     // temporary check to set Z flag or not - to be removed when switching to podman 5
@@ -327,6 +330,15 @@ export class ApplicationManager extends Publisher<ApplicationState[]> implements
               envs = [`MODEL_ENDPOINT=${endPoint}`];
             }
           }
+        } else if (options.dependencies?.llamaStack) {
+          let stack = await this.llamaStackManager.getLlamaStackContainer();
+          if (!stack) {
+            await this.llamaStackManager.createLlamaStackContainer(options.connection, labels ?? {});
+            stack = await this.llamaStackManager.getLlamaStackContainer();
+          }
+          if (stack) {
+            envs = [`MODEL_ENDPOINT=http://host.containers.internal:${stack.port}`];
+          }
         }
         if (image.ports.length > 0) {
           healthcheck = {

packages/backend/src/models/ApplicationOptions.ts

@@ -18,13 +18,14 @@
 
 import type { ContainerProviderConnection } from '@podman-desktop/api';
 import type { ModelInfo } from '@shared/models/IModelInfo';
-import type { Recipe } from '@shared/models/IRecipe';
+import type { Recipe, RecipeDependencies } from '@shared/models/IRecipe';
 
 export type ApplicationOptions = ApplicationOptionsDefault | ApplicationOptionsWithModelInference;
 
 export interface ApplicationOptionsDefault {
   connection: ContainerProviderConnection;
   recipe: Recipe;
+  dependencies?: RecipeDependencies;
 }
 
 export type ApplicationOptionsWithModelInference = ApplicationOptionsDefault & {

packages/backend/src/studio-api-impl.ts

@@ -246,12 +246,14 @@ export class StudioApiImpl implements StudioAPI {
       opts = {
         connection,
         recipe,
+        dependencies: options.dependencies,
         model,
       };
     } else {
       opts = {
         connection,
         recipe,
+        dependencies: options.dependencies,
       };
     }
 

packages/backend/src/studio.ts

@@ -358,6 +358,7 @@ export class Studio {
       this.#telemetry,
       this.#podManager,
       this.#recipeManager,
+      this.#llamaStackManager,
     );
     this.#applicationManager.init();
     this.#extensionContext.subscriptions.push(this.#applicationManager);

packages/frontend/src/lib/RecipeCardTags.ts

@@ -22,7 +22,7 @@ import { gte } from 'semver';
 const USE_CASES = ['natural-language-processing', 'audio', 'computer-vision'];
 const LANGUAGES = ['java', 'javascript', 'python'];
 export const FRAMEWORKS = ['langchain', 'langchain4j', 'quarkus', 'react', 'streamlit', 'vectordb'];
-export const TOOLS = ['none', 'llama-cpp', 'whisper-cpp'];
+export const TOOLS = ['none', 'llama-cpp', 'whisper-cpp', 'llama-stack'];
 
 // Defaulting to Podman Desktop min version we need to run
 let version: string = '1.8.0';

packages/frontend/src/pages/StartRecipe.spec.ts

@@ -69,6 +69,14 @@ const fakeRecipe: Recipe = {
   categories: [],
 } as unknown as Recipe;
 
+const fakeLlamaStackRecipe: Recipe = {
+  id: 'dummy-llama-stack-recipe-id',
+  backend: 'llama-stack',
+  name: 'Dummy Llama Stack Recipe',
+  description: 'Dummy description',
+  categories: [],
+} as unknown as Recipe;
+
 const fakeRecommendedModel: ModelInfo = {
   id: 'dummy-model-1',
   backend: InferenceType.LLAMA_CPP,
@@ -100,7 +108,7 @@ beforeEach(() => {
   router.location.query.clear();
 
   vi.mocked(CatalogStore).catalog = readable<ApplicationCatalog>({
-    recipes: [fakeRecipe],
+    recipes: [fakeRecipe, fakeLlamaStackRecipe],
     models: [],
     categories: [],
     version: '',
@@ -147,7 +155,7 @@ test('Recipe Local Repository should be visible when defined', async () => {
   expect(span.textContent).toBe('dummy-recipe-path');
 });
 
-test('Submit button should be disabled when no model is selected', async () => {
+test('Submit button should be disabled when model is required and no model is selected', async () => {
   vi.mocked(ModelsInfoStore).modelsInfo = readable([]);
 
   render(StartRecipe, {
@@ -159,6 +167,18 @@ test('Submit button should be disabled when no model is selected', async () => {
   expect(button).toBeDisabled();
 });
 
+test('Submit button should be enabled when model is not required', async () => {
+  vi.mocked(ModelsInfoStore).modelsInfo = readable([]);
+
+  render(StartRecipe, {
+    recipeId: 'dummy-llama-stack-recipe-id',
+  });
+
+  const button = screen.getByTitle(`Start ${fakeLlamaStackRecipe.name} recipe`);
+  expect(button).toBeDefined();
+  expect(button).toBeEnabled();
+});
+
 test('First recommended model should be selected as default model', async () => {
   const { container } = render(StartRecipe, {
     recipeId: 'dummy-recipe-id',
@@ -265,6 +285,29 @@ test('Submit button should call requestPullApplication with proper arguments', a
       connection: containerProviderConnection,
       recipeId: fakeRecipe.id,
       modelId: fakeRecommendedModel.id,
+      dependencies: {
+        llamaStack: false,
+      },
+    });
+  });
+});
+
+test('Submit button should call requestPullApplication with proper arguments for llama-stack recipe', async () => {
+  render(StartRecipe, {
+    recipeId: 'dummy-llama-stack-recipe-id',
+  });
+
+  const button = screen.getByTitle(`Start ${fakeLlamaStackRecipe.name} recipe`);
+  expect(button).toBeEnabled();
+  await fireEvent.click(button);
+
+  await vi.waitFor(() => {
+    expect(studioClient.requestPullApplication).toHaveBeenCalledWith({
+      connection: containerProviderConnection,
+      recipeId: fakeLlamaStackRecipe.id,
+      dependencies: {
+        llamaStack: true,
+      },
     });
   });
 });

packages/frontend/src/pages/StartRecipe.svelte

@@ -2,7 +2,7 @@
 import { faFolder, faRocket, faUpRightFromSquare, faWarning } from '@fortawesome/free-solid-svg-icons';
 import { catalog } from '/@/stores/catalog';
 import Fa from 'svelte-fa';
-import type { Recipe } from '@shared/models/IRecipe';
+import type { Recipe, RecipePullOptions, RecipePullOptionsWithModelInference } from '@shared/models/IRecipe';
 import type { LocalRepository } from '@shared/models/ILocalRepository';
 import { findLocalRepositoryByRecipeId } from '/@/utils/localRepositoriesUtils';
 import { localRepositories } from '/@/stores/localRepositories';
@@ -53,6 +53,16 @@ let completed: boolean = $state(false);
 
 let errorMsg: string | undefined = $state(undefined);
 
+let formValid = $derived.by<boolean>((): boolean => {
+  if (!recipe) {
+    return false;
+  }
+  if (!isModelNeeded(recipe)) {
+    return true;
+  }
+  return !!model;
+});
+
 $effect(() => {
   // Select default connection
   if (!containerProviderConnection && startedContainerProviderConnectionInfo.length > 0) {
@@ -100,16 +110,22 @@ function populateModelFromTasks(trackedTasks: Task[]): void {
 }
 
 async function submit(): Promise<void> {
-  if (!recipe || !model) return;
+  if (!recipe || !formValid) return;
 
   errorMsg = undefined;
 
   try {
-    const trackingId = await studioClient.requestPullApplication({
+    const options: RecipePullOptions = {
       recipeId: $state.snapshot(recipe.id),
-      modelId: $state.snapshot(model.id),
       connection: $state.snapshot(containerProviderConnection),
-    });
+      dependencies: {
+        llamaStack: recipe.backend === 'llama-stack',
+      },
+    };
+    if (model) {
+      (options as RecipePullOptionsWithModelInference).modelId = $state.snapshot(model.id);
+    }
+    const trackingId = await studioClient.requestPullApplication(options);
     router.location.query.set('trackingId', trackingId);
   } catch (err: unknown) {
     console.error('Something wrong while trying to create the inference server.', err);
@@ -124,6 +140,10 @@ export function goToUpPage(): void {
 function handleOnClick(): void {
   router.goto(`/recipe/${recipeId}/running`);
 }
+
+function isModelNeeded(recipe: Recipe): boolean {
+  return recipe.backend !== 'llama-stack';
+}
 </script>
 
 <FormPage
@@ -183,17 +203,18 @@ function handleOnClick(): void {
                 bind:value={containerProviderConnection}
                 containerProviderConnections={startedContainerProviderConnectionInfo} />
             {/if}
-
-            <!-- model form -->
-            <label for="select-model" class="pt-4 block mb-2 font-bold text-[var(--pd-content-card-header-text)]"
-              >Model</label>
-            <ModelSelect bind:value={model} disabled={loading} recommended={recipe.recommended} models={models} />
-            {#if model && model.file === undefined}
-              <div class="text-gray-800 text-sm flex items-center">
-                <Fa class="mr-2" icon={faWarning} />
-                <span role="alert"
-                  >The selected model will be downloaded. This action can take some time depending on your connection</span>
-              </div>
+            {#if isModelNeeded(recipe)}
+              <!-- model form -->
+              <label for="select-model" class="pt-4 block mb-2 font-bold text-[var(--pd-content-card-header-text)]"
+                >Model</label>
+              <ModelSelect bind:value={model} disabled={loading} recommended={recipe.recommended} models={models} />
+              {#if model && model.file === undefined}
+                <div class="text-gray-800 text-sm flex items-center">
+                  <Fa class="mr-2" icon={faWarning} />
+                  <span role="alert"
+                    >The selected model will be downloaded. This action can take some time depending on your connection</span>
+                </div>
+              {/if}
             {/if}
           </div>
 
@@ -209,7 +230,7 @@ function handleOnClick(): void {
                   title="Start {recipe.name} recipe"
                   inProgress={loading}
                   on:click={submit}
-                  disabled={!model || loading || !containerProviderConnection}
+                  disabled={!formValid || loading || !containerProviderConnection}
                   icon={faRocket}>
                   Start {recipe.name} recipe
                 </Button>

packages/shared/src/models/IRecipe.ts

@@ -24,12 +24,17 @@ export type RecipePullOptions = RecipePullOptionsDefault | RecipePullOptionsWith
 export interface RecipePullOptionsDefault {
   connection?: ContainerProviderConnectionInfo;
   recipeId: string;
+  dependencies?: RecipeDependencies;
 }
 
 export type RecipePullOptionsWithModelInference = RecipePullOptionsDefault & {
   modelId: string;
 };
 
+export interface RecipeDependencies {
+  llamaStack?: boolean;
+}
+
 export function isRecipePullOptionsWithModelInference(
   options: RecipePullOptions,
 ): options is RecipePullOptionsWithModelInference {