diff --git a/PACKAGING-GUIDE.md b/PACKAGING-GUIDE.md index 5c338f108..f4686a016 100644 --- a/PACKAGING-GUIDE.md +++ b/PACKAGING-GUIDE.md @@ -33,15 +33,16 @@ The catalog also lists the models that may be associated to recipes. A model is citizen in AI Lab as they will be listed in the Models page and can be tested through the playground. A model has the following attributes: -- ```id```: a unique identifier for the model -- ```name```: the model name -- ```description```: a detailed description about the model -- ```registry```: the model registry where the model is stored -- ```popularity```: an integer field giving the rating of the model. Can be thought as the number of stars -- ```license```: the license under which the model is available -- ```url```: the URL used to download the model -- ```memory```: the memory footprint of the model in bytes, as computed by the workflow `.github/workflows/compute-model-sizes.yaml` -- ```sha256```: the SHA-256 checksum to be used to verify the downloaded model is identical to the original. It is optional and it must be HEX encoded + +- `id`: a unique identifier for the model +- `name`: the model name +- `description`: a detailed description about the model +- `registry`: the model registry where the model is stored +- `popularity`: an integer field giving the rating of the model. Can be thought as the number of stars +- `license`: the license under which the model is available +- `url`: the URL used to download the model +- `memory`: the memory footprint of the model in bytes, as computed by the workflow `.github/workflows/compute-model-sizes.yaml` +- `sha256`: the SHA-256 checksum to be used to verify the downloaded model is identical to the original. It is optional and it must be HEX encoded #### Recipes @@ -49,40 +50,44 @@ A recipe is a sample AI application that is packaged as one or several container source code and AI Lab will make sure the container images are built prior to launching the containers. A recipe has the following attributes: -- ```id```: a unique identifier to the recipe -- ```name```: the recipe name -- ```description```: a detailed description about the recipe -- ```repository```: the URL where the recipe code can be retrieved -- ```ref```: an optional ref in the repository to checkout (a branch name, tag name, or commit full id - short commit id won't be recognized). If not defined, the default branch will be used -- ```categories```: an array of category id to be associated by this recipe -- ```basedir```: an optional path within the repository where the ai-lab.yaml file is located. If not provided, the ai-lab.yaml is assumed to be located at the root the repository -- ```readme```: a markdown description of the recipe -- ```models```: an array of model id to be associated with this recipe + +- `id`: a unique identifier to the recipe +- `name`: the recipe name +- `description`: a detailed description about the recipe +- `repository`: the URL where the recipe code can be retrieved +- `ref`: an optional ref in the repository to checkout (a branch name, tag name, or commit full id - short commit id won't be recognized). If not defined, the default branch will be used +- `categories`: an array of category id to be associated by this recipe +- `basedir`: an optional path within the repository where the ai-lab.yaml file is located. If not provided, the ai-lab.yaml is assumed to be located at the root the repository +- `readme`: a markdown description of the recipe +- `models`: an array of model id to be associated with this recipe +- `backends`: an array of backends from which models may be associated with this recipe. The backends are used to filter models when associating them to a recipe. #### Recipe configuration file -The configuration file is called ```ai-lab.yaml``` and follows the following syntax. +The configuration file is called `ai-lab.yaml` and follows the following syntax. -The root elements are called ```version``` and ```application```. +The root elements are called `version` and `application`. -```version``` represents the version of the specifications that ai-lab adheres to (so far, the only accepted value here is `v1.0`). +`version` represents the version of the specifications that ai-lab adheres to (so far, the only accepted value here is `v1.0`). -```application``` contains an attribute called ```containers``` whose syntax is an array of objects containing the following attributes: -- ```name```: the name of the container -- ```contextdir```: the context directory used to build the container. -- ```containerfile```: the containerfile used to build the image -- ```model-service```: a boolean flag used to indicate if the container is running the model or not -- ```arch```: an optional array of architecture for which this image is compatible with. The values follow the -[GOARCH specification](https://go.dev/src/go/build/syslist.go) -- ```gpu-env```: an optional array of GPU environment for which this image is compatible with. The only accepted value here is cuda. -- ```ports```: an optional array of ports for which the application listens to. +`application` contains an attribute called `containers` whose syntax is an array of objects containing the following attributes: + +- `name`: the name of the container +- `contextdir`: the context directory used to build the container. +- `containerfile`: the containerfile used to build the image +- `model-service`: a boolean flag used to indicate if the container is running the model or not +- `arch`: an optional array of architecture for which this image is compatible with. The values follow the + [GOARCH specification](https://go.dev/src/go/build/syslist.go) +- `gpu-env`: an optional array of GPU environment for which this image is compatible with. The only accepted value here is cuda. +- `ports`: an optional array of ports for which the application listens to. - `image`: an optional image name to be used when building the container image. -The container that is running the service (having the ```model-service``` flag equal to ```true```) can use at runtime -the model managed by AI Lab through an environment variable ```MODEL_PATH``` whose value is the full path name of the +The container that is running the service (having the `model-service` flag equal to `true`) can use at runtime +the model managed by AI Lab through an environment variable `MODEL_PATH` whose value is the full path name of the model file. Below is given an example of such a configuration file: + ```yaml application: containers: diff --git a/packages/backend/src/assets/ai.json b/packages/backend/src/assets/ai.json index 44a4b214f..693648a61 100644 --- a/packages/backend/src/assets/ai.json +++ b/packages/backend/src/assets/ai.json @@ -19,7 +19,7 @@ "hf.instructlab.merlinite-7b-lab-GGUF", "hf.lmstudio-community.granite-3.0-8b-instruct-GGUF" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["python"], "frameworks": ["streamlit", "langchain"] }, @@ -34,7 +34,7 @@ "basedir": "recipes/natural_language_processing/chatbot-pydantic-ai", "readme": "# Chatbot Pydantic Application\n\n This recipe helps developers start building their own custom LLM enabled chat applications. It consists of two main components: the Model Service and the AI Application.\n\n There are a few options today for local Model Serving, but this recipe will use [`llama-cpp-python`](https://github.com/abetlen/llama-cpp-python) and their OpenAI compatible Model Service. There is a Containerfile provided that can be used to build this Model Service within the repo, [`model_servers/llamacpp_python/base/Containerfile`](/model_servers/llamacpp_python/base/Containerfile).\n\n The AI Application will connect to the Model Service via its OpenAI compatible API. The recipe relies on [Langchain's](https://python.langchain.com/docs/get_started/introduction) python package to simplify communication with the Model Service and uses [Streamlit](https://streamlit.io/) for the UI layer. You can find an example of the chat application below.\n\n![](/assets/chatbot_ui.png) \n\n\n## Try the Chat Application\n\nThe [Podman Desktop](https://podman-desktop.io) [AI Lab Extension](https://github.com/containers/podman-desktop-extension-ai-lab) includes this recipe among others. To try it out, open `Recipes Catalog` -> `Chatbot Pydantic AI` and follow the instructions to start the application.\n\n# Build the Application\n\nThe rest of this document will explain how to build and run the application from the terminal, and will\ngo into greater detail on how each container in the Pod above is built, run, and \nwhat purpose it serves in the overall application. All the recipes use a central [Makefile](../../common/Makefile.common) that includes variables populated with default values to simplify getting started. Please review the [Makefile docs](../../common/README.md), to learn about further customizing your application.\n\n\nThis application requires a model, a model service and an AI inferencing application.\n\n* [Quickstart](#quickstart)\n* [Download a model](#download-a-model)\n* [Build the Model Service](#build-the-model-service)\n* [Deploy the Model Service](#deploy-the-model-service)\n* [Build the AI Application](#build-the-ai-application)\n* [Deploy the AI Application](#deploy-the-ai-application)\n* [Interact with the AI Application](#interact-with-the-ai-application)\n* [Embed the AI Application in a Bootable Container Image](#embed-the-ai-application-in-a-bootable-container-image)\n\n\n## Quickstart\nTo run the application with pre-built images from `quay.io/ai-lab`, use `make quadlet`. This command\nbuilds the application's metadata and generates Kubernetes YAML at `./build/chatbot-pydantic-ai.yaml` to spin up a Pod that can then be launched locally.\nTry it with:\n\n```\nmake quadlet\npodman kube play build/chatbot-pydantic-ai.yaml\n```\n\nThis will take a few minutes if the model and model-server container images need to be downloaded. \nThe Pod is named `chatbot-pydantic-ai`, so you may use [Podman](https://podman.io) to manage the Pod and its containers:\n\n```\npodman pod list\npodman ps\n```\n\nOnce the Pod and its containers are running, the application can be accessed at `http://localhost:8501`. \nPlease refer to the section below for more details about [interacting with the chatbot-pydantic-ai application](#interact-with-the-ai-application).\n\nTo stop and remove the Pod, run:\n\n```\npodman pod stop chatbot-pydantic-ai\npodman pod rm chatbot-pydantic-ai\n```\n\n## Download a model\n\nIf you are just getting started, we recommend using [granite-7b-lab](https://huggingface.co/instructlab/granite-7b-lab). This is a well\nperformant mid-sized model with an apache-2.0 license. In order to use it with our Model Service we need it converted\nand quantized into the [GGUF format](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md). There are a number of\nways to get a GGUF version of granite-7b-lab, but the simplest is to download a pre-converted one from\n[huggingface.co](https://huggingface.co) here: https://huggingface.co/instructlab/granite-7b-lab-GGUF.\n\nThe recommended model can be downloaded using the code snippet below:\n\n```bash\ncd ../../../models\ncurl -sLO https://huggingface.co/instructlab/granite-7b-lab-GGUF/resolve/main/granite-7b-lab-Q4_K_M.gguf\ncd ../recipes/natural_language_processing/chatbot-pydantic-ai\n```\n\n_A full list of supported open models is forthcoming._ \n\n\n## Build the Model Service\n\nThe complete instructions for building and deploying the Model Service can be found in the\n[llamacpp_python model-service document](../../../model_servers/llamacpp_python/README.md).\n\nThe Model Service can be built from make commands from the [llamacpp_python directory](../../../model_servers/llamacpp_python/).\n\n```bash\n# from path model_servers/llamacpp_python from repo containers/ai-lab-recipes\nmake build\n```\nCheckout the [Makefile](../../../model_servers/llamacpp_python/Makefile) to get more details on different options for how to build.\n\n## Deploy the Model Service\n\nThe local Model Service relies on a volume mount to the localhost to access the model files. It also employs environment variables to dictate the model used and where its served. You can start your local Model Service using the following `make` command from `model_servers/llamacpp_python` set with reasonable defaults:\n\n```bash\n# from path model_servers/llamacpp_python from repo containers/ai-lab-recipes\nmake run\n```\n\n## Build the AI Application\n\nThe AI Application can be built from the make command:\n\n```bash\n# Run this from the current directory (path recipes/natural_language_processing/chatbot-pydantic-ai from repo containers/ai-lab-recipes)\nmake build\n```\n\n## Deploy the AI Application\n\nMake sure the Model Service is up and running before starting this container image. When starting the AI Application container image we need to direct it to the correct `MODEL_ENDPOINT`. This could be any appropriately hosted Model Service (running locally or in the cloud) using an OpenAI compatible API. In our case the Model Service is running inside the Podman machine so we need to provide it with the appropriate address `10.88.0.1`. To deploy the AI application use the following:\n\n```bash\n# Run this from the current directory (path recipes/natural_language_processing/chatbot-pydantic-ai from repo containers/ai-lab-recipes)\nmake run \n```\n\n## Interact with the AI Application\n\nEverything should now be up an running with the chat application available at [`http://localhost:8501`](http://localhost:8501). By using this recipe and getting this starting point established, users should now have an easier time customizing and building their own LLM enabled chatbot-pydantic-ai applications. \n\n## Embed the AI Application in a Bootable Container Image\n\nTo build a bootable container image that includes this sample chatbot-pydantic-ai workload as a service that starts when a system is booted, run: `make -f Makefile bootc`. You can optionally override the default image / tag you want to give the make command by specifying it as follows: `make -f Makefile BOOTC_IMAGE= bootc`.\n\nSubstituting the bootc/Containerfile FROM command is simple using the Makefile FROM option.\n\n```bash\nmake FROM=registry.redhat.io/rhel9/rhel-bootc:9.4 bootc\n```\n\nSelecting the ARCH for the bootc/Containerfile is simple using the Makefile ARCH= variable.\n\n```\nmake ARCH=x86_64 bootc\n```\n\nThe magic happens when you have a bootc enabled system running. If you do, and you'd like to update the operating system to the OS you just built\nwith the chatbot-pydantic-ai application, it's as simple as ssh-ing into the bootc system and running:\n\n```bash\nbootc switch quay.io/ai-lab/chatbot-pydantic-ai-bootc:latest\n```\n\nUpon a reboot, you'll see that the chatbot-pydantic-ai service is running on the system. Check on the service with:\n\n```bash\nssh user@bootc-system-ip\nsudo systemctl status chatbot-pydantic-ai\n```\n\n### What are bootable containers?\n\nWhat's a [bootable OCI container](https://containers.github.io/bootc/) and what's it got to do with AI?\n\nThat's a good question! We think it's a good idea to embed AI workloads (or any workload!) into bootable images at _build time_ rather than\nat _runtime_. This extends the benefits, such as portability and predictability, that containerizing applications provides to the operating system.\nBootable OCI images bake exactly what you need to run your workloads into the operating system at build time by using your favorite containerization\ntools. Might I suggest [podman](https://podman.io/)?\n\nOnce installed, a bootc enabled system can be updated by providing an updated bootable OCI image from any OCI\nimage registry with a single `bootc` command. This works especially well for fleets of devices that have fixed workloads - think\nfactories or appliances. Who doesn't want to add a little AI to their appliance, am I right?\n\nBootable images lend toward immutable operating systems, and the more immutable an operating system is, the less that can go wrong at runtime!\n\n#### Creating bootable disk images\n\nYou can convert a bootc image to a bootable disk image using the\n[quay.io/centos-bootc/bootc-image-builder](https://github.com/osbuild/bootc-image-builder) container image.\n\nThis container image allows you to build and deploy [multiple disk image types](../../common/README_bootc_image_builder.md) from bootc container images.\n\nDefault image types can be set via the DISK_TYPE Makefile variable.\n\n`make bootc-image-builder DISK_TYPE=ami`\n", "recommended": ["hf.MaziyarPanahi.Mistral-7B-Instruct-v0.3.Q4_K_M"], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["python"], "frameworks": ["streamlit", "PydanticAI"] }, @@ -56,7 +56,7 @@ "hf.instructlab.merlinite-7b-lab-GGUF", "hf.lmstudio-community.granite-3.0-8b-instruct-GGUF" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["python"], "frameworks": ["streamlit", "langchain"] }, @@ -78,7 +78,7 @@ "hf.instructlab.merlinite-7b-lab-GGUF", "hf.lmstudio-community.granite-3.0-8b-instruct-GGUF" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["python"], "frameworks": ["streamlit", "langchain"] }, @@ -100,7 +100,7 @@ "hf.TheBloke.mistral-7b-code-16k-qlora.Q4_K_M", "hf.TheBloke.mistral-7b-codealpaca-lora.Q4_K_M" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["python"], "frameworks": ["streamlit", "langchain"] }, @@ -122,7 +122,7 @@ "hf.instructlab.merlinite-7b-lab-GGUF", "hf.lmstudio-community.granite-3.0-8b-instruct-GGUF" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["python"], "frameworks": ["streamlit", "langchain", "vectordb"] }, @@ -144,7 +144,7 @@ "hf.instructlab.merlinite-7b-lab-GGUF", "hf.lmstudio-community.granite-3.0-8b-instruct-GGUF" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["javascript"], "frameworks": ["react", "langchain", "vectordb"] }, @@ -166,7 +166,7 @@ "hf.instructlab.merlinite-7b-lab-GGUF", "hf.lmstudio-community.granite-3.0-8b-instruct-GGUF" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["java"], "frameworks": ["quarkus", "langchain4j"] }, @@ -188,7 +188,7 @@ "hf.instructlab.merlinite-7b-lab-GGUF", "hf.lmstudio-community.granite-3.0-8b-instruct-GGUF" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["javascript"], "frameworks": ["react", "langchain"] }, @@ -206,7 +206,7 @@ "hf.ibm-granite.granite-3.3-8b-instruct-GGUF", "hf.MaziyarPanahi.Mistral-7B-Instruct-v0.3.Q4_K_M" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["python"], "frameworks": ["streamlit", "langchain"] }, @@ -224,7 +224,7 @@ "hf.ibm-granite.granite-3.3-8b-instruct-GGUF", "hf.MaziyarPanahi.Mistral-7B-Instruct-v0.3.Q4_K_M" ], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["javascript"], "frameworks": ["langchain.js", "langgraph", "fastify"] }, @@ -239,7 +239,7 @@ "basedir": "recipes/natural_language_processing/graph-rag", "readme": "# Graph RAG (Retrieval Augmented Generation) Chat Application\nThis demo provides a recipe to build out a custom Graph RAG (Graph Retrieval Augmented Generation) application using the repo LightRag which abstracts Microsoft's GraphRag implementation. It consists of two main components; the Model Service, and the AI Application with a built in Database.\nThere are a few options today for local Model Serving, but this recipe will use [`llama-cpp-python`](https://github.com/abetlen/llama-cpp-python) and their OpenAI compatible Model Service. There is a Containerfile provided that can be used to build this Model Service within the repo, [`model_servers/llamacpp_python/base/Containerfile`](/model_servers/llamacpp_python/base/Containerfile).\nLightRag simplifies development by handling the Vectordb setup automatically, while also offering experienced developers the flexibility to choose from various Vectordb options based on their preferences for usability and scalability.\nOur AI Application will connect to our Model Service via it's OpenAI compatible API. In this example we rely on [Langchain's](https://python.langchain.com/docs/get_started/introduction) python package to simplify communication with our Model Service and we use [Streamlit](https://streamlit.io/) for our UI layer. Below please see an example of the RAG application. \n\n## Try the RAG chat application\nThe [Podman Desktop](https://podman-desktop.io) [AI Lab Extension](https://github.com/containers/podman-desktop-extension-ai-lab) includes this recipe among others. To try it out, open `Recipes Catalog` -> `Graph Rag` and follow the instructions to start the application.\n\n## Models that work with this Recipe\nNot all models work with this Recipe try out mistral or llama models! \n\n# Build the Application\nThe rest of this document will explain how to build and run the application from the terminal, and will go into greater detail on how each container in the Pod above is built, run, and what purpose it serves in the overall application. All the recipes use a central [Makefile](../../common/Makefile.common) that includes variables populated with default values to simplify getting started. Please review the [Makefile docs](../../common/README.md), to learn about further customizing your application.\n\n## Quickstart\nTo run the application with pre-built images from `quay.io/ai-lab`, use `make quadlet`. This command builds the application's metadata and generates Kubernetes YAML at `./build/graph-rag.yaml` to spin up a Pod that can then be launched locally. Try it with:\n```\nmake quadlet\npodman kube play build/graph-rag.yaml\n```\nThis will take a few minutes if the model and model-server container images need to be downloaded. \nThe Pod is named `graph-rag`, so you may use [Podman](https://podman.io) to manage the Pod and its containers:\n```\npodman pod list\npodman ps\n```\nOnce the Pod and its containers are running, the application can be accessed at `http://localhost:8501`. However, if you started the app via the podman desktop UI, a random port will be assigned instead of `8501`. Please use the AI App Details `Open AI App` button to access it instead. Please refer to the section below for more details about [interacting with the Graph Rag application](#interact-with-the-ai-application).\nTo stop and remove the Pod, run:\n```\npodman pod stop graph-rag\npodman pod rm graph-rag\n```\n\n## Download a model\nIf you are just getting started, we recommend using [granite-7b-lab](https://huggingface.co/instructlab/granite-7b-lab). This is a well performant mid-sized model with an apache-2.0 license. In order to use it with our Model Service we need it converted and quantized into the [GGUF format](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md). There are a number of ways to get a GGUF version of granite-7b-lab, but the simplest is to download a pre-converted one from [huggingface.co](https://huggingface.co) here: https://huggingface.co/instructlab/granite-7b-lab-GGUF.\nThe recommended model can be downloaded using the code snippet below:\n```bash\ncd ../../../models\ncurl -sLO https://huggingface.co/instructlab/granite-7b-lab-GGUF/resolve/main/granite-7b-lab-Q4_K_M.gguf\ncd ../recipes/natural_language_processing/graph-rag\n```\n_A full list of supported open models is forthcoming._ \n\n## Build the Model Service\nThe complete instructions for building and deploying the Model Service can be found in the [llamacpp_python model-service document](../../../model_servers/llamacpp_python/README.md).\nThe Model Service can be built from make commands from the [llamacpp_python directory](../../../model_servers/llamacpp_python/).\n```bash\n# from path model_servers/llamacpp_python from repo containers/ai-lab-recipes\nmake build\n```\nCheckout the [Makefile](../../../model_servers/llamacpp_python/Makefile) to get more details on different options for how to build.\n\n## Deploy the Model Service\nThe local Model Service relies on a volume mount to the localhost to access the model files. It also employs environment variables to dictate the model used and where its served. You can start your local Model Service using the following `make` command from `model_servers/llamacpp_python` set with reasonable defaults:\n```bash\n# from path model_servers/llamacpp_python from repo containers/ai-lab-recipes\nmake run\n```\n\n## Build the AI Application\nThe AI Application can be built from the make command:\n```bash\n# Run this from the current directory (path recipes/natural_language_processing/graph-rag from repo containers/ai-lab-recipes)\nmake build\n```\n\n## Deploy the AI Application\nMake sure the Model Service is up and running before starting this container image. When starting the AI Application container image we need to direct it to the correct `MODEL_ENDPOINT`. This could be any appropriately hosted Model Service (running locally or in the cloud) using an OpenAI compatible API. In our case the Model Service is running inside the Podman machine so we need to provide it with the appropriate address `10.88.0.1`. To deploy the AI application use the following:\n```bash\n# Run this from the current directory (path recipes/natural_language_processing/graph-rag from repo containers/ai-lab-recipes)\nmake run \n```\n\n## Interact with the AI Application\nEverything should now be up an running with the chat application available at [`http://localhost:8501`](http://localhost:8501). By using this recipe and getting this starting point established, users should now have an easier time customizing and building their own LLM enabled graph-rag applications. \n\n## Embed the AI Application in a Bootable Container Image\nTo build a bootable container image that includes this sample graph-rag workload as a service that starts when a system is booted, run: `make -f Makefile bootc`. You can optionally override the default image / tag you want to give the make command by specifying it as follows: `make -f Makefile BOOTC_IMAGE= bootc`.\nSubstituting the bootc/Containerfile FROM command is simple using the Makefile FROM option.\n```bash\nmake FROM=registry.redhat.io/rhel9/rhel-bootc:9.4 bootc\n```\nSelecting the ARCH for the bootc/Containerfile is simple using the Makefile ARCH= variable.\n```\nmake ARCH=x86_64 bootc\n```\nThe magic happens when you have a bootc enabled system running. If you do, and you'd like to update the operating system to the OS you just built\nwith the graph-rag application, it's as simple as ssh-ing into the bootc system and running:\n```bash\nbootc switch quay.io/ai-lab/graph-rag-bootc:latest\n```\nUpon a reboot, you'll see that the graph-rag service is running on the system. Check on the service with:\n```bash\nssh user@bootc-system-ip\nsudo systemctl status graph-rag\n```\n\n### What are bootable containers?\nWhat's a [bootable OCI container](https://containers.github.io/bootc/) and what's it got to do with AI?\nThat's a good question! We think it's a good idea to embed AI workloads (or any workload!) into bootable images at _build time_ rather than at _runtime_. This extends the benefits, such as portability and predictability, that containerizing applications provides to the operating system. Bootable OCI images bake exactly what you need to run your workloads into the operating system at build time by using your favorite containerization tools. Might I suggest [podman](https://podman.io/)?\nOnce installed, a bootc enabled system can be updated by providing an updated bootable OCI image from any OCI image registry with a single `bootc` command. This works especially well for fleets of devices that have fixed workloads - think factories or appliances. Who doesn't want to add a little AI to their appliance, am I right?\nBootable images lend toward immutable operating systems, and the more immutable an operating system is, the less that can go wrong at runtime!\n\n#### Creating bootable disk images\nYou can convert a bootc image to a bootable disk image using the [quay.io/centos-bootc/bootc-image-builder](https://github.com/osbuild/bootc-image-builder) container image.\nThis container image allows you to build and deploy [multiple disk image types](../../common/README_bootc_image_builder.md) from bootc container images.\nDefault image types can be set via the DISK_TYPE Makefile variable.\n`make bootc-image-builder DISK_TYPE=ami`", "recommended": ["hf.instructlab.granite-7b-lab-GGUF"], - "backend": "llama-cpp", + "backends": ["llama-cpp", "openvino"], "languages": ["python"], "frameworks": ["streamlit", "lightrag"] }, @@ -254,7 +254,7 @@ "basedir": "recipes/audio/audio_to_text", "readme": "# Audio to Text Application\n\nThis recipe helps developers start building their own custom AI enabled audio transcription applications. It consists of two main components: the Model Service and the AI Application.\n\nThere are a few options today for local Model Serving, but this recipe will use [`whisper-cpp`](https://github.com/ggerganov/whisper.cpp.git) and its included Model Service. There is a Containerfile provided that can be used to build this Model Service within the repo, [`model_servers/whispercpp/base/Containerfile`](/model_servers/whispercpp/base/Containerfile).\n\nThe AI Application will connect to the Model Service via an API. The recipe relies on [Langchain's](https://python.langchain.com/docs/get_started/introduction) python package to simplify communication with the Model Service and uses [Streamlit](https://streamlit.io/) for the UI layer. You can find an example of the audio to text application below.\n\n\n![](/assets/whisper.png) \n\n## Try the Audio to Text Application:\n\nThe [Podman Desktop](https://podman-desktop.io) [AI Lab Extension](https://github.com/containers/podman-desktop-extension-ai-lab) includes this recipe among others. To try it out, open `Recipes Catalog` -> `Audio to Text` and follow the instructions to start the application.\n\n# Build the Application\n\nThe rest of this document will explain how to build and run the application from the terminal, and will go into greater detail on how each container in the application above is built, run, and what purpose it serves in the overall application. All the recipes use a central [Makefile](../../common/Makefile.common) that includes variables populated with default values to simplify getting started. Please review the [Makefile docs](../../common/README.md), to learn about further customizing your application.\n\n* [Download a model](#download-a-model)\n* [Build the Model Service](#build-the-model-service)\n* [Deploy the Model Service](#deploy-the-model-service)\n* [Build the AI Application](#build-the-ai-application)\n* [Deploy the AI Application](#deploy-the-ai-application)\n* [Interact with the AI Application](#interact-with-the-ai-application)\n * [Input audio files](#input-audio-files)\n\n## Download a model\n\nIf you are just getting started, we recommend using [ggerganov/whisper.cpp](https://huggingface.co/ggerganov/whisper.cpp).\nThis is a well performant model with an MIT license.\nIt's simple to download a pre-converted whisper model from [huggingface.co](https://huggingface.co)\nhere: https://huggingface.co/ggerganov/whisper.cpp. There are a number of options, but we recommend to start with `ggml-small.bin`.\n\nThe recommended model can be downloaded using the code snippet below:\n\n```bash\ncd ../../../models\ncurl -sLO https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin\ncd ../recipes/audio/audio_to_text\n```\n\n_A full list of supported open models is forthcoming._\n\n\n## Build the Model Service\n\nThe complete instructions for building and deploying the Model Service can be found in the [whispercpp model-service document](../../../model_servers/whispercpp/README.md).\n\n```bash\n# from path model_servers/whispercpp from repo containers/ai-lab-recipes\nmake build\n```\nCheckout the [Makefile](../../../model_servers/whispercpp/Makefile) to get more details on different options for how to build.\n\n## Deploy the Model Service\n\nThe local Model Service relies on a volume mount to the localhost to access the model files. It also employs environment variables to dictate the model used and where its served. You can start your local Model Service using the following `make` command from `model_servers/whispercpp` set with reasonable defaults:\n\n```bash\n# from path model_servers/whispercpp from repo containers/ai-lab-recipes\nmake run\n```\n\n## Build the AI Application\n\nNow that the Model Service is running we want to build and deploy our AI Application. Use the provided Containerfile to build the AI Application\nimage from the [`audio-to-text/`](./) directory.\n\n```bash\n# from path recipes/audio/audio_to_text from repo containers/ai-lab-recipes\npodman build -t audio-to-text app\n```\n### Deploy the AI Application\n\nMake sure the Model Service is up and running before starting this container image.\nWhen starting the AI Application container image we need to direct it to the correct `MODEL_ENDPOINT`.\nThis could be any appropriately hosted Model Service (running locally or in the cloud) using a compatible API.\nThe following Podman command can be used to run your AI Application:\n\n```bash\npodman run --rm -it -p 8501:8501 -e MODEL_ENDPOINT=http://10.88.0.1:8001/inference audio-to-text \n```\n\n### Interact with the AI Application\n\nOnce the streamlit application is up and running, you should be able to access it at `http://localhost:8501`.\nFrom here, you can upload audio files from your local machine and translate the audio files as shown below.\n\nBy using this recipe and getting this starting point established,\nusers should now have an easier time customizing and building their own AI enabled applications.\n\n#### Input audio files\n\nWhisper.cpp requires as an input 16-bit WAV audio files.\nTo convert your input audio files to 16-bit WAV format you can use `ffmpeg` like this:\n\n```bash\nffmpeg -i -ar 16000 -ac 1 -c:a pcm_s16le \n```\n", "recommended": ["hf.ggerganov.whisper.cpp"], - "backend": "whisper-cpp", + "backends": ["whisper-cpp"], "languages": ["python"], "frameworks": ["streamlit"] }, @@ -269,7 +269,7 @@ "basedir": "recipes/computer_vision/object_detection", "readme": "# Object Detection\n\nThis recipe helps developers start building their own custom AI enabled object detection applications. It consists of two main components: the Model Service and the AI Application.\n\nThere are a few options today for local Model Serving, but this recipe will use our FastAPI [`object_detection_python`](../../../model_servers/object_detection_python/src/object_detection_server.py) model server. There is a Containerfile provided that can be used to build this Model Service within the repo, [`model_servers/object_detection_python/base/Containerfile`](/model_servers/object_detection_python/base/Containerfile).\n\nThe AI Application will connect to the Model Service via an API. The recipe relies on [Streamlit](https://streamlit.io/) for the UI layer. You can find an example of the object detection application below.\n\n![](/assets/object_detection.png) \n\n## Try the Object Detection Application:\n\nThe [Podman Desktop](https://podman-desktop.io) [AI Lab Extension](https://github.com/containers/podman-desktop-extension-ai-lab) includes this recipe among others. To try it out, open `Recipes Catalog` -> `Object Detection` and follow the instructions to start the application.\n\n# Build the Application\n\nThe rest of this document will explain how to build and run the application from the terminal, and will go into greater detail on how each container in the application above is built, run, and what purpose it serves in the overall application. All the Model Server elements of the recipe use a central Model Server [Makefile](../../../model_servers/common/Makefile.common) that includes variables populated with default values to simplify getting started. Currently we do not have a Makefile for the Application elements of the Recipe, but this coming soon, and will leverage the recipes common [Makefile](../../common/Makefile.common) to provide variable configuration and reasonable defaults to this Recipe's application.\n\n* [Download a model](#download-a-model)\n* [Build the Model Service](#build-the-model-service)\n* [Deploy the Model Service](#deploy-the-model-service)\n* [Build the AI Application](#build-the-ai-application)\n* [Deploy the AI Application](#deploy-the-ai-application)\n* [Interact with the AI Application](#interact-with-the-ai-application)\n\n## Download a model\n\nIf you are just getting started, we recommend using [facebook/detr-resnet-101](https://huggingface.co/facebook/detr-resnet-101).\nThis is a well performant model with an Apache-2.0 license.\nIt's simple to download a copy of the model from [huggingface.co](https://huggingface.co)\n\nYou can use the `download-model-facebook-detr-resnet-101` make target in the `model_servers/object_detection_python` directory to download and move the model into the models directory for you:\n\n```bash\n# from path model_servers/object_detection_python from repo containers/ai-lab-recipes\n make download-model-facebook-detr-resnet-101\n```\n\n## Build the Model Service\n\nThe You can build the Model Service from the [object_detection_python model-service directory](../../../model_servers/object_detection_python).\n\n```bash\n# from path model_servers/object_detection_python from repo containers/ai-lab-recipes\nmake build\n```\n\nCheckout the [Makefile](../../../model_servers/object_detection_python/Makefile) to get more details on different options for how to build.\n\n## Deploy the Model Service\n\nThe local Model Service relies on a volume mount to the localhost to access the model files. It also employs environment variables to dictate the model used and where its served. You can start your local Model Service using the following `make` command from the [`model_servers/object_detection_python`](../../../model_servers/object_detection_python) directory, which will be set with reasonable defaults:\n\n```bash\n# from path model_servers/object_detection_python from repo containers/ai-lab-recipes\nmake run\n```\n\nAs stated above, by default the model service will use [`facebook/detr-resnet-101`](https://huggingface.co/facebook/detr-resnet-101). However you can use other compatabale models. Simply pass the new `MODEL_NAME` and `MODEL_PATH` to the make command. Make sure the model is downloaded and exists in the [models directory](../../../models/):\n\n```bash\n# from path model_servers/object_detection_python from repo containers/ai-lab-recipes\nmake MODEL_NAME=facebook/detr-resnet-50 MODEL_PATH=/models/facebook/detr-resnet-50 run\n```\n\n## Build the AI Application\n\nNow that the Model Service is running we want to build and deploy our AI Application. Use the provided Containerfile to build the AI Application\nimage from the [`object_detection/`](./) recipe directory.\n\n```bash\n# from path recipes/computer_vision/object_detection from repo containers/ai-lab-recipes\npodman build -t object_detection_client .\n```\n\n### Deploy the AI Application\n\nMake sure the Model Service is up and running before starting this container image.\nWhen starting the AI Application container image we need to direct it to the correct `MODEL_ENDPOINT`.\nThis could be any appropriately hosted Model Service (running locally or in the cloud) using a compatible API.\nThe following Podman command can be used to run your AI Application:\n\n```bash\npodman run -p 8501:8501 -e MODEL_ENDPOINT=http://10.88.0.1:8000/detection object_detection_client\n```\n\n### Interact with the AI Application\n\nOnce the client is up a running, you should be able to access it at `http://localhost:8501`. From here you can upload images from your local machine and detect objects in the image as shown below. \n\nBy using this recipe and getting this starting point established,\nusers should now have an easier time customizing and building their own AI enabled applications.\n", "recommended": ["hf.facebook.detr-resnet-101"], - "backend": "none", + "backends": ["none"], "languages": ["python"], "frameworks": ["streamlit"] } diff --git a/packages/backend/src/managers/catalogManager.ts b/packages/backend/src/managers/catalogManager.ts index b84f45ad5..024d003af 100644 --- a/packages/backend/src/managers/catalogManager.ts +++ b/packages/backend/src/managers/catalogManager.ts @@ -289,9 +289,14 @@ export class CatalogManager extends Publisher implements Dis result = res; break; } - case 'tools': - result = result.filter(r => values.includes(r.backend ?? '')); + case 'tools': { + let res: Recipe[] = []; + for (const value of values) { + res = [...res, ...result.filter(r => r.backends?.includes(value))]; + } + result = res; break; + } case 'frameworks': { let res: Recipe[] = []; for (const value of values) { @@ -325,13 +330,13 @@ export class CatalogManager extends Publisher implements Dis choices.tools = this.filterRecipes(subfilters).choices.tools; } else { choices.tools = result - .map(r => r.backend) + .flatMap(r => r.backends) .filter(b => b !== undefined) .filter((value, index, array) => array.indexOf(value) === index) .sort((a, b) => a.localeCompare(b)) .map(t => ({ name: t, - count: result.filter(r => r.backend === t).length, + count: result.filter(r => r.backends?.includes(t)).length, })); } diff --git a/packages/backend/src/managers/recipes/RecipeManager.ts b/packages/backend/src/managers/recipes/RecipeManager.ts index 9d4ee61ae..24180b7e1 100644 --- a/packages/backend/src/managers/recipes/RecipeManager.ts +++ b/packages/backend/src/managers/recipes/RecipeManager.ts @@ -106,7 +106,7 @@ export class RecipeManager implements Disposable { let inferenceServer: InferenceServer | undefined; // if the recipe has a defined backend, we gives priority to using an inference server - if (recipe.backend && recipe.backend === model.backend) { + if (model.backend && recipe.backends?.includes(model.backend)) { let task: Task | undefined; try { inferenceServer = this.inferenceManager.findServerByModel(model); diff --git a/packages/backend/src/tests/ai-test.json b/packages/backend/src/tests/ai-test.json index 06bc2a36b..4b970169d 100644 --- a/packages/backend/src/tests/ai-test.json +++ b/packages/backend/src/tests/ai-test.json @@ -27,7 +27,7 @@ "description": "", "repository": "", "readme": "", - "backend": "tool1", + "backends": ["tool1"], "languages": ["lang1", "lang10"], "frameworks": ["fw1", "fw10"] }, @@ -38,7 +38,7 @@ "description": "", "repository": "", "readme": "", - "backend": "tool2", + "backends": ["tool2"], "languages": ["lang2", "lang10"], "frameworks": ["fw2", "fw10"] }, @@ -49,7 +49,7 @@ "description": "", "repository": "", "readme": "", - "backend": "tool3", + "backends": ["tool3"], "languages": ["lang3", "lang11"], "frameworks": ["fw2", "fw10", "fw11"] } diff --git a/packages/backend/src/utils/catalogUtils.spec.ts b/packages/backend/src/utils/catalogUtils.spec.ts index a738da6f6..7cece01f7 100644 --- a/packages/backend/src/utils/catalogUtils.spec.ts +++ b/packages/backend/src/utils/catalogUtils.spec.ts @@ -173,6 +173,98 @@ describe('sanitize', () => { expect(catalog.recipes[0].languages).toStrictEqual(['lang1']); expect(catalog.recipes[0].frameworks).toStrictEqual(['fw1']); }); + + test('should return backend recipe as string', () => { + const raw = { + version: '1.0', + recipes: [ + { + id: 'chatbot', + description: 'This is a Streamlit chat demo application.', + name: 'ChatBot', + repository: 'https://github.com/containers/ai-lab-recipes', + ref: 'v1.1.3', + icon: 'natural-language-processing', + categories: ['natural-language-processing'], + basedir: 'recipes/natural_language_processing/chatbot', + readme: '', + recommended: ['hf.instructlab.granite-7b-lab-GGUF', 'hf.instructlab.merlinite-7b-lab-GGUF'], + backends: ['llama-cpp'], + languages: ['lang1'], + frameworks: ['fw1'], + }, + ], + models: [ + { + id: 'Mistral-7B-Instruct-v0.3-Q4_K_M.gguf', + name: 'Mistral-7B-Instruct-v0.3-Q4_K_M', + description: 'Model imported from path\\Mistral-7B-Instruct-v0.3-Q4_K_M.gguf', + hw: 'CPU', + file: { + path: 'path', + file: 'Mistral-7B-Instruct-v0.3-Q4_K_M.gguf', + size: 4372812000, + creation: '2024-06-19T12:14:12.489Z', + }, + memory: 4372812000, + }, + ], + }; + expect(hasCatalogWrongFormat(raw)).toBeFalsy(); + const catalog = sanitize(raw); + expect(catalog.version).equals(CatalogFormat.CURRENT); + expect(catalog.models[0].backend).toBeUndefined(); + expect(catalog.models[0].name).equals('Mistral-7B-Instruct-v0.3-Q4_K_M'); + expect(catalog.recipes[0].languages).toStrictEqual(['lang1']); + expect(catalog.recipes[0].frameworks).toStrictEqual(['fw1']); + expect(catalog.recipes[0].backends).toStrictEqual(['llama-cpp']); + }); + + test('should return multiple backend recipe as array', () => { + const raw = { + version: '1.0', + recipes: [ + { + id: 'chatbot', + description: 'This is a Streamlit chat demo application.', + name: 'ChatBot', + repository: 'https://github.com/containers/ai-lab-recipes', + ref: 'v1.1.3', + icon: 'natural-language-processing', + categories: ['natural-language-processing'], + basedir: 'recipes/natural_language_processing/chatbot', + readme: '', + recommended: ['hf.instructlab.granite-7b-lab-GGUF', 'hf.instructlab.merlinite-7b-lab-GGUF'], + backends: ['llama-cpp', 'openvino'], + languages: ['lang1'], + frameworks: ['fw1'], + }, + ], + models: [ + { + id: 'Mistral-7B-Instruct-v0.3-Q4_K_M.gguf', + name: 'Mistral-7B-Instruct-v0.3-Q4_K_M', + description: 'Model imported from path\\Mistral-7B-Instruct-v0.3-Q4_K_M.gguf', + hw: 'CPU', + file: { + path: 'path', + file: 'Mistral-7B-Instruct-v0.3-Q4_K_M.gguf', + size: 4372812000, + creation: '2024-06-19T12:14:12.489Z', + }, + memory: 4372812000, + }, + ], + }; + expect(hasCatalogWrongFormat(raw)).toBeFalsy(); + const catalog = sanitize(raw); + expect(catalog.version).equals(CatalogFormat.CURRENT); + expect(catalog.models[0].backend).toBeUndefined(); + expect(catalog.models[0].name).equals('Mistral-7B-Instruct-v0.3-Q4_K_M'); + expect(catalog.recipes[0].languages).toStrictEqual(['lang1']); + expect(catalog.recipes[0].frameworks).toStrictEqual(['fw1']); + expect(catalog.recipes[0].backends).toStrictEqual(['llama-cpp', 'openvino']); + }); }); describe('merge', () => { diff --git a/packages/backend/src/utils/catalogUtils.ts b/packages/backend/src/utils/catalogUtils.ts index 1f2080993..2ef7a7b15 100644 --- a/packages/backend/src/utils/catalogUtils.ts +++ b/packages/backend/src/utils/catalogUtils.ts @@ -114,6 +114,15 @@ export function isStringArray(obj: unknown): obj is Array { return Array.isArray(obj) && obj.every(item => typeof item === 'string'); } +function sanitizeBackends(recipe: object): string[] | undefined { + if ('backend' in recipe && typeof recipe.backend === 'string') { + return [recipe.backend]; + } else if ('backends' in recipe && Array.isArray(recipe.backends)) { + return recipe.backends; + } + return undefined; +} + export function sanitizeRecipe(recipe: unknown): Recipe { if ( isNonNullObject(recipe) && @@ -143,7 +152,7 @@ export function sanitizeRecipe(recipe: unknown): Recipe { icon: 'icon' in recipe && typeof recipe.icon === 'string' ? recipe.icon : undefined, basedir: 'basedir' in recipe && typeof recipe.basedir === 'string' ? recipe.basedir : undefined, recommended: 'recommended' in recipe && isStringArray(recipe.recommended) ? recipe.recommended : undefined, - backend: 'backend' in recipe && typeof recipe.backend === 'string' ? recipe.backend : undefined, + backends: sanitizeBackends(recipe), languages: 'languages' in recipe && isStringArray(recipe.languages) ? recipe.languages : undefined, frameworks: 'frameworks' in recipe && isStringArray(recipe.frameworks) ? recipe.frameworks : undefined, }; diff --git a/packages/frontend/src/lib/RecipeCardTags.spec.ts b/packages/frontend/src/lib/RecipeCardTags.spec.ts index fbf357146..a3f186ecc 100644 --- a/packages/frontend/src/lib/RecipeCardTags.spec.ts +++ b/packages/frontend/src/lib/RecipeCardTags.spec.ts @@ -31,7 +31,7 @@ const recipe = { categories: ['natural-language-processing', 'audio'], languages: ['java', 'python'], frameworks: ['langchain', 'vectordb'], - backend: 'whisper-cpp', + backends: ['whisper-cpp'], }; class ResizeObserver { diff --git a/packages/frontend/src/lib/RecipeCardTags.svelte b/packages/frontend/src/lib/RecipeCardTags.svelte index ab808085c..8250eb389 100644 --- a/packages/frontend/src/lib/RecipeCardTags.svelte +++ b/packages/frontend/src/lib/RecipeCardTags.svelte @@ -12,7 +12,7 @@ let { recipe }: Props = $props(); const TAGS: string[] = [ ...recipe.categories, - ...(recipe.backend !== undefined ? [recipe.backend] : []), + ...(recipe.backends ?? []), ...(recipe.frameworks ?? []), ...(recipe.languages ?? []), ]; diff --git a/packages/frontend/src/pages/StartRecipe.spec.ts b/packages/frontend/src/pages/StartRecipe.spec.ts index 83b27a1b3..f5b28a3c0 100644 --- a/packages/frontend/src/pages/StartRecipe.spec.ts +++ b/packages/frontend/src/pages/StartRecipe.spec.ts @@ -60,16 +60,25 @@ vi.mock('../utils/client', async () => ({ }, })); -const fakeRecipe: Recipe = { - id: 'dummy-recipe-id', - backend: InferenceType.LLAMA_CPP, - name: 'Dummy Recipe', - description: 'Dummy description', +const fakeRecipe1: Recipe = { + id: 'dummy-recipe-id1', + backends: [InferenceType.LLAMA_CPP], + name: 'Dummy Recipe 1', + description: 'Dummy description 1', + recommended: ['dummy-model-1'], + categories: [], +} as unknown as Recipe; + +const fakeRecipe2: Recipe = { + id: 'dummy-recipe-id2', + backends: [InferenceType.LLAMA_CPP, InferenceType.OPENVINO], + name: 'Dummy Recipe 2', + description: 'Dummy description 2', recommended: ['dummy-model-1'], categories: [], } as unknown as Recipe; -const fakeRecommendedModel: ModelInfo = { +const fakeRecommendedModel1: ModelInfo = { id: 'dummy-model-1', backend: InferenceType.LLAMA_CPP, name: 'Dummy Model 1', @@ -79,10 +88,20 @@ const fakeRecommendedModel: ModelInfo = { }, } as unknown as ModelInfo; -const fakeRemoteModel: ModelInfo = { +const fakeRecommendedModel2: ModelInfo = { id: 'dummy-model-2', - backend: InferenceType.LLAMA_CPP, + backend: InferenceType.OPENVINO, name: 'Dummy Model 2', + file: { + file: 'dummy-model-file', + path: 'dummy-model-path', + }, +} as unknown as ModelInfo; + +const fakeRemoteModel: ModelInfo = { + id: 'dummy-model-3', + backend: InferenceType.LLAMA_CPP, + name: 'Dummy Model 3', } as unknown as ModelInfo; const containerProviderConnection: ContainerProviderConnectionInfo = { @@ -100,13 +119,13 @@ beforeEach(() => { router.location.query.clear(); vi.mocked(CatalogStore).catalog = readable({ - recipes: [fakeRecipe], + recipes: [fakeRecipe1, fakeRecipe2], models: [], categories: [], version: '', }); vi.mocked(ConnectionStore).containerProviderConnections = readable([containerProviderConnection]); - vi.mocked(ModelsInfoStore).modelsInfo = readable([fakeRecommendedModel, fakeRemoteModel]); + vi.mocked(ModelsInfoStore).modelsInfo = readable([fakeRecommendedModel1, fakeRecommendedModel2, fakeRemoteModel]); vi.mocked(LocalRepositoryStore).localRepositories = readable([]); vi.mocked(TaskStore).tasks = readable([]); @@ -118,12 +137,12 @@ beforeEach(() => { test('Recipe name should be visible', async () => { render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); const span = screen.getByLabelText('Recipe name'); expect(span).toBeDefined(); - expect(span.textContent).toBe(fakeRecipe.name); + expect(span.textContent).toBe(fakeRecipe1.name); }); test('Recipe Local Repository should be visible when defined', async () => { @@ -133,13 +152,13 @@ test('Recipe Local Repository should be visible when defined', async () => { path: 'dummy-recipe-path', sourcePath: 'dummy-recipe-path', labels: { - 'recipe-id': fakeRecipe.id, + 'recipe-id': fakeRecipe1.id, }, }, ]); render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); const span = screen.getByLabelText('Recipe local path'); @@ -151,22 +170,22 @@ test('Submit button should be disabled when no model is selected', async () => { vi.mocked(ModelsInfoStore).modelsInfo = readable([]); render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); - const button = screen.getByTitle(`Start ${fakeRecipe.name} recipe`); + const button = screen.getByTitle(`Start ${fakeRecipe1.name} recipe`); expect(button).toBeDefined(); expect(button).toBeDisabled(); }); test('First recommended model should be selected as default model', async () => { const { container } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); await vi.waitFor(() => { const option = getSelectedOption<{ value: string }>(container); - expect(option?.value).toBe(fakeRecommendedModel.id); + expect(option?.value).toBe(fakeRecommendedModel1.id); }); }); @@ -212,7 +231,7 @@ async function selectOption(container: HTMLElement, label: string): Promise { const { container } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); await selectOption(container, fakeRemoteModel.name); @@ -227,44 +246,70 @@ test('Selecting model not downloaded should display a warning', async () => { }); }); +test('Check single backend recipe has models only for that backend', async () => { + const { container } = render(StartRecipe, { + recipeId: 'dummy-recipe-id1', + }); + + const input = screen.getByLabelText('Select Model'); + await fireEvent.pointerUp(input); // they are using the pointer up event instead of click. + // get all options available + const items = container.querySelectorAll('div[class~="list-item"]'); + // ensure we have two options + expect(items.length).toBe(2); +}); + +test('Check multiple backends recipe has models for all backends', async () => { + const { container } = render(StartRecipe, { + recipeId: 'dummy-recipe-id2', + }); + + const input = screen.getByLabelText('Select Model'); + await fireEvent.pointerUp(input); // they are using the pointer up event instead of click. + // get all options available + const items = container.querySelectorAll('div[class~="list-item"]'); + // ensure we have two options + expect(items.length).toBe(3); +}); + test('Selecting model downloaded should not display a warning', async () => { const { container } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); - await selectOption(container, fakeRecommendedModel.name); + await selectOption(container, fakeRecommendedModel1.name); const span = screen.queryByRole('alert'); expect(span).toBeNull(); }); test('Selecting model should enable submit button', async () => { const { container } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); - await selectOption(container, fakeRecommendedModel.name); + await selectOption(container, fakeRecommendedModel1.name); - const button = screen.getByTitle(`Start ${fakeRecipe.name} recipe`); + const button = screen.getByTitle(`Start ${fakeRecipe1.name} recipe`); expect(button).toBeDefined(); expect(button).not.toBeDisabled(); }); test('Submit button should call requestPullApplication with proper arguments', async () => { const { container } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); - await selectOption(container, fakeRecommendedModel.name); + await selectOption(container, fakeRecommendedModel1.name); - const button = screen.getByTitle(`Start ${fakeRecipe.name} recipe`); + const button = screen.getByTitle(`Start ${fakeRecipe1.name} recipe`); expect(button).toBeEnabled(); await fireEvent.click(button); await vi.waitFor(() => { expect(studioClient.requestPullApplication).toHaveBeenCalledWith({ connection: containerProviderConnection, - recipeId: fakeRecipe.id, - modelId: fakeRecommendedModel.id, + recipeId: fakeRecipe1.id, + modelId: fakeRecommendedModel1.id, }); }); }); @@ -274,12 +319,12 @@ test('Submit button should call requestPullApplication with proper arguments', a vi.mocked(ConnectionStore).containerProviderConnections = readable([]); const { container, getByTitle } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); - await selectOption(container, fakeRecommendedModel.name); + await selectOption(container, fakeRecommendedModel1.name); - const button = getByTitle(`Start ${fakeRecipe.name} recipe`); + const button = getByTitle(`Start ${fakeRecipe1.name} recipe`); expect(button).toBeDisabled(); }); @@ -295,12 +340,12 @@ test('Loading task should make the submit button disabled', async () => { } as Task, ]); const { container } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); - await selectOption(container, fakeRecommendedModel.name); + await selectOption(container, fakeRecommendedModel1.name); - const button = screen.getByTitle(`Start ${fakeRecipe.name} recipe`); + const button = screen.getByTitle(`Start ${fakeRecipe1.name} recipe`); expect(button).not.toBeDisabled(); await fireEvent.click(button); @@ -317,7 +362,7 @@ test('Completed task should make the open details button visible', async () => { state: 'success', labels: { trackingId: 'fake-tracking-id', - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }, } as Task, ]); @@ -325,7 +370,7 @@ test('Completed task should make the open details button visible', async () => { router.location.query.set('trackingId', 'fake-tracking-id'); const { container } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', trackingId: 'fake-tracking-id', }); @@ -349,10 +394,10 @@ test('trackingId in router query should use it to display related tasks', () => router.location.query.set('trackingId', 'fake-tracking-id'); render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', trackingId: 'fake-tracking-id', }); - const button = screen.getByTitle(`Start ${fakeRecipe.name} recipe`); + const button = screen.getByTitle(`Start ${fakeRecipe1.name} recipe`); expect(button).toBeDisabled(); }); @@ -364,7 +409,7 @@ test('restoring page should use model-id from tasks to restore the value in the state: 'loading', labels: { trackingId: 'fake-tracking-id', - 'model-id': fakeRecommendedModel.id, + 'model-id': fakeRecommendedModel1.id, }, } as Task, ]); @@ -372,13 +417,13 @@ test('restoring page should use model-id from tasks to restore the value in the router.location.query.set('trackingId', 'fake-tracking-id'); const { container } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); return await vi.waitFor(() => { const input = container.querySelector('input[name="select-model"][type="hidden"]'); if (!input) throw new Error('input not found'); - expect(JSON.parse((input as HTMLInputElement).value).label).toBe(fakeRecommendedModel.name); + expect(JSON.parse((input as HTMLInputElement).value).label).toBe(fakeRecommendedModel1.name); }); }); @@ -387,7 +432,7 @@ test('no containerProviderConnections should have no running container error', a vi.mocked(ConnectionStore).containerProviderConnections = readable([]); const { getByRole } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); const alert = getByRole('alert'); @@ -400,7 +445,7 @@ test('no container error should disappear if one get available', async () => { vi.mocked(ConnectionStore).containerProviderConnections = store; const { getByRole, queryByRole } = render(StartRecipe, { - recipeId: 'dummy-recipe-id', + recipeId: 'dummy-recipe-id1', }); // First we should have the error diff --git a/packages/frontend/src/pages/StartRecipe.svelte b/packages/frontend/src/pages/StartRecipe.svelte index 804fd18e3..89a9b8ee1 100644 --- a/packages/frontend/src/pages/StartRecipe.svelte +++ b/packages/frontend/src/pages/StartRecipe.svelte @@ -42,7 +42,7 @@ let startedContainerProviderConnectionInfo: ContainerProviderConnectionInfo[] = let localPath: LocalRepository | undefined = $derived(findLocalRepositoryByRecipeId($localRepositories, recipe?.id)); // Filter all models based on backend property let models: ModelInfo[] = $derived( - $modelsInfo.filter(model => (model.backend ?? InferenceType.NONE) === (recipe?.backend ?? InferenceType.NONE)), + $modelsInfo.filter(model => recipe?.backends?.includes(model.backend ?? InferenceType.NONE)), ); // Hold the selected model let model: ModelInfo | undefined = $state(undefined); diff --git a/packages/shared/src/models/IRecipe.ts b/packages/shared/src/models/IRecipe.ts index f7a5ff0ff..8eba33ea4 100644 --- a/packages/shared/src/models/IRecipe.ts +++ b/packages/shared/src/models/IRecipe.ts @@ -53,10 +53,10 @@ export interface Recipe { basedir?: string; recommended?: string[]; /** - * The backend field aims to target which inference + * The backends field aims to target which inference * server the recipe requires */ - backend?: string; + backends?: string[]; languages?: string[]; frameworks?: string[]; }