diff --git a/packages/types/src/providers/index.ts b/packages/types/src/providers/index.ts index 5f1c08041f..8b0624ef2a 100644 --- a/packages/types/src/providers/index.ts +++ b/packages/types/src/providers/index.ts @@ -8,6 +8,7 @@ export * from "./groq.js" export * from "./lite-llm.js" export * from "./lm-studio.js" export * from "./mistral.js" +export * from "./ollama.js" export * from "./openai.js" export * from "./openrouter.js" export * from "./requesty.js" diff --git a/packages/types/src/providers/lm-studio.ts b/packages/types/src/providers/lm-studio.ts index f83bbc1039..9e39ae5608 100644 --- a/packages/types/src/providers/lm-studio.ts +++ b/packages/types/src/providers/lm-studio.ts @@ -1 +1,19 @@ +import type { ModelInfo } from "../model.js" + export const LMSTUDIO_DEFAULT_TEMPERATURE = 0 + +// LM Studio +// https://lmstudio.ai/docs/cli/ls +export const lMStudioDefaultModelId = "mistralai/devstral-small-2505" +export const lMStudioDefaultModelInfo: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsImages: true, + supportsComputerUse: true, + supportsPromptCache: true, + inputPrice: 0, + outputPrice: 0, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + description: "LM Studio hosted models", +} diff --git a/packages/types/src/providers/ollama.ts b/packages/types/src/providers/ollama.ts new file mode 100644 index 0000000000..d269da8f4d --- /dev/null +++ b/packages/types/src/providers/ollama.ts @@ -0,0 +1,17 @@ +import type { ModelInfo } from "../model.js" + +// Ollama +// https://ollama.com/models +export const ollamaDefaultModelId = "devstral:24b" +export const ollamaDefaultModelInfo: ModelInfo = { + maxTokens: 4096, + contextWindow: 200_000, + supportsImages: true, + supportsComputerUse: true, + supportsPromptCache: true, + inputPrice: 0, + outputPrice: 0, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + description: "Ollama hosted models", +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 56106bf3b7..e391953820 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -576,6 +576,9 @@ importers: '@google/genai': specifier: ^1.0.0 version: 1.3.0(@modelcontextprotocol/sdk@1.12.0) + '@lmstudio/sdk': + specifier: ^1.1.1 + version: 1.2.0 '@mistralai/mistralai': specifier: ^1.3.6 version: 1.6.1(zod@3.25.61) @@ -2280,6 +2283,12 @@ packages: cpu: [x64] os: [win32] + '@lmstudio/lms-isomorphic@0.4.5': + resolution: {integrity: sha512-Or9KS1Iz3LC7D7WMe4zbqAqKOlDsVcrvMoQFBhmydzzxOg+eYBM5gtfgMMjcwjM0BuUVPhYOjTWEyfXpqfVJzg==} + + '@lmstudio/sdk@1.2.0': + resolution: {integrity: sha512-Eoolmi1cSuGXmLYwtn6pD9eOwjMTb+bQ4iv+i/EYz/hCc+HtbfJamoKfyyw4FogRc03RHsXHe1X18voR40D+2g==} + '@manypkg/find-root@1.1.0': resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} @@ -6771,6 +6780,9 @@ packages: jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} + jsonschema@1.5.0: + resolution: {integrity: sha512-K+A9hhqbn0f3pJX17Q/7H6yQfD/5OXgdrR5UE12gMXCiN9D5Xq2o5mddV2QEcX/bjla99ASsAAQUyMCCRWAEhw==} + jsonwebtoken@9.0.2: resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==} engines: {node: '>=12', npm: '>=6'} @@ -11335,6 +11347,24 @@ snapshots: '@libsql/win32-x64-msvc@0.5.13': optional: true + '@lmstudio/lms-isomorphic@0.4.5': + dependencies: + ws: 8.18.2 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@lmstudio/sdk@1.2.0': + dependencies: + '@lmstudio/lms-isomorphic': 0.4.5 + chalk: 4.1.2 + jsonschema: 1.5.0 + zod: 3.25.61 + zod-to-json-schema: 3.24.5(zod@3.25.61) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + '@manypkg/find-root@1.1.0': dependencies: '@babel/runtime': 7.27.4 @@ -16318,6 +16348,8 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 + jsonschema@1.5.0: {} + jsonwebtoken@9.0.2: dependencies: jws: 3.2.2 diff --git a/src/api/providers/fetchers/__tests__/fixtures/lmstudio-model-details.json b/src/api/providers/fetchers/__tests__/fixtures/lmstudio-model-details.json new file mode 100644 index 0000000000..43f5505710 --- /dev/null +++ b/src/api/providers/fetchers/__tests__/fixtures/lmstudio-model-details.json @@ -0,0 +1,14 @@ +{ + "mistralai/devstral-small-2505": { + "type": "llm", + "modelKey": "mistralai/devstral-small-2505", + "format": "safetensors", + "displayName": "Devstral Small 2505", + "path": "mistralai/devstral-small-2505", + "sizeBytes": 13277565112, + "architecture": "mistral", + "vision": false, + "trainedForToolUse": false, + "maxContextLength": 131072 + } +} diff --git a/src/api/providers/fetchers/__tests__/fixtures/ollama-model-details.json b/src/api/providers/fetchers/__tests__/fixtures/ollama-model-details.json new file mode 100644 index 0000000000..e0b6ac03d9 --- /dev/null +++ b/src/api/providers/fetchers/__tests__/fixtures/ollama-model-details.json @@ -0,0 +1,58 @@ +{ + "qwen3-2to16:latest": { + "license": " Apache License\\n Version 2.0, January 2004\\n...", + "modelfile": "model.modelfile,# To build a new Modelfile based on this, replace FROM with:...", + "parameters": "repeat_penalty 1\\nstop \\\\nstop...", + "template": "{{- if .Messages }}\\n{{- if or .System .Tools }}<|im_start|>system...", + "details": { + "parent_model": "/Users/brad/.ollama/models/blobs/sha256-3291abe70f16ee9682de7bfae08db5373ea9d6497e614aaad63340ad421d6312", + "format": "gguf", + "family": "qwen3", + "families": ["qwen3"], + "parameter_size": "32.8B", + "quantization_level": "Q4_K_M" + }, + "model_info": { + "general.architecture": "qwen3", + "general.basename": "Qwen3", + "general.file_type": 15, + "general.parameter_count": 32762123264, + "general.quantization_version": 2, + "general.size_label": "32B", + "general.type": "model", + "qwen3.attention.head_count": 64, + "qwen3.attention.head_count_kv": 8, + "qwen3.attention.key_length": 128, + "qwen3.attention.layer_norm_rms_epsilon": 0.000001, + "qwen3.attention.value_length": 128, + "qwen3.block_count": 64, + "qwen3.context_length": 40960, + "qwen3.embedding_length": 5120, + "qwen3.feed_forward_length": 25600, + "qwen3.rope.freq_base": 1000000, + "tokenizer.ggml.add_bos_token": false, + "tokenizer.ggml.bos_token_id": 151643, + "tokenizer.ggml.eos_token_id": 151645, + "tokenizer.ggml.merges": null, + "tokenizer.ggml.model": "gpt2", + "tokenizer.ggml.padding_token_id": 151643, + "tokenizer.ggml.pre": "qwen2", + "tokenizer.ggml.token_type": null, + "tokenizer.ggml.tokens": null + }, + "tensors": [ + { + "name": "output.weight", + "type": "Q6_K", + "shape": [5120, 151936] + }, + { + "name": "output_norm.weight", + "type": "F32", + "shape": [5120] + } + ], + "capabilities": ["completion", "tools"], + "modified_at": "2025-06-02T22:16:13.644123606-04:00" + } +} diff --git a/src/api/providers/fetchers/__tests__/lmstudio.test.ts b/src/api/providers/fetchers/__tests__/lmstudio.test.ts new file mode 100644 index 0000000000..59b4388785 --- /dev/null +++ b/src/api/providers/fetchers/__tests__/lmstudio.test.ts @@ -0,0 +1,197 @@ +import axios from "axios" +import { vi, describe, it, expect, beforeEach } from "vitest" +import { LMStudioClient, LLM, LLMInstanceInfo } from "@lmstudio/sdk" // LLMInfo is a type +import { getLMStudioModels, parseLMStudioModel } from "../lmstudio" +import { ModelInfo, lMStudioDefaultModelInfo } from "@roo-code/types" // ModelInfo is a type + +// Mock axios +vi.mock("axios") +const mockedAxios = axios as any + +// Mock @lmstudio/sdk +const mockGetModelInfo = vi.fn() +const mockListLoaded = vi.fn() +vi.mock("@lmstudio/sdk", () => { + return { + LMStudioClient: vi.fn().mockImplementation(() => ({ + llm: { + listLoaded: mockListLoaded, + }, + })), + } +}) +const MockedLMStudioClientConstructor = LMStudioClient as any + +describe("LMStudio Fetcher", () => { + beforeEach(() => { + vi.clearAllMocks() + MockedLMStudioClientConstructor.mockClear() + mockListLoaded.mockClear() + mockGetModelInfo.mockClear() + }) + + describe("parseLMStudioModel", () => { + it("should correctly parse raw LLMInfo to ModelInfo", () => { + const rawModel: LLMInstanceInfo = { + type: "llm", + modelKey: "mistralai/devstral-small-2505", + format: "safetensors", + displayName: "Devstral Small 2505", + path: "mistralai/devstral-small-2505", + sizeBytes: 13277565112, + architecture: "mistral", + identifier: "mistralai/devstral-small-2505", + instanceReference: "RAP5qbeHVjJgBiGFQ6STCuTJ", + vision: false, + trainedForToolUse: false, + maxContextLength: 131072, + contextLength: 7161, + } + + const expectedModelInfo: ModelInfo = { + ...lMStudioDefaultModelInfo, + description: `${rawModel.displayName} - ${rawModel.path}`, + contextWindow: rawModel.contextLength, + supportsPromptCache: true, + supportsImages: rawModel.vision, + supportsComputerUse: false, + maxTokens: rawModel.contextLength, + inputPrice: 0, + outputPrice: 0, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + } + + const result = parseLMStudioModel(rawModel) + expect(result).toEqual(expectedModelInfo) + }) + }) + + describe("getLMStudioModels", () => { + const baseUrl = "http://localhost:1234" + const lmsUrl = "ws://localhost:1234" + + const mockRawModel: LLMInstanceInfo = { + architecture: "test-arch", + identifier: "mistralai/devstral-small-2505", + instanceReference: "RAP5qbeHVjJgBiGFQ6STCuTJ", + modelKey: "test-model-key-1", + path: "/path/to/test-model-1", + type: "llm", + displayName: "Test Model One", + maxContextLength: 2048, + contextLength: 7161, + paramsString: "1B params, 2k context", + vision: true, + format: "gguf", + sizeBytes: 1000000000, + trainedForToolUse: false, // Added + } + + it("should fetch and parse models successfully", async () => { + mockedAxios.get.mockResolvedValueOnce({ data: { status: "ok" } }) + mockListLoaded.mockResolvedValueOnce([{ getModelInfo: mockGetModelInfo }]) + mockGetModelInfo.mockResolvedValueOnce(mockRawModel) + + const result = await getLMStudioModels(baseUrl) + + expect(mockedAxios.get).toHaveBeenCalledTimes(1) + expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`) + expect(MockedLMStudioClientConstructor).toHaveBeenCalledTimes(1) + expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: lmsUrl }) + expect(mockListLoaded).toHaveBeenCalledTimes(1) + + const expectedParsedModel = parseLMStudioModel(mockRawModel) + expect(result).toEqual({ [mockRawModel.modelKey]: expectedParsedModel }) + }) + + it("should use default baseUrl if an empty string is provided", async () => { + const defaultBaseUrl = "http://localhost:1234" + const defaultLmsUrl = "ws://localhost:1234" + mockedAxios.get.mockResolvedValueOnce({ data: {} }) + mockListLoaded.mockResolvedValueOnce([]) + + await getLMStudioModels("") + + expect(mockedAxios.get).toHaveBeenCalledWith(`${defaultBaseUrl}/v1/models`) + expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: defaultLmsUrl }) + }) + + it("should transform https baseUrl to wss for LMStudioClient", async () => { + const httpsBaseUrl = "https://securehost:4321" + const wssLmsUrl = "wss://securehost:4321" + mockedAxios.get.mockResolvedValueOnce({ data: {} }) + mockListLoaded.mockResolvedValueOnce([]) + + await getLMStudioModels(httpsBaseUrl) + + expect(mockedAxios.get).toHaveBeenCalledWith(`${httpsBaseUrl}/v1/models`) + expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: wssLmsUrl }) + }) + + it("should return an empty object if lmsUrl is unparsable", async () => { + const unparsableBaseUrl = "http://localhost:invalid:port" // Leads to ws://localhost:invalid:port + + const result = await getLMStudioModels(unparsableBaseUrl) + + expect(result).toEqual({}) + expect(mockedAxios.get).not.toHaveBeenCalled() + expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled() + }) + + it("should return an empty object and log error if axios.get fails with a generic error", async () => { + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}) + const networkError = new Error("Network connection failed") + mockedAxios.get.mockRejectedValueOnce(networkError) + + const result = await getLMStudioModels(baseUrl) + + expect(mockedAxios.get).toHaveBeenCalledTimes(1) + expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`) + expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled() + expect(mockListLoaded).not.toHaveBeenCalled() + expect(consoleErrorSpy).toHaveBeenCalledWith( + `Error fetching LMStudio models: ${JSON.stringify(networkError, Object.getOwnPropertyNames(networkError), 2)}`, + ) + expect(result).toEqual({}) + consoleErrorSpy.mockRestore() + }) + + it("should return an empty object and log info if axios.get fails with ECONNREFUSED", async () => { + const consoleInfoSpy = vi.spyOn(console, "warn").mockImplementation(() => {}) + const econnrefusedError = new Error("Connection refused") + ;(econnrefusedError as any).code = "ECONNREFUSED" + mockedAxios.get.mockRejectedValueOnce(econnrefusedError) + + const result = await getLMStudioModels(baseUrl) + + expect(mockedAxios.get).toHaveBeenCalledTimes(1) + expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`) + expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled() + expect(mockListLoaded).not.toHaveBeenCalled() + expect(consoleInfoSpy).toHaveBeenCalledWith(`Error connecting to LMStudio at ${baseUrl}`) + expect(result).toEqual({}) + consoleInfoSpy.mockRestore() + }) + + it("should return an empty object and log error if listDownloadedModels fails", async () => { + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}) + const listError = new Error("LMStudio SDK internal error") + + mockedAxios.get.mockResolvedValueOnce({ data: {} }) + mockListLoaded.mockRejectedValueOnce(listError) + + const result = await getLMStudioModels(baseUrl) + + expect(mockedAxios.get).toHaveBeenCalledTimes(1) + expect(MockedLMStudioClientConstructor).toHaveBeenCalledTimes(1) + expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: lmsUrl }) + expect(mockListLoaded).toHaveBeenCalledTimes(1) + expect(consoleErrorSpy).toHaveBeenCalledWith( + `Error fetching LMStudio models: ${JSON.stringify(listError, Object.getOwnPropertyNames(listError), 2)}`, + ) + expect(result).toEqual({}) + consoleErrorSpy.mockRestore() + }) + }) +}) diff --git a/src/api/providers/fetchers/__tests__/ollama.test.ts b/src/api/providers/fetchers/__tests__/ollama.test.ts new file mode 100644 index 0000000000..cada0a4b60 --- /dev/null +++ b/src/api/providers/fetchers/__tests__/ollama.test.ts @@ -0,0 +1,133 @@ +import axios from "axios" +import path from "path" +import { vi, describe, it, expect, beforeEach } from "vitest" +import { getOllamaModels, parseOllamaModel } from "../ollama" +import ollamaModelsData from "./fixtures/ollama-model-details.json" + +// Mock axios +vi.mock("axios") +const mockedAxios = axios as any + +describe("Ollama Fetcher", () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe("parseOllamaModel", () => { + it("should correctly parse Ollama model info", () => { + const modelData = ollamaModelsData["qwen3-2to16:latest"] + const parsedModel = parseOllamaModel(modelData) + + expect(parsedModel).toEqual({ + maxTokens: 40960, + contextWindow: 40960, + supportsImages: false, + supportsComputerUse: false, + supportsPromptCache: true, + inputPrice: 0, + outputPrice: 0, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + description: "Family: qwen3, Context: 40960, Size: 32.8B", + }) + }) + }) + + describe("getOllamaModels", () => { + it("should fetch model list from /api/tags and details for each model from /api/show", async () => { + const baseUrl = "http://localhost:11434" + const modelName = "devstral2to16:latest" + + const mockApiTagsResponse = { + models: [ + { + name: modelName, + model: modelName, + modified_at: "2025-06-03T09:23:22.610222878-04:00", + size: 14333928010, + digest: "6a5f0c01d2c96c687d79e32fdd25b87087feb376bf9838f854d10be8cf3c10a5", + details: { + family: "llama", + families: ["llama"], + format: "gguf", + parameter_size: "23.6B", + parent_model: "", + quantization_level: "Q4_K_M", + }, + }, + ], + } + const mockApiShowResponse = { + license: "Mock License", + modelfile: "FROM /path/to/blob\nTEMPLATE {{ .Prompt }}", + parameters: "num_ctx 4096\nstop_token ", + template: "{{ .System }}USER: {{ .Prompt }}ASSISTANT:", + modified_at: "2025-06-03T09:23:22.610222878-04:00", + details: { + parent_model: "", + format: "gguf", + family: "llama", + families: ["llama"], + parameter_size: "23.6B", + quantization_level: "Q4_K_M", + }, + model_info: { + "ollama.context_length": 4096, + "some.other.info": "value", + }, + capabilities: ["completion"], + } + + mockedAxios.get.mockResolvedValueOnce({ data: mockApiTagsResponse }) + mockedAxios.post.mockResolvedValueOnce({ data: mockApiShowResponse }) + + const result = await getOllamaModels(baseUrl) + + expect(mockedAxios.get).toHaveBeenCalledTimes(1) + expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/api/tags`) + + expect(mockedAxios.post).toHaveBeenCalledTimes(1) + expect(mockedAxios.post).toHaveBeenCalledWith(`${baseUrl}/api/show`, { model: modelName }) + + expect(typeof result).toBe("object") + expect(result).not.toBeInstanceOf(Array) + expect(Object.keys(result).length).toBe(1) + expect(result[modelName]).toBeDefined() + + const expectedParsedDetails = parseOllamaModel(mockApiShowResponse as any) + expect(result[modelName]).toEqual(expectedParsedDetails) + }) + + it("should return an empty list if the initial /api/tags call fails", async () => { + const baseUrl = "http://localhost:11434" + mockedAxios.get.mockRejectedValueOnce(new Error("Network error")) + const consoleInfoSpy = vi.spyOn(console, "error").mockImplementation(() => {}) // Spy and suppress output + + const result = await getOllamaModels(baseUrl) + + expect(mockedAxios.get).toHaveBeenCalledTimes(1) + expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/api/tags`) + expect(mockedAxios.post).not.toHaveBeenCalled() + expect(result).toEqual({}) + }) + + it("should log an info message and return an empty object on ECONNREFUSED", async () => { + const baseUrl = "http://localhost:11434" + const consoleInfoSpy = vi.spyOn(console, "warn").mockImplementation(() => {}) // Spy and suppress output + + const econnrefusedError = new Error("Connection refused") as any + econnrefusedError.code = "ECONNREFUSED" + mockedAxios.get.mockRejectedValueOnce(econnrefusedError) + + const result = await getOllamaModels(baseUrl) + + expect(mockedAxios.get).toHaveBeenCalledTimes(1) + expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/api/tags`) + expect(mockedAxios.post).not.toHaveBeenCalled() + expect(consoleInfoSpy).toHaveBeenCalledWith(`Failed connecting to Ollama at ${baseUrl}`) + expect(result).toEqual({}) + + consoleInfoSpy.mockRestore() // Restore original console.info + }) + }) +}) diff --git a/src/api/providers/fetchers/lmstudio.ts b/src/api/providers/fetchers/lmstudio.ts new file mode 100644 index 0000000000..ea1a590f1e --- /dev/null +++ b/src/api/providers/fetchers/lmstudio.ts @@ -0,0 +1,54 @@ +import { ModelInfo, lMStudioDefaultModelInfo } from "@roo-code/types" +import { LLM, LLMInfo, LLMInstanceInfo, LMStudioClient } from "@lmstudio/sdk" +import axios from "axios" + +export const parseLMStudioModel = (rawModel: LLMInstanceInfo): ModelInfo => { + const modelInfo: ModelInfo = Object.assign({}, lMStudioDefaultModelInfo, { + description: `${rawModel.displayName} - ${rawModel.path}`, + contextWindow: rawModel.contextLength, + supportsPromptCache: true, + supportsImages: rawModel.vision, + supportsComputerUse: false, + maxTokens: rawModel.contextLength, + }) + + return modelInfo +} + +export async function getLMStudioModels(baseUrl = "http://localhost:1234"): Promise> { + // clearing the input can leave an empty string; use the default in that case + baseUrl = baseUrl === "" ? "http://localhost:1234" : baseUrl + + const models: Record = {} + // ws is required to connect using the LMStudio library + const lmsUrl = baseUrl.replace(/^http:\/\//, "ws://").replace(/^https:\/\//, "wss://") + + try { + if (!URL.canParse(lmsUrl)) { + return models + } + + // test the connection to LM Studio first + // errors will be caught further down + await axios.get(`${baseUrl}/v1/models`) + + const client = new LMStudioClient({ baseUrl: lmsUrl }) + const response = (await client.llm.listLoaded().then((models: LLM[]) => { + return Promise.all(models.map((m) => m.getModelInfo())) + })) as Array + + for (const lmstudioModel of response) { + models[lmstudioModel.modelKey] = parseLMStudioModel(lmstudioModel) + } + } catch (error) { + if (error.code === "ECONNREFUSED") { + console.warn(`Error connecting to LMStudio at ${baseUrl}`) + } else { + console.error( + `Error fetching LMStudio models: ${JSON.stringify(error, Object.getOwnPropertyNames(error), 2)}`, + ) + } + } + + return models +} diff --git a/src/api/providers/fetchers/modelCache.ts b/src/api/providers/fetchers/modelCache.ts index 12d636bc46..5956187e41 100644 --- a/src/api/providers/fetchers/modelCache.ts +++ b/src/api/providers/fetchers/modelCache.ts @@ -14,6 +14,9 @@ import { getGlamaModels } from "./glama" import { getUnboundModels } from "./unbound" import { getLiteLLMModels } from "./litellm" import { GetModelsOptions } from "../../../shared/api" +import { getOllamaModels } from "./ollama" +import { getLMStudioModels } from "./lmstudio" + const memoryCache = new NodeCache({ stdTTL: 5 * 60, checkperiod: 5 * 60 }) async function writeModels(router: RouterName, data: ModelRecord) { @@ -68,6 +71,12 @@ export const getModels = async (options: GetModelsOptions): Promise // Type safety ensures apiKey and baseUrl are always provided for litellm models = await getLiteLLMModels(options.apiKey, options.baseUrl) break + case "ollama": + models = await getOllamaModels(options.baseUrl) + break + case "lmstudio": + models = await getLMStudioModels(options.baseUrl) + break default: { // Ensures router is exhaustively checked if RouterName is a strict union const exhaustiveCheck: never = provider diff --git a/src/api/providers/fetchers/ollama.ts b/src/api/providers/fetchers/ollama.ts new file mode 100644 index 0000000000..8de2c1a918 --- /dev/null +++ b/src/api/providers/fetchers/ollama.ts @@ -0,0 +1,100 @@ +import axios from "axios" +import { ModelInfo, ollamaDefaultModelInfo } from "@roo-code/types" +import { z } from "zod" + +const OllamaModelDetailsSchema = z.object({ + family: z.string(), + families: z.array(z.string()), + format: z.string(), + parameter_size: z.string(), + parent_model: z.string(), + quantization_level: z.string(), +}) + +const OllamaModelSchema = z.object({ + details: OllamaModelDetailsSchema, + digest: z.string(), + model: z.string(), + modified_at: z.string(), + name: z.string(), + size: z.number(), +}) + +const OllamaModelInfoResponseSchema = z.object({ + modelfile: z.string(), + parameters: z.string(), + template: z.string(), + details: OllamaModelDetailsSchema, + model_info: z.record(z.string(), z.any()), + capabilities: z.array(z.string()).optional(), +}) + +const OllamaModelsResponseSchema = z.object({ + models: z.array(OllamaModelSchema), +}) + +type OllamaModelsResponse = z.infer + +type OllamaModelInfoResponse = z.infer + +export const parseOllamaModel = (rawModel: OllamaModelInfoResponse): ModelInfo => { + const contextKey = Object.keys(rawModel.model_info).find((k) => k.includes("context_length")) + const contextWindow = + contextKey && typeof rawModel.model_info[contextKey] === "number" ? rawModel.model_info[contextKey] : undefined + + const modelInfo: ModelInfo = Object.assign({}, ollamaDefaultModelInfo, { + description: `Family: ${rawModel.details.family}, Context: ${contextWindow}, Size: ${rawModel.details.parameter_size}`, + contextWindow: contextWindow || ollamaDefaultModelInfo.contextWindow, + supportsPromptCache: true, + supportsImages: rawModel.capabilities?.includes("vision"), + supportsComputerUse: false, + maxTokens: contextWindow || ollamaDefaultModelInfo.contextWindow, + }) + + return modelInfo +} + +export async function getOllamaModels(baseUrl = "http://localhost:11434"): Promise> { + const models: Record = {} + + // clearing the input can leave an empty string; use the default in that case + baseUrl = baseUrl === "" ? "http://localhost:11434" : baseUrl + + try { + if (!URL.canParse(baseUrl)) { + return models + } + + const response = await axios.get(`${baseUrl}/api/tags`) + const parsedResponse = OllamaModelsResponseSchema.safeParse(response.data) + let modelInfoPromises = [] + + if (parsedResponse.success) { + for (const ollamaModel of parsedResponse.data.models) { + modelInfoPromises.push( + axios + .post(`${baseUrl}/api/show`, { + model: ollamaModel.model, + }) + .then((ollamaModelInfo) => { + models[ollamaModel.name] = parseOllamaModel(ollamaModelInfo.data) + }), + ) + } + + await Promise.all(modelInfoPromises) + } else { + console.error(`Error parsing Ollama models response: ${JSON.stringify(parsedResponse.error, null, 2)}`) + } + } catch (error) { + if (error.code === "ECONNREFUSED") { + console.warn(`Failed connecting to Ollama at ${baseUrl}`) + } else { + console.error( + `Error fetching Ollama models: ${JSON.stringify(error, Object.getOwnPropertyNames(error), 2)}`, + ) + } + } + + return models +} diff --git a/src/api/providers/ollama.ts b/src/api/providers/ollama.ts index 7f384e9a98..a7713ba421 100644 --- a/src/api/providers/ollama.ts +++ b/src/api/providers/ollama.ts @@ -1,6 +1,5 @@ import { Anthropic } from "@anthropic-ai/sdk" import OpenAI from "openai" -import axios from "axios" import { type ModelInfo, openAiModelInfoSaneDefaults, DEEP_SEEK_DEFAULT_TEMPERATURE } from "@roo-code/types" @@ -111,17 +110,3 @@ export class OllamaHandler extends BaseProvider implements SingleCompletionHandl } } } - -export async function getOllamaModels(baseUrl = "http://localhost:11434") { - try { - if (!URL.canParse(baseUrl)) { - return [] - } - - const response = await axios.get(`${baseUrl}/api/tags`) - const modelsArray = response.data?.models?.map((model: any) => model.name) || [] - return [...new Set(modelsArray)] - } catch (error) { - return [] - } -} diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index efa49f268d..b85797fb1e 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -2155,6 +2155,8 @@ describe("ClineProvider - Router Models", () => { glama: mockModels, unbound: mockModels, litellm: mockModels, + ollama: {}, + lmstudio: {}, }, }) }) @@ -2197,6 +2199,8 @@ describe("ClineProvider - Router Models", () => { requesty: {}, glama: mockModels, unbound: {}, + ollama: {}, + lmstudio: {}, litellm: {}, }, }) @@ -2216,6 +2220,13 @@ describe("ClineProvider - Router Models", () => { values: { provider: "unbound" }, }) + expect(mockPostMessage).toHaveBeenCalledWith({ + type: "singleRouterModelFetchResponse", + success: false, + error: "Unbound API error", + values: { provider: "unbound" }, + }) + expect(mockPostMessage).toHaveBeenCalledWith({ type: "singleRouterModelFetchResponse", success: false, @@ -2299,6 +2310,8 @@ describe("ClineProvider - Router Models", () => { glama: mockModels, unbound: mockModels, litellm: {}, + ollama: {}, + lmstudio: {}, }, }) }) diff --git a/src/core/webview/__tests__/webviewMessageHandler.spec.ts b/src/core/webview/__tests__/webviewMessageHandler.spec.ts index e15b18ccdb..46ace3ce85 100644 --- a/src/core/webview/__tests__/webviewMessageHandler.spec.ts +++ b/src/core/webview/__tests__/webviewMessageHandler.spec.ts @@ -73,6 +73,8 @@ describe("webviewMessageHandler - requestRouterModels", () => { glama: mockModels, unbound: mockModels, litellm: mockModels, + ollama: {}, + lmstudio: {}, }, }) }) @@ -158,6 +160,8 @@ describe("webviewMessageHandler - requestRouterModels", () => { glama: mockModels, unbound: mockModels, litellm: {}, + ollama: {}, + lmstudio: {}, }, }) }) @@ -193,6 +197,8 @@ describe("webviewMessageHandler - requestRouterModels", () => { glama: mockModels, unbound: {}, litellm: {}, + ollama: {}, + lmstudio: {}, }, }) @@ -222,11 +228,11 @@ describe("webviewMessageHandler - requestRouterModels", () => { it("handles Error objects and string errors correctly", async () => { // Mock providers to fail with different error types mockGetModels - .mockRejectedValueOnce(new Error("Structured error message")) // Error object - .mockRejectedValueOnce("String error message") // String error - .mockRejectedValueOnce({ message: "Object with message" }) // Object error - .mockResolvedValueOnce({}) // Success - .mockResolvedValueOnce({}) // Success + .mockRejectedValueOnce(new Error("Structured error message")) // openrouter + .mockRejectedValueOnce(new Error("Requesty API error")) // requesty + .mockRejectedValueOnce(new Error("Glama API error")) // glama + .mockRejectedValueOnce(new Error("Unbound API error")) // unbound + .mockRejectedValueOnce(new Error("LiteLLM connection failed")) // litellm await webviewMessageHandler(mockClineProvider, { type: "requestRouterModels", @@ -243,16 +249,30 @@ describe("webviewMessageHandler - requestRouterModels", () => { expect(mockClineProvider.postMessageToWebview).toHaveBeenCalledWith({ type: "singleRouterModelFetchResponse", success: false, - error: "String error message", + error: "Requesty API error", values: { provider: "requesty" }, }) expect(mockClineProvider.postMessageToWebview).toHaveBeenCalledWith({ type: "singleRouterModelFetchResponse", success: false, - error: "[object Object]", + error: "Glama API error", values: { provider: "glama" }, }) + + expect(mockClineProvider.postMessageToWebview).toHaveBeenCalledWith({ + type: "singleRouterModelFetchResponse", + success: false, + error: "Unbound API error", + values: { provider: "unbound" }, + }) + + expect(mockClineProvider.postMessageToWebview).toHaveBeenCalledWith({ + type: "singleRouterModelFetchResponse", + success: false, + error: "LiteLLM connection failed", + values: { provider: "litellm" }, + }) }) it("prefers config values over message values for LiteLLM", async () => { diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index c5433497dc..1ad9a060af 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -29,9 +29,7 @@ import { singleCompletionHandler } from "../../utils/single-completion-handler" import { searchCommits } from "../../utils/git" import { exportSettings, importSettings } from "../config/importExport" import { getOpenAiModels } from "../../api/providers/openai" -import { getOllamaModels } from "../../api/providers/ollama" import { getVsCodeLmModels } from "../../api/providers/vscode-lm" -import { getLmStudioModels } from "../../api/providers/lm-studio" import { openMention } from "../mentions" import { TelemetrySetting } from "../../shared/TelemetrySetting" import { getWorkspacePath } from "../../utils/path" @@ -335,6 +333,8 @@ export const webviewMessageHandler = async ( glama: {}, unbound: {}, litellm: {}, + ollama: {}, + lmstudio: {}, } const safeGetModels = async (options: GetModelsOptions): Promise => { @@ -356,6 +356,9 @@ export const webviewMessageHandler = async ( { key: "unbound", options: { provider: "unbound", apiKey: apiConfiguration.unboundApiKey } }, ] + // Don't fetch Ollama and LM Studio models by default anymore + // They have their own specific handlers: requestOllamaModels and requestLmStudioModels + const litellmApiKey = apiConfiguration.litellmApiKey || message?.values?.litellmApiKey const litellmBaseUrl = apiConfiguration.litellmBaseUrl || message?.values?.litellmBaseUrl if (litellmApiKey && litellmBaseUrl) { @@ -372,13 +375,31 @@ export const webviewMessageHandler = async ( }), ) - const fetchedRouterModels: Partial> = { ...routerModels } + const fetchedRouterModels: Partial> = { + ...routerModels, + // Initialize ollama and lmstudio with empty objects since they use separate handlers + ollama: {}, + lmstudio: {}, + } results.forEach((result, index) => { const routerName = modelFetchPromises[index].key // Get RouterName using index if (result.status === "fulfilled") { fetchedRouterModels[routerName] = result.value.models + + // Ollama and LM Studio settings pages still need these events + if (routerName === "ollama" && Object.keys(result.value.models).length > 0) { + provider.postMessageToWebview({ + type: "ollamaModels", + ollamaModels: Object.keys(result.value.models), + }) + } else if (routerName === "lmstudio" && Object.keys(result.value.models).length > 0) { + provider.postMessageToWebview({ + type: "lmStudioModels", + lmStudioModels: Object.keys(result.value.models), + }) + } } else { // Handle rejection: Post a specific error message for this provider const errorMessage = result.reason instanceof Error ? result.reason.message : String(result.reason) @@ -399,7 +420,50 @@ export const webviewMessageHandler = async ( type: "routerModels", routerModels: fetchedRouterModels as Record, }) + + break + case "requestOllamaModels": { + // Specific handler for Ollama models only + const { apiConfiguration: ollamaApiConfig } = await provider.getState() + try { + const ollamaModels = await getModels({ + provider: "ollama", + baseUrl: ollamaApiConfig.ollamaBaseUrl, + }) + + if (Object.keys(ollamaModels).length > 0) { + provider.postMessageToWebview({ + type: "ollamaModels", + ollamaModels: Object.keys(ollamaModels), + }) + } + } catch (error) { + // Silently fail - user hasn't configured Ollama yet + console.debug("Ollama models fetch failed:", error) + } break + } + case "requestLmStudioModels": { + // Specific handler for LM Studio models only + const { apiConfiguration: lmStudioApiConfig } = await provider.getState() + try { + const lmStudioModels = await getModels({ + provider: "lmstudio", + baseUrl: lmStudioApiConfig.lmStudioBaseUrl, + }) + + if (Object.keys(lmStudioModels).length > 0) { + provider.postMessageToWebview({ + type: "lmStudioModels", + lmStudioModels: Object.keys(lmStudioModels), + }) + } + } catch (error) { + // Silently fail - user hasn't configured LM Studio yet + console.debug("LM Studio models fetch failed:", error) + } + break + } case "requestOpenAiModels": if (message?.values?.baseUrl && message?.values?.apiKey) { const openAiModels = await getOpenAiModels( @@ -411,16 +475,6 @@ export const webviewMessageHandler = async ( provider.postMessageToWebview({ type: "openAiModels", openAiModels }) } - break - case "requestOllamaModels": - const ollamaModels = await getOllamaModels(message.text) - // TODO: Cache like we do for OpenRouter, etc? - provider.postMessageToWebview({ type: "ollamaModels", ollamaModels }) - break - case "requestLmStudioModels": - const lmStudioModels = await getLmStudioModels(message.text) - // TODO: Cache like we do for OpenRouter, etc? - provider.postMessageToWebview({ type: "lmStudioModels", lmStudioModels }) break case "requestVsCodeLmModels": const vsCodeLmModels = await getVsCodeLmModels() diff --git a/src/package.json b/src/package.json index ef8a8f197a..bd24274ab9 100644 --- a/src/package.json +++ b/src/package.json @@ -369,6 +369,7 @@ "@aws-sdk/client-bedrock-runtime": "^3.779.0", "@aws-sdk/credential-providers": "^3.806.0", "@google/genai": "^1.0.0", + "@lmstudio/sdk": "^1.1.1", "@mistralai/mistralai": "^1.3.6", "@modelcontextprotocol/sdk": "^1.9.0", "@qdrant/js-client-rest": "^1.14.0", diff --git a/src/shared/api.ts b/src/shared/api.ts index 8ad8828658..d1bfa2794b 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -6,7 +6,7 @@ export type ApiHandlerOptions = Omit // RouterName -const routerNames = ["openrouter", "requesty", "glama", "unbound", "litellm"] as const +const routerNames = ["openrouter", "requesty", "glama", "unbound", "litellm", "ollama", "lmstudio"] as const export type RouterName = (typeof routerNames)[number] @@ -82,3 +82,5 @@ export type GetModelsOptions = | { provider: "requesty"; apiKey?: string } | { provider: "unbound"; apiKey?: string } | { provider: "litellm"; apiKey: string; baseUrl: string } + | { provider: "ollama"; baseUrl?: string } + | { provider: "lmstudio"; baseUrl?: string } diff --git a/webview-ui/src/components/settings/ApiOptions.tsx b/webview-ui/src/components/settings/ApiOptions.tsx index c55999efbd..c5cd3f1e42 100644 --- a/webview-ui/src/components/settings/ApiOptions.tsx +++ b/webview-ui/src/components/settings/ApiOptions.tsx @@ -162,9 +162,9 @@ const ApiOptions = ({ }, }) } else if (selectedProvider === "ollama") { - vscode.postMessage({ type: "requestOllamaModels", text: apiConfiguration?.ollamaBaseUrl }) + vscode.postMessage({ type: "requestOllamaModels" }) } else if (selectedProvider === "lmstudio") { - vscode.postMessage({ type: "requestLmStudioModels", text: apiConfiguration?.lmStudioBaseUrl }) + vscode.postMessage({ type: "requestLmStudioModels" }) } else if (selectedProvider === "vscode-lm") { vscode.postMessage({ type: "requestVsCodeLmModels" }) } else if (selectedProvider === "litellm") { diff --git a/webview-ui/src/components/settings/providers/LMStudio.tsx b/webview-ui/src/components/settings/providers/LMStudio.tsx index 9177457039..17af44871b 100644 --- a/webview-ui/src/components/settings/providers/LMStudio.tsx +++ b/webview-ui/src/components/settings/providers/LMStudio.tsx @@ -1,4 +1,4 @@ -import { useCallback, useState } from "react" +import { useCallback, useState, useMemo } from "react" import { useEvent } from "react-use" import { Trans } from "react-i18next" import { Checkbox } from "vscrui" @@ -8,6 +8,7 @@ import type { ProviderSettings } from "@roo-code/types" import { useAppTranslation } from "@src/i18n/TranslationContext" import { ExtensionMessage } from "@roo/ExtensionMessage" +import { useRouterModels } from "@src/components/ui/hooks/useRouterModels" import { inputEventTransform } from "../transforms" @@ -20,6 +21,7 @@ export const LMStudio = ({ apiConfiguration, setApiConfigurationField }: LMStudi const { t } = useAppTranslation() const [lmStudioModels, setLmStudioModels] = useState([]) + const routerModels = useRouterModels() const handleInputChange = useCallback( ( @@ -47,6 +49,48 @@ export const LMStudio = ({ apiConfiguration, setApiConfigurationField }: LMStudi useEvent("message", onMessage) + // Check if the selected model exists in the fetched models + const modelNotAvailable = useMemo(() => { + const selectedModel = apiConfiguration?.lmStudioModelId + if (!selectedModel) return false + + // Check if model exists in local LM Studio models + if (lmStudioModels.length > 0 && lmStudioModels.includes(selectedModel)) { + return false // Model is available locally + } + + // If we have router models data for LM Studio + if (routerModels.data?.lmstudio) { + const availableModels = Object.keys(routerModels.data.lmstudio) + // Show warning if model is not in the list (regardless of how many models there are) + return !availableModels.includes(selectedModel) + } + + // If neither source has loaded yet, don't show warning + return false + }, [apiConfiguration?.lmStudioModelId, routerModels.data, lmStudioModels]) + + // Check if the draft model exists + const draftModelNotAvailable = useMemo(() => { + const draftModel = apiConfiguration?.lmStudioDraftModelId + if (!draftModel) return false + + // Check if model exists in local LM Studio models + if (lmStudioModels.length > 0 && lmStudioModels.includes(draftModel)) { + return false // Model is available locally + } + + // If we have router models data for LM Studio + if (routerModels.data?.lmstudio) { + const availableModels = Object.keys(routerModels.data.lmstudio) + // Show warning if model is not in the list (regardless of how many models there are) + return !availableModels.includes(draftModel) + } + + // If neither source has loaded yet, don't show warning + return false + }, [apiConfiguration?.lmStudioDraftModelId, routerModels.data, lmStudioModels]) + return ( <> + {modelNotAvailable && ( +
+
+
+
+ {t("settings:validation.modelAvailability", { modelId: apiConfiguration?.lmStudioModelId })} +
+
+
+ )} {lmStudioModels.length > 0 && ( {t("settings:providers.lmStudio.draftModelDesc")}
+ {draftModelNotAvailable && ( +
+
+
+
+ {t("settings:validation.modelAvailability", { + modelId: apiConfiguration?.lmStudioDraftModelId, + })} +
+
+
+ )}
{lmStudioModels.length > 0 && ( <> diff --git a/webview-ui/src/components/settings/providers/Ollama.tsx b/webview-ui/src/components/settings/providers/Ollama.tsx index 27fd2a5189..e118f68b46 100644 --- a/webview-ui/src/components/settings/providers/Ollama.tsx +++ b/webview-ui/src/components/settings/providers/Ollama.tsx @@ -1,4 +1,4 @@ -import { useState, useCallback } from "react" +import { useState, useCallback, useMemo } from "react" import { useEvent } from "react-use" import { VSCodeTextField, VSCodeRadioGroup, VSCodeRadio } from "@vscode/webview-ui-toolkit/react" @@ -7,6 +7,7 @@ import type { ProviderSettings } from "@roo-code/types" import { ExtensionMessage } from "@roo/ExtensionMessage" import { useAppTranslation } from "@src/i18n/TranslationContext" +import { useRouterModels } from "@src/components/ui/hooks/useRouterModels" import { inputEventTransform } from "../transforms" @@ -19,6 +20,7 @@ export const Ollama = ({ apiConfiguration, setApiConfigurationField }: OllamaPro const { t } = useAppTranslation() const [ollamaModels, setOllamaModels] = useState([]) + const routerModels = useRouterModels() const handleInputChange = useCallback( ( @@ -46,6 +48,27 @@ export const Ollama = ({ apiConfiguration, setApiConfigurationField }: OllamaPro useEvent("message", onMessage) + // Check if the selected model exists in the fetched models + const modelNotAvailable = useMemo(() => { + const selectedModel = apiConfiguration?.ollamaModelId + if (!selectedModel) return false + + // Check if model exists in local ollama models + if (ollamaModels.length > 0 && ollamaModels.includes(selectedModel)) { + return false // Model is available locally + } + + // If we have router models data for Ollama + if (routerModels.data?.ollama) { + const availableModels = Object.keys(routerModels.data.ollama) + // Show warning if model is not in the list (regardless of how many models there are) + return !availableModels.includes(selectedModel) + } + + // If neither source has loaded yet, don't show warning + return false + }, [apiConfiguration?.ollamaModelId, routerModels.data, ollamaModels]) + return ( <> + {modelNotAvailable && ( +
+
+
+
+ {t("settings:validation.modelAvailability", { modelId: apiConfiguration?.ollamaModelId })} +
+
+
+ )} {ollamaModels.length > 0 && ( { requesty: {}, unbound: {}, litellm: {}, + ollama: {}, + lmstudio: {}, } const allowAllOrganization: OrganizationAllowList = { diff --git a/webview-ui/src/utils/validate.ts b/webview-ui/src/utils/validate.ts index 9fc393eafe..1040b60c2d 100644 --- a/webview-ui/src/utils/validate.ts +++ b/webview-ui/src/utils/validate.ts @@ -227,6 +227,12 @@ export function validateModelId(apiConfiguration: ProviderSettings, routerModels case "requesty": modelId = apiConfiguration.requestyModelId break + case "ollama": + modelId = apiConfiguration.ollamaModelId + break + case "lmstudio": + modelId = apiConfiguration.lmStudioModelId + break case "litellm": modelId = apiConfiguration.litellmModelId break