Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
81f9ad4
feat: Add Gemini as an embedding provider for codebase indexing
ChuKhaLi May 25, 2025
8137b6d
i18n: Update settings translations
ChuKhaLi May 25, 2025
3738fc3
add missing type for gemini
ChuKhaLi May 27, 2025
d98378f
feat: Add geminiOptions to configuration and remove redundant model I…
ChuKhaLi May 28, 2025
43562bb
feat: Enhance Gemini embedder with token limit handling and model pro…
ChuKhaLi May 28, 2025
8d48e72
feat: Implement rate limit configuration for Gemini embedder and add …
ChuKhaLi May 28, 2025
1cdfce0
feat: Add geminiEmbeddingDimension to configuration and update relate…
ChuKhaLi May 28, 2025
30399ed
feat: Add embedding dimension and select dimension placeholder to loc…
ChuKhaLi May 28, 2025
b0a22b3
fix add missing type defined
ChuKhaLi May 28, 2025
7604759
Update webview-ui/src/i18n/locales/zh-CN/settings.json
ChuKhaLi May 28, 2025
9377f4e
Update webview-ui/src/i18n/locales/ja/settings.json
ChuKhaLi May 28, 2025
b855acf
feat: Add geminiOptions to configuration tests and update model dimen…
ChuKhaLi May 28, 2025
ee15d91
Merge branch 'main' into feat/codebase-indexing-add-gemini-as-provider
ChuKhaLi Jun 10, 2025
1b38421
Removes embedding queue
ChuKhaLi Jun 10, 2025
3f67cc5
Removes unused embedding queue
ChuKhaLi Jun 10, 2025
87a9bfc
Adds tests for CodeIndexConfigManager
ChuKhaLi Jun 10, 2025
383bbf2
Merge branch 'main' into feat/codebase-indexing-add-gemini-as-provider
ChuKhaLi Jun 22, 2025
92ab05a
get locale from main branch
ChuKhaLi Jun 22, 2025
867fa6e
Implement Sliding Window Rate Limiter and Retry Handler for Gemini Em…
ChuKhaLi Jun 22, 2025
8f90433
Remove unnecessary logging in CodeIndexGeminiEmbedder's embedding API…
ChuKhaLi Jun 22, 2025
14117b7
feat(i18n): Add missing Gemini related translations for Indonesian (i…
ChuKhaLi Jun 22, 2025
2149430
feat(code-index): make taskType optional for Gemini embeddings depend…
ChuKhaLi Jun 22, 2025
f5f9257
fix(tests): update config-manager tests to match new implementation
ChuKhaLi Jun 22, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion packages/types/src/codebase-index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@ import { z } from "zod"
export const codebaseIndexConfigSchema = z.object({
codebaseIndexEnabled: z.boolean().optional(),
codebaseIndexQdrantUrl: z.string().optional(),
codebaseIndexEmbedderProvider: z.enum(["openai", "ollama", "openai-compatible"]).optional(),
codebaseIndexEmbedderProvider: z.enum(["openai", "ollama", "openai-compatible", "gemini"]).optional(),
codebaseIndexEmbedderBaseUrl: z.string().optional(),
codebaseIndexEmbedderModelId: z.string().optional(),
geminiEmbeddingTaskType: z.string().optional(),
geminiEmbeddingDimension: z.number().optional(),
})

export type CodebaseIndexConfig = z.infer<typeof codebaseIndexConfigSchema>
Expand All @@ -22,6 +24,7 @@ export const codebaseIndexModelsSchema = z.object({
openai: z.record(z.string(), z.object({ dimension: z.number() })).optional(),
ollama: z.record(z.string(), z.object({ dimension: z.number() })).optional(),
"openai-compatible": z.record(z.string(), z.object({ dimension: z.number() })).optional(),
gemini: z.record(z.string(), z.object({ dimension: z.number() })).optional(),
})

export type CodebaseIndexModels = z.infer<typeof codebaseIndexModelsSchema>
Expand Down
3 changes: 3 additions & 0 deletions packages/types/src/provider-settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,8 @@ const lmStudioSchema = baseProviderSettingsSchema.extend({
const geminiSchema = apiModelIdProviderModelSchema.extend({
geminiApiKey: z.string().optional(),
googleGeminiBaseUrl: z.string().optional(),
geminiEmbeddingTaskType: z.string().optional(),
geminiEmbeddingDimension: z.number().optional(),
})

const openAiNativeSchema = apiModelIdProviderModelSchema.extend({
Expand Down Expand Up @@ -258,6 +260,7 @@ export const providerSettingsSchema = z.object({
})

export type ProviderSettings = z.infer<typeof providerSettingsSchema>

export const PROVIDER_SETTINGS_KEYS = providerSettingsSchema.keyof().options

export const MODEL_ID_KEYS: Partial<keyof ProviderSettings>[] = [
Expand Down
10 changes: 5 additions & 5 deletions scripts/update-contributors.js
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,14 @@ async function readReadme() {
* @param {Array} contributors Array of contributor objects from GitHub API
* @returns {string} HTML for contributors section
*/
const EXCLUDED_LOGIN_SUBSTRINGS = ['[bot]', 'R00-B0T'];
const EXCLUDED_LOGIN_EXACTS = ['cursor', 'roomote'];
const EXCLUDED_LOGIN_SUBSTRINGS = ["[bot]", "R00-B0T"]
const EXCLUDED_LOGIN_EXACTS = ["cursor", "roomote"]

function formatContributorsSection(contributors) {
// Filter out GitHub Actions bot, cursor, and roomote
const filteredContributors = contributors.filter((c) =>
!EXCLUDED_LOGIN_SUBSTRINGS.some(sub => c.login.includes(sub)) &&
!EXCLUDED_LOGIN_EXACTS.includes(c.login)
const filteredContributors = contributors.filter(
(c) =>
!EXCLUDED_LOGIN_SUBSTRINGS.some((sub) => c.login.includes(sub)) && !EXCLUDED_LOGIN_EXACTS.includes(c.login),
)

// Start building with Markdown table format
Expand Down
2 changes: 1 addition & 1 deletion src/api/providers/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ type GeminiHandlerOptions = ApiHandlerOptions & {
export class GeminiHandler extends BaseProvider implements SingleCompletionHandler {
protected options: ApiHandlerOptions

private client: GoogleGenAI
protected client: GoogleGenAI

constructor({ isVertex, ...options }: GeminiHandlerOptions) {
super()
Expand Down
11 changes: 7 additions & 4 deletions src/api/providers/xai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,15 @@ export class XAIHandler extends BaseProvider implements SingleCompletionHandler
if (chunk.usage) {
// Extract detailed token information if available
// First check for prompt_tokens_details structure (real API response)
const promptDetails = "prompt_tokens_details" in chunk.usage ? chunk.usage.prompt_tokens_details : null;
const cachedTokens = promptDetails && "cached_tokens" in promptDetails ? promptDetails.cached_tokens : 0;
const promptDetails = "prompt_tokens_details" in chunk.usage ? chunk.usage.prompt_tokens_details : null
const cachedTokens = promptDetails && "cached_tokens" in promptDetails ? promptDetails.cached_tokens : 0

// Fall back to direct fields in usage (used in test mocks)
const readTokens = cachedTokens || ("cache_read_input_tokens" in chunk.usage ? (chunk.usage as any).cache_read_input_tokens : 0);
const writeTokens = "cache_creation_input_tokens" in chunk.usage ? (chunk.usage as any).cache_creation_input_tokens : 0;
const readTokens =
cachedTokens ||
("cache_read_input_tokens" in chunk.usage ? (chunk.usage as any).cache_read_input_tokens : 0)
const writeTokens =
"cache_creation_input_tokens" in chunk.usage ? (chunk.usage as any).cache_creation_input_tokens : 0

yield {
type: "usage",
Expand Down
118 changes: 92 additions & 26 deletions src/services/code-index/__tests__/config-manager.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ describe("CodeIndexConfigManager", () => {
modelId: undefined,
openAiOptions: { openAiNativeApiKey: "" },
ollamaOptions: { ollamaBaseUrl: "" },
geminiOptions: {
apiModelId: undefined,
geminiApiKey: "",
geminiEmbeddingDimension: undefined,
geminiEmbeddingTaskType: undefined,
rateLimitSeconds: undefined,
},
openAiCompatibleOptions: undefined,
qdrantUrl: "http://localhost:6333",
qdrantApiKey: "",
searchMinScore: 0.4,
Expand Down Expand Up @@ -67,6 +75,20 @@ describe("CodeIndexConfigManager", () => {
modelId: "text-embedding-3-large",
openAiOptions: { openAiNativeApiKey: "test-openai-key" },
ollamaOptions: { ollamaBaseUrl: "" },
geminiOptions: {
apiModelId: "text-embedding-3-large",
geminiApiKey: "",
geminiEmbeddingDimension: undefined,
geminiEmbeddingTaskType: undefined,
rateLimitSeconds: {
codebaseIndexEnabled: true,
codebaseIndexQdrantUrl: "http://qdrant.local",
codebaseIndexEmbedderProvider: "openai",
codebaseIndexEmbedderBaseUrl: "",
codebaseIndexEmbedderModelId: "text-embedding-3-large",
},
},
openAiCompatibleOptions: undefined,
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
searchMinScore: 0.4,
Expand Down Expand Up @@ -101,9 +123,17 @@ describe("CodeIndexConfigManager", () => {
modelId: "text-embedding-3-large",
openAiOptions: { openAiNativeApiKey: "" },
ollamaOptions: { ollamaBaseUrl: "" },
geminiOptions: {
apiModelId: "text-embedding-3-large",
geminiApiKey: "",
geminiEmbeddingDimension: undefined,
geminiEmbeddingTaskType: undefined,
rateLimitSeconds: undefined,
},
openAiCompatibleOptions: {
baseUrl: "https://api.example.com/v1",
apiKey: "test-openai-compatible-key",
modelDimension: undefined,
},
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
Expand Down Expand Up @@ -140,6 +170,13 @@ describe("CodeIndexConfigManager", () => {
modelId: "custom-model",
openAiOptions: { openAiNativeApiKey: "" },
ollamaOptions: { ollamaBaseUrl: "" },
geminiOptions: {
apiModelId: "custom-model",
geminiApiKey: "",
geminiEmbeddingDimension: undefined,
geminiEmbeddingTaskType: undefined,
rateLimitSeconds: undefined,
},
openAiCompatibleOptions: {
baseUrl: "https://api.example.com/v1",
apiKey: "test-openai-compatible-key",
Expand Down Expand Up @@ -180,9 +217,17 @@ describe("CodeIndexConfigManager", () => {
modelId: "custom-model",
openAiOptions: { openAiNativeApiKey: "" },
ollamaOptions: { ollamaBaseUrl: "" },
geminiOptions: {
apiModelId: "custom-model",
geminiApiKey: "",
geminiEmbeddingDimension: undefined,
geminiEmbeddingTaskType: undefined,
rateLimitSeconds: undefined,
},
openAiCompatibleOptions: {
baseUrl: "https://api.example.com/v1",
apiKey: "test-openai-compatible-key",
modelDimension: undefined,
},
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
Expand Down Expand Up @@ -219,6 +264,13 @@ describe("CodeIndexConfigManager", () => {
modelId: "custom-model",
openAiOptions: { openAiNativeApiKey: "" },
ollamaOptions: { ollamaBaseUrl: "" },
geminiOptions: {
apiModelId: "custom-model",
geminiApiKey: "",
geminiEmbeddingDimension: undefined,
geminiEmbeddingTaskType: undefined,
rateLimitSeconds: undefined,
},
openAiCompatibleOptions: {
baseUrl: "https://api.example.com/v1",
apiKey: "test-openai-compatible-key",
Expand Down Expand Up @@ -939,6 +991,19 @@ describe("CodeIndexConfigManager", () => {
modelId: "text-embedding-3-large",
openAiOptions: { openAiNativeApiKey: "test-openai-key" },
ollamaOptions: { ollamaBaseUrl: undefined },
geminiEmbeddingDimension: undefined,
geminiOptions: {
apiModelId: "text-embedding-3-large",
geminiApiKey: "",
geminiEmbeddingDimension: undefined,
geminiEmbeddingTaskType: undefined,
rateLimitSeconds: {
codebaseIndexEnabled: true,
codebaseIndexQdrantUrl: "http://qdrant.local",
codebaseIndexEmbedderProvider: "openai",
codebaseIndexEmbedderModelId: "text-embedding-3-large",
},
},
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
searchMinScore: 0.4,
Expand Down Expand Up @@ -967,69 +1032,70 @@ describe("CodeIndexConfigManager", () => {

describe("initialization and restart prevention", () => {
it("should not require restart when configuration hasn't changed between calls", async () => {
// Setup initial configuration - start with enabled and configured to avoid initial transition restart
mockContextProxy.getGlobalState.mockReturnValue({
const config = {
codebaseIndexEnabled: true,
codebaseIndexQdrantUrl: "http://qdrant.local",
codebaseIndexEmbedderProvider: "openai",
codebaseIndexEmbedderModelId: "text-embedding-3-small",
})
}
mockContextProxy.getGlobalState.mockReturnValue(config)
mockContextProxy.getSecret.mockImplementation((key: string) => {
if (key === "codeIndexOpenAiKey") return "test-key"
return undefined
})

// First load - this will initialize the config manager with current state
await configManager.loadConfiguration()
// First load
const result1 = await configManager.loadConfiguration()
expect(result1.requiresRestart).toBe(true) // Restarts on first valid config

// Second load with same configuration - should not require restart
const secondResult = await configManager.loadConfiguration()
expect(secondResult.requiresRestart).toBe(false)
// Second load with same config
const result2 = await configManager.loadConfiguration()
expect(result2.requiresRestart).toBe(false)
})

it("should properly initialize with current config to prevent false restarts", async () => {
// Setup configuration
mockContextProxy.getGlobalState.mockReturnValue({
codebaseIndexEnabled: false, // Start disabled to avoid transition restart
const config = {
codebaseIndexEnabled: true,
codebaseIndexQdrantUrl: "http://qdrant.local",
codebaseIndexEmbedderProvider: "openai",
codebaseIndexEmbedderModelId: "text-embedding-3-small",
})
}
mockContextProxy.getGlobalState.mockReturnValue(config)
mockContextProxy.getSecret.mockImplementation((key: string) => {
if (key === "codeIndexOpenAiKey") return "test-key"
return undefined
})

// Create a new config manager (simulating what happens in CodeIndexManager.initialize)
const newConfigManager = new CodeIndexConfigManager(mockContextProxy)
// Initialize with the current config
await configManager.loadConfiguration()

// Load configuration - should not require restart since the manager should be initialized with current config
const result = await newConfigManager.loadConfiguration()
// First load should not require restart
const result = await configManager.loadConfiguration()
expect(result.requiresRestart).toBe(false)
})

it("should not require restart when settings are saved but code indexing config unchanged", async () => {
// This test simulates the original issue: handleExternalSettingsChange() being called
// when other settings are saved, but code indexing settings haven't changed

// Setup initial state - enabled and configured
mockContextProxy.getGlobalState.mockReturnValue({
// Initial config
const initialConfig = {
codebaseIndexEnabled: true,
codebaseIndexQdrantUrl: "http://qdrant.local",
codebaseIndexEmbedderProvider: "openai",
codebaseIndexEmbedderModelId: "text-embedding-3-small",
})
}
mockContextProxy.getGlobalState.mockReturnValue(initialConfig)
mockContextProxy.getSecret.mockImplementation((key: string) => {
if (key === "codeIndexOpenAiKey") return "test-key"
return undefined
})

// First load to establish baseline
await configManager.loadConfiguration()

// Simulate external settings change where code indexing config hasn't changed
// (this is what happens when other settings are saved)
const result = await configManager.loadConfiguration()
// Simulate saving settings by creating a new manager and initializing
const newConfigManager = new CodeIndexConfigManager(mockContextProxy)
await newConfigManager.loadConfiguration()

// Load config again with the same settings
const result = await newConfigManager.loadConfiguration()
expect(result.requiresRestart).toBe(false)
})
})
Expand Down
6 changes: 3 additions & 3 deletions src/services/code-index/__tests__/service-factory.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ describe("CodeIndexServiceFactory", () => {
factory.createVectorStore()

// Assert
expect(mockGetModelDimension).toHaveBeenCalledWith("openai", testModelId)
expect(mockGetModelDimension).toHaveBeenCalledWith("openai", testModelId, undefined)
expect(MockedQdrantVectorStore).toHaveBeenCalledWith(
"/test/workspace",
"http://localhost:6333",
Expand All @@ -319,7 +319,7 @@ describe("CodeIndexServiceFactory", () => {
factory.createVectorStore()

// Assert
expect(mockGetModelDimension).toHaveBeenCalledWith("ollama", testModelId)
expect(mockGetModelDimension).toHaveBeenCalledWith("ollama", testModelId, undefined)
expect(MockedQdrantVectorStore).toHaveBeenCalledWith(
"/test/workspace",
"http://localhost:6333",
Expand Down Expand Up @@ -469,7 +469,7 @@ describe("CodeIndexServiceFactory", () => {
factory.createVectorStore()

// Assert
expect(mockGetModelDimension).toHaveBeenCalledWith("openai", "default-model")
expect(mockGetModelDimension).toHaveBeenCalledWith("openai", "default-model", undefined)
expect(MockedQdrantVectorStore).toHaveBeenCalledWith(
"/test/workspace",
"http://localhost:6333",
Expand Down
Loading