fix: sanitize non-ASCII characters in API keys for HTTP headers

- Added sanitizeForHeader() method to replace non-ASCII characters with ?
- Added isAsciiOnly() method to check if string contains only ASCII
- Added warning when API key contains non-ASCII characters
- Added comprehensive tests for API key sanitization
- Fixed ESLint warnings by using charCodeAt instead of regex with control chars

Fixes #7959 - ByteString conversion error with Unicode characters

Author: roomote

src/services/code-index/embedders/__tests__/openai-compatible.spec.ts

@@ -114,6 +114,157 @@ describe("OpenAICompatibleEmbedder", () => {
 				"embeddings:validation.baseUrlRequired",
 			)
 		})
+
+		it("should warn when API key contains non-ASCII characters", () => {
+			const apiKeyWithUnicode = "test-key-•-with-unicode"
+			const warnSpy = vitest.spyOn(console, "warn")
+
+			embedder = new OpenAICompatibleEmbedder(testBaseUrl, apiKeyWithUnicode, testModelId)
+
+			expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("API key contains non-ASCII characters"))
+			expect(embedder).toBeDefined()
+		})
+
+		it("should not warn when API key contains only ASCII characters", () => {
+			const warnSpy = vitest.spyOn(console, "warn")
+
+			embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
+
+			expect(warnSpy).not.toHaveBeenCalledWith(expect.stringContaining("API key contains non-ASCII characters"))
+		})
+	})
+
+	describe("API key sanitization", () => {
+		it("should sanitize non-ASCII characters in API key for direct HTTP requests", async () => {
+			const apiKeyWithUnicode = "test-key-•-with-unicode-§"
+			const sanitizedKey = "test-key-?-with-unicode-?"
+			const fullUrl = "https://api.example.com/v1/embeddings"
+
+			embedder = new OpenAICompatibleEmbedder(fullUrl, apiKeyWithUnicode, testModelId)
+
+			const mockFetch = vitest.fn().mockResolvedValue({
+				ok: true,
+				status: 200,
+				json: async () => ({
+					data: [{ embedding: [0.1, 0.2, 0.3] }],
+					usage: { prompt_tokens: 10, total_tokens: 15 },
+				}),
+				text: async () => "",
+			})
+			global.fetch = mockFetch
+
+			await embedder.createEmbeddings(["test text"])
+
+			expect(mockFetch).toHaveBeenCalledWith(
+				fullUrl,
+				expect.objectContaining({
+					headers: expect.objectContaining({
+						"api-key": sanitizedKey,
+						Authorization: `Bearer ${sanitizedKey}`,
+					}),
+				}),
+			)
+		})
+
+		it("should handle API keys with emoji and special Unicode characters", async () => {
+			const apiKeyWithEmoji = "key-😀-test-™-api"
+			// Emoji (😀) is multi-byte and gets replaced with ?? (one for each byte)
+			const sanitizedKey = "key-??-test-?-api"
+			const fullUrl = "https://api.example.com/v1/embeddings"
+
+			embedder = new OpenAICompatibleEmbedder(fullUrl, apiKeyWithEmoji, testModelId)
+
+			const mockFetch = vitest.fn().mockResolvedValue({
+				ok: true,
+				status: 200,
+				json: async () => ({
+					data: [{ embedding: [0.1, 0.2, 0.3] }],
+					usage: { prompt_tokens: 10, total_tokens: 15 },
+				}),
+				text: async () => "",
+			})
+			global.fetch = mockFetch
+
+			await embedder.createEmbeddings(["test"])
+
+			expect(mockFetch).toHaveBeenCalledWith(
+				fullUrl,
+				expect.objectContaining({
+					headers: expect.objectContaining({
+						"api-key": sanitizedKey,
+						Authorization: `Bearer ${sanitizedKey}`,
+					}),
+				}),
+			)
+		})
+
+		it("should preserve ASCII characters when sanitizing", async () => {
+			const apiKeyMixed = "abc123-•-XYZ789-§-!@#$%^&*()"
+			const sanitizedKey = "abc123-?-XYZ789-?-!@#$%^&*()"
+			const fullUrl = "https://api.example.com/v1/embeddings"
+
+			embedder = new OpenAICompatibleEmbedder(fullUrl, apiKeyMixed, testModelId)
+
+			const mockFetch = vitest.fn().mockResolvedValue({
+				ok: true,
+				status: 200,
+				json: async () => ({
+					data: [{ embedding: [0.1, 0.2, 0.3] }],
+					usage: { prompt_tokens: 10, total_tokens: 15 },
+				}),
+				text: async () => "",
+			})
+			global.fetch = mockFetch
+
+			await embedder.createEmbeddings(["test"])
+
+			expect(mockFetch).toHaveBeenCalledWith(
+				fullUrl,
+				expect.objectContaining({
+					headers: expect.objectContaining({
+						"api-key": sanitizedKey,
+						Authorization: `Bearer ${sanitizedKey}`,
+					}),
+				}),
+			)
+		})
+
+		it("should handle empty API key gracefully", () => {
+			expect(() => new OpenAICompatibleEmbedder(testBaseUrl, "", testModelId)).toThrow(
+				"embeddings:validation.apiKeyRequired",
+			)
+		})
+
+		it("should handle API key that is entirely non-ASCII", async () => {
+			const apiKeyAllUnicode = "•§™€£¥"
+			const sanitizedKey = "??????"
+			const fullUrl = "https://api.example.com/v1/embeddings"
+
+			embedder = new OpenAICompatibleEmbedder(fullUrl, apiKeyAllUnicode, testModelId)
+
+			const mockFetch = vitest.fn().mockResolvedValue({
+				ok: true,
+				status: 200,
+				json: async () => ({
+					data: [{ embedding: [0.1, 0.2, 0.3] }],
+					usage: { prompt_tokens: 10, total_tokens: 15 },
+				}),
+				text: async () => "",
+			})
+			global.fetch = mockFetch
+
+			await embedder.createEmbeddings(["test"])
+
+			expect(mockFetch).toHaveBeenCalledWith(
+				fullUrl,
+				expect.objectContaining({
+					headers: expect.objectContaining({
+						"api-key": sanitizedKey,
+						Authorization: `Bearer ${sanitizedKey}`,
+					}),
+				}),
+			)
+		})
 	})
 
 	describe("embedderInfo", () => {

src/services/code-index/embedders/openai-compatible.ts

@@ -49,6 +49,32 @@ export class OpenAICompatibleEmbedder implements IEmbedder {
 		mutex: new Mutex(),
 	}
 
+	/**
+	 * Sanitizes a string to ensure it only contains ASCII characters suitable for HTTP headers.
+	 * Non-ASCII characters are replaced with '?' to maintain the string structure.
+	 * @param value The string to sanitize
+	 * @returns The sanitized string containing only ASCII characters
+	 */
+	private static sanitizeForHeader(value: string): string {
+		// Replace any non-ASCII characters (code > 127) with '?'
+		// Using charCodeAt to avoid ESLint no-control-regex warning
+		return value
+			.split("")
+			.map((char) => (char.charCodeAt(0) > 127 ? "?" : char))
+			.join("")
+	}
+
+	/**
+	 * Validates if a string contains only ASCII characters.
+	 * @param value The string to validate
+	 * @returns true if the string contains only ASCII characters, false otherwise
+	 */
+	private static isAsciiOnly(value: string): boolean {
+		// Check if all characters have code points <= 127
+		// Using every() to avoid ESLint no-control-regex warning
+		return value.split("").every((char) => char.charCodeAt(0) <= 127)
+	}
+
 	/**
 	 * Creates a new OpenAI Compatible embedder
 	 * @param baseUrl The base URL for the OpenAI-compatible API endpoint
@@ -64,6 +90,14 @@ export class OpenAICompatibleEmbedder implements IEmbedder {
 			throw new Error(t("embeddings:validation.apiKeyRequired"))
 		}
 
+		// Warn if API key contains non-ASCII characters
+		if (!OpenAICompatibleEmbedder.isAsciiOnly(apiKey)) {
+			console.warn(
+				"API key contains non-ASCII characters. These will be replaced with '?' for HTTP header compatibility. " +
+					"Please ensure your API key contains only ASCII characters for proper authentication.",
+			)
+		}
+
 		this.baseUrl = baseUrl
 		this.apiKey = apiKey
 		this.embeddingsClient = new OpenAI({
@@ -195,14 +229,17 @@ export class OpenAICompatibleEmbedder implements IEmbedder {
 		batchTexts: string[],
 		model: string,
 	): Promise<OpenAIEmbeddingResponse> {
+		// Sanitize the API key to ensure it only contains ASCII characters
+		const sanitizedApiKey = OpenAICompatibleEmbedder.sanitizeForHeader(this.apiKey)
+
 		const response = await fetch(url, {
 			method: "POST",
 			headers: {
 				"Content-Type": "application/json",
 				// Azure OpenAI uses 'api-key' header, while OpenAI uses 'Authorization'
 				// We'll try 'api-key' first for Azure compatibility
-				"api-key": this.apiKey,
-				Authorization: `Bearer ${this.apiKey}`,
+				"api-key": sanitizedApiKey,
+				Authorization: `Bearer ${sanitizedApiKey}`,
 			},
 			body: JSON.stringify({
 				input: batchTexts,