[Pending Evals] Update the docs-vectorize to use AI Search as backend

apps/docs-vectorize/CHANGELOG.md → apps/docs-ai-search/CHANGELOG.md

@@ -1,4 +1,13 @@
-# docs-vectorize
+# docs-ai-search
+
+## 0.5.0
+
+### Minor Changes
+
+- Changed backend from Vectorize to AI Search for documentation search
+  - Now uses Cloudflare AI Search (AutoRAG) for contextual search of the Cloudflare Developer Documentation
+  - Maintains full backward compatibility - same XML response format and tool interface
+  - Package renamed from `docs-vectorize` to `docs-ai-search` to reflect the new backend
 
 ## 0.4.3
 

apps/docs-vectorize/README.md → apps/docs-ai-search/README.md

@@ -1,8 +1,8 @@
-# Cloudflare Documentation MCP Server (via Vectorize) 🔭
+# Cloudflare Documentation MCP Server (via AI Search) 🔭
 
-This is a [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction) server that supports remote MCP connections. It connects to a Vectorize DB (in this case, indexed w/ the Cloudflare docs)
+This is a [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction) server that supports remote MCP connections. It uses Cloudflare AI Search (AutoRAG) to provide contextual search of the Cloudflare Developer Documentation.
 
-The Cloudflare account this worker is deployed on already has this Vectorize DB setup and indexed.
+The Cloudflare account this worker is deployed on has an AI Search instance configured with the complete Cloudflare Developer Documentation.
 
 ## 🔨 Available Tools
 
@@ -16,7 +16,7 @@ Currently available tools:
 
 - `Do Cloudflare Workers costs depend on response sizes? I want to serve some images (map tiles) from an R2 bucket and I'm concerned about costs.`
 - `How many indexes are supported in Workers Analytics Engine? Give an example using the Workers binding api.`
-- `Can you give me some information on how to use the Workers AutoRAG binding`
+- `Can you give me some information on how to use the Workers AI Search binding`
 
 ## Access the remote MCP server from any MCP Client
 

apps/docs-vectorize/src/docs-vectorize.app.ts → apps/docs-ai-search/src/docs-ai-search.app.ts

@@ -5,9 +5,9 @@ import { getEnv } from '@repo/mcp-common/src/env'
 import { registerPrompts } from '@repo/mcp-common/src/prompts/docs-vectorize.prompts'
 import { initSentry } from '@repo/mcp-common/src/sentry'
 import { CloudflareMCPServer } from '@repo/mcp-common/src/server'
-import { registerDocsTools } from '@repo/mcp-common/src/tools/docs-vectorize.tools'
+import { registerDocsTools } from '@repo/mcp-common/src/tools/docs-ai-search.tools'
 
-import type { Env } from './docs-vectorize.context'
+import type { Env } from './docs-ai-search.context'
 
 const env = getEnv<Env>()
 

apps/docs-vectorize/src/docs-vectorize.context.ts → apps/docs-ai-search/src/docs-ai-search.context.ts

@@ -1,4 +1,4 @@
-import type { CloudflareDocumentationMCP } from './docs-vectorize.app'
+import type { CloudflareDocumentationMCP } from './docs-ai-search.app'
 
 export interface Env {
 	ENVIRONMENT: 'development' | 'staging' | 'production'

apps/docs-vectorize/vitest.config.ts → apps/docs-ai-search/vitest.config.ts

@@ -1,6 +1,6 @@
 import { defineWorkersConfig } from '@cloudflare/vitest-pool-workers/config'
 
-import type { Env } from './src/docs-vectorize.context'
+import type { Env } from './src/docs-ai-search.context'
 
 export interface TestEnv extends Env {
 	CLOUDFLARE_MOCK_ACCOUNT_ID: string

apps/docs-vectorize/wrangler.jsonc → apps/docs-ai-search/wrangler.jsonc

@@ -4,7 +4,7 @@
  */
 {
 	"$schema": "node_modules/wrangler/config-schema.json",
-	"main": "src/docs-vectorize.app.ts",
+	"main": "src/docs-ai-search.app.ts",
 	"compatibility_date": "2025-03-10",
 	"compatibility_flags": ["nodejs_compat"],
 	"name": "mcp-cloudflare-docs-vectorize-dev",

packages/mcp-common/src/tools/docs-ai-search.tools.ts

@@ -0,0 +1,217 @@
+import { z } from 'zod'
+
+import type { CloudflareMcpAgentNoAccount } from '../types/cloudflare-mcp-agent.types'
+
+interface RequiredEnv {
+	AI: Ai
+}
+
+// Zod schema for AI Search response validation
+const AiSearchResponseSchema = z.object({
+	object: z.string(),
+	search_query: z.string(),
+	data: z.array(z.object({
+		file_id: z.string(),
+		filename: z.string(),
+		score: z.number(),
+		attributes: z.object({
+			modified_date: z.number().optional(),
+			folder: z.string().optional(),
+		}).catchall(z.any()),
+		content: z.array(z.object({
+			id: z.string(),
+			type: z.string(),
+			text: z.string(),
+		})),
+	})),
+	has_more: z.boolean(),
+	next_page: z.string().nullable(),
+})
+
+
+/**
+ * Registers the docs search tool with the MCP server using AI Search
+ * @param agent The MCP server instance
+ */
+export function registerDocsTools(agent: CloudflareMcpAgentNoAccount, env: RequiredEnv) {
+	agent.server.tool(
+		'search_cloudflare_documentation',
+		`Search the Cloudflare documentation.
+
+		This tool should be used to answer any question about Cloudflare products or features, including:
+		- Workers, Pages, R2, Images, Stream, D1, Durable Objects, KV, Workflows, Hyperdrive, Queues
+		- AutoRAG, Workers AI, Vectorize, AI Gateway, Browser Rendering
+		- Zero Trust, Access, Tunnel, Gateway, Browser Isolation, WARP, DDOS, Magic Transit, Magic WAN
+		- CDN, Cache, DNS, Zaraz, Argo, Rulesets, Terraform, Account and Billing
+
+		Results are returned as semantically similar chunks to the query.
+		`,
+		{
+			query: z.string(),
+		},
+		{
+			title: 'Search Cloudflare docs',
+			annotations: {
+				readOnlyHint: true,
+			},
+		},
+		async ({ query }) => {
+			const results = await queryAiSearch(env.AI, query)
+			const resultsAsXml = results
+				.map((result) => {
+					return `<result>
+<url>${result.url}</url>
+<title>${result.title}</title>
+<text>
+${result.text}
+</text>
+</result>`
+				})
+				.join('\n')
+			return {
+				content: [{ type: 'text', text: resultsAsXml }],
+			}
+		}
+	)
+
+	// Note: this is a tool instead of a prompt because
+	// prompt support is much less common than tools.
+	agent.server.tool(
+		'migrate_pages_to_workers_guide',
+		`ALWAYS read this guide before migrating Pages projects to Workers.`,
+		{},
+		{
+			title: 'Get Pages migration guide',
+			annotations: {
+				readOnlyHint: true,
+			},
+		},
+		async () => {
+			const res = await fetch(
+				'https://developers.cloudflare.com/workers/prompts/pages-to-workers.txt',
+				{
+					cf: { cacheEverything: true, cacheTtl: 3600 },
+				}
+			)
+
+			if (!res.ok) {
+				return {
+					content: [{ type: 'text', text: 'Error: Failed to fetch guide. Please try again.' }],
+				}
+			}
+
+			return {
+				content: [
+					{
+						type: 'text',
+						text: await res.text(),
+					},
+				],
+			}
+		}
+	)
+}
+
+async function queryAiSearch(ai: Ai, query: string) {
+	const rawResponse = await doWithRetries(() =>
+		ai.autorag("docs-mcp-rag").search({
+			query,
+		})
+	)
+
+	// Parse and validate the response using Zod
+	const response = AiSearchResponseSchema.parse(rawResponse)
+
+	return response.data.map((item) => ({
+		similarity: item.score,
+		id: item.file_id,
+		url: sourceToUrl(item.filename),
+		title: extractTitle(item.filename),
+		text: item.content.map(c => c.text).join('\n'),
+	}))
+}
+
+function sourceToUrl(filename: string): string {
+	// Convert filename to URL format
+	// Example: "workers/configuration/index.md" -> "https://developers.cloudflare.com/workers/configuration/"
+	return (
+		'https://developers.cloudflare.com/' +
+		filename
+			.replace(/index\.mdx?$/, '')
+			.replace(/\.mdx?$/, '')
+	)
+}
+
+function extractTitle(filename: string): string {
+	// Extract a reasonable title from the filename
+	// Example: "workers/configuration/index.md" -> "Configuration"
+	const parts = filename.replace(/\.mdx?$/, '').split('/')
+	const lastPart = parts[parts.length - 1]
+
+	if (lastPart === 'index') {
+		// Use the parent directory name if filename is index
+		return parts[parts.length - 2] || 'Documentation'
+	}
+
+	// Convert kebab-case or snake_case to title case
+	return lastPart
+		.replace(/[-_]/g, ' ')
+		.replace(/\b\w/g, l => l.toUpperCase())
+}
+
+/**
+ * Retries an action with exponential backoff, only for retryable errors
+ * @template T
+ * @param {() => Promise<T>} action
+ */
+async function doWithRetries<T>(action: () => Promise<T>) {
+	const NUM_RETRIES = 5
+	const INIT_RETRY_MS = 100
+
+	for (let i = 0; i <= NUM_RETRIES; i++) {
+		try {
+			return await action()
+		} catch (e) {
+			// Check if error is retryable (system errors, not user errors)
+			const isRetryable = isRetryableError(e)
+
+			console.error(`AI Search attempt ${i + 1} failed:`, e)
+
+			if (!isRetryable || i === NUM_RETRIES) {
+				throw e
+			}
+
+			// Exponential backoff with jitter
+			const delay = Math.random() * INIT_RETRY_MS * Math.pow(2, i)
+			await scheduler.wait(delay)
+		}
+	}
+	// Should never reach here – last loop iteration should throw
+	throw new Error('An unknown error occurred')
+}
+
+/**
+ * Determines if an error is retryable based on error type and status
+ */
+function isRetryableError(error: unknown): boolean {
+	// Handle HTTP errors from fetch-like responses
+	if (error && typeof error === 'object' && 'status' in error) {
+		const status = (error as { status: number }).status
+		// Retry server errors (5xx) and rate limits (429), not client errors (4xx)
+		return status >= 500 || status === 429
+	}
+
+	// Handle network errors, timeouts, etc.
+	if (error instanceof Error) {
+		const errorMessage = error.message.toLowerCase()
+		return (
+			errorMessage.includes('timeout') ||
+			errorMessage.includes('network') ||
+			errorMessage.includes('connection') ||
+			errorMessage.includes('fetch')
+		)
+	}
+
+	// Default to retryable for unknown errors (conservative approach)
+	return true
+}