Creating searchable docs mcp server

Author: codyde

app/api/[transport]/route.ts

@@ -0,0 +1,72 @@
+import {createMcpHandler} from "mcp-handler";
+import {z} from "zod";
+
+import {formatMatchAsBlock, searchIndex} from "../search/searchIndex";
+import {readDocContent} from "../shared/docs-utils";
+
+const handler = createMcpHandler(
+  (server) => {
+    server.tool(
+      "search_docs",
+      "Search the precomputed markdown index and return matching documentation entry points.",
+      {
+        query: z.string().min(1),
+        limit: z.number().int().min(1).max(25).default(5),
+      },
+      async ({query, limit}) => {
+        const matches = await searchIndex(query, limit);
+        const contentText = matches.length
+          ? matches.map(formatMatchAsBlock).join("\n\n")
+          : "No matches found.";
+
+        return {
+          content: [{type: "text", text: contentText}],
+        };
+      }
+    );
+
+    server.tool(
+      "get_doc",
+      "Fetch raw markdown from the documentation exports. Reads local files when available, otherwise fetches from DOCS_PUBLIC_BASE.",
+      {
+        path: z.string().min(1),
+      },
+      async ({path}) => {
+        const content = await readDocContent(path);
+        return {
+          content: [{type: "text", text: content}],
+        };
+      }
+    );
+  },
+  {
+    // Optional server options
+  },
+  {
+    basePath: "/api",
+    maxDuration: 60,
+    verboseLogs: false,
+  }
+);
+
+function normalizeRequest(request: Request): Request {
+  const url = new URL(request.url);
+  if (url.pathname.endsWith("/") && url.pathname.length > 1) {
+    url.pathname = url.pathname.slice(0, -1);
+  }
+
+  return new Request(url.toString(), {
+    method: request.method,
+    headers: request.headers,
+    body: request.body,
+    // @ts-ignore - duplex is needed for streaming
+    duplex: "half",
+  });
+}
+
+function wrappedHandler(request: Request) {
+  const normalizedRequest = normalizeRequest(request);
+  return handler(normalizedRequest);
+}
+
+export {wrappedHandler as GET, wrappedHandler as POST, wrappedHandler as DELETE};

app/api/search/route.ts

@@ -0,0 +1,33 @@
+import {NextRequest, NextResponse} from "next/server";
+
+import {mapMatchToResponse, searchIndex} from "./searchIndex";
+
+export const runtime = "nodejs";
+
+export async function GET(request: NextRequest) {
+  const {searchParams} = new URL(request.url);
+  const query = searchParams.get("q") ?? "";
+  const limitParam = searchParams.get("limit");
+  const limit = limitParam ? Math.min(25, Math.max(1, Number(limitParam))) : 10;
+
+  try {
+    const matches = await searchIndex(query, limit);
+    const results = matches.map(mapMatchToResponse);
+
+    return NextResponse.json({
+      query,
+      limit,
+      count: results.length,
+      results,
+    });
+  } catch (error) {
+    return NextResponse.json(
+      {
+        query,
+        limit,
+        error: error instanceof Error ? error.message : "Unknown error",
+      },
+      {status: 500}
+    );
+  }
+}

app/api/search/searchIndex.ts

@@ -0,0 +1,222 @@
+import {promises as fs} from "node:fs";
+import path from "node:path";
+
+import {buildDocUrl} from "../shared/docs-utils";
+
+const SEARCH_INDEX_PATH = path.join(process.cwd(), "public", "search-index.json");
+
+type RawSearchIndexEntry = {
+  path: string;
+  title: string;
+  hierarchy: string[];
+  summary: string;
+  content: string;
+};
+
+type SearchIndexFile = {
+  generatedAt: string;
+  total: number;
+  entries: RawSearchIndexEntry[];
+};
+
+export type SearchMatch = {
+  path: string;
+  title: string;
+  hierarchy: string[];
+  summary: string;
+  snippet: string | null;
+  score: number;
+  matchedTokens: number;
+};
+
+type CachedEntry = RawSearchIndexEntry & {
+  pathLower: string;
+  titleLower: string;
+  hierarchyLower: string[];
+  contentLower: string;
+};
+
+let searchIndexPromise: Promise<CachedEntry[]> | null = null;
+
+async function loadSearchIndexInternal(): Promise<CachedEntry[]> {
+  const raw = await fs.readFile(SEARCH_INDEX_PATH, "utf8");
+  const parsed = JSON.parse(raw) as SearchIndexFile;
+  return parsed.entries.map(entry => ({
+    ...entry,
+    pathLower: entry.path.toLowerCase(),
+    titleLower: entry.title.toLowerCase(),
+    hierarchyLower: entry.hierarchy.map(segment => segment.toLowerCase()),
+    contentLower: entry.content.toLowerCase(),
+  }));
+}
+
+export async function ensureSearchIndex(): Promise<CachedEntry[]> {
+  if (!searchIndexPromise) {
+    searchIndexPromise = loadSearchIndexInternal().catch(error => {
+      searchIndexPromise = null;
+      throw error;
+    });
+  }
+
+  return searchIndexPromise;
+}
+
+function scoreEntry(entry: CachedEntry, tokens: string[]) {
+  let score = 0;
+  let matchedTokens = 0;
+
+  for (const token of tokens) {
+    let tokenMatched = false;
+
+    if (entry.titleLower.includes(token)) {
+      score += 6;
+      tokenMatched = true;
+    }
+
+    if (entry.pathLower.includes(token)) {
+      score += 4;
+      tokenMatched = true;
+    }
+
+    if (entry.hierarchyLower.some(segment => segment.includes(token))) {
+      score += 3;
+      tokenMatched = true;
+    }
+
+    if (entry.contentLower.includes(token)) {
+      score += 1;
+      tokenMatched = true;
+    }
+
+    if (tokenMatched) {
+      matchedTokens += 1;
+    }
+  }
+
+  if (matchedTokens === 0) {
+    return null;
+  }
+
+  score += getInstallBias(entry);
+
+  return {score, matchedTokens};
+}
+
+function buildSnippet(entry: CachedEntry, tokens: string[]): string | null {
+  const lines = entry.content.split(/\r?\n/);
+  for (const line of lines) {
+    const lineLower = line.toLowerCase();
+    if (tokens.some(token => lineLower.includes(token))) {
+      const trimmed = line.trim();
+      if (trimmed.length === 0) {
+        continue;
+      }
+      return trimmed.length > 200 ? `${trimmed.slice(0, 199)}…` : trimmed;
+    }
+  }
+  return null;
+}
+
+export async function searchIndex(query: string, limit: number): Promise<SearchMatch[]> {
+  const tokens = query
+    .toLowerCase()
+    .split(/\s+/)
+    .map(token => token.trim())
+    .filter(Boolean);
+
+  if (tokens.length === 0) {
+    return [];
+  }
+
+  const entries = await ensureSearchIndex();
+  const matches: SearchMatch[] = [];
+
+  for (const entry of entries) {
+    const scoreResult = scoreEntry(entry, tokens);
+    if (!scoreResult) {
+      continue;
+    }
+
+    matches.push({
+      path: entry.path,
+      title: entry.title,
+      hierarchy: entry.hierarchy,
+      summary: entry.summary,
+      snippet: buildSnippet(entry, tokens),
+      score: scoreResult.score,
+      matchedTokens: scoreResult.matchedTokens,
+    });
+  }
+
+  matches.sort((a, b) => {
+    if (b.score !== a.score) {
+      return b.score - a.score;
+    }
+    if (b.matchedTokens !== a.matchedTokens) {
+      return b.matchedTokens - a.matchedTokens;
+    }
+    return a.path.localeCompare(b.path);
+  });
+
+  return matches.slice(0, limit);
+}
+
+function getInstallBias(entry: CachedEntry): number {
+  const segments = entry.pathLower.split("/");
+  const fileName = segments[segments.length - 1] ?? "";
+  const baseName = fileName.replace(/\.md$/, "");
+
+  let bias = 0;
+
+  // Top-level platform doc like "platforms/react.md"
+  if (segments[0] === "platforms" && segments.length === 2) {
+    bias += 40;
+  }
+
+  // JavaScript guide root doc like "platforms/javascript/guides/react.md"
+  if (
+    segments[0] === "platforms" &&
+    segments[1] === "javascript" &&
+    segments[2] === "guides" &&
+    segments.length === 4
+  ) {
+    bias += 50;
+  }
+
+  // Files under an install directory get a boost
+  if (segments.includes("install")) {
+    bias += 20;
+  }
+
+  // Common install filenames get additional weight
+  if (["install", "installation", "setup", "getting-started"].includes(baseName)) {
+    bias += 25;
+  }
+
+  return bias;
+}
+
+export function formatMatchAsBlock(match: SearchMatch): string {
+  const header = `# ${match.hierarchy.join(" > ")}`;
+  const link = `[${match.title}](${match.path})`;
+  const lines = [header, link];
+
+  if (match.snippet) {
+    lines.push(match.snippet);
+  }
+
+  return lines.join("\n");
+}
+
+export function mapMatchToResponse(match: SearchMatch) {
+  return {
+    path: match.path,
+    title: match.title,
+    hierarchy: match.hierarchy,
+    summary: match.summary,
+    snippet: match.snippet,
+    url: buildDocUrl(match.path),
+    score: match.score,
+    matchedTokens: match.matchedTokens,
+  };
+}

app/api/shared/docs-utils.ts

@@ -0,0 +1,43 @@
+import {promises as fs} from "node:fs";
+import path from "node:path";
+
+export const MD_EXPORTS_ROOT = path.join(process.cwd(), "public", "md-exports");
+export const DOCS_PUBLIC_BASE = process.env.DOCS_PUBLIC_BASE ?? "https://docs.sentry.io";
+
+export function normalizeDocPath(inputPath: string): string {
+  const trimmed = inputPath.trim();
+  const withoutLeadingSlash = trimmed.replace(/^\/+/, "");
+  const normalized = path.normalize(withoutLeadingSlash);
+
+  if (normalized.startsWith("..")) {
+    throw new Error("Invalid doc path: outside allowed directory");
+  }
+
+  return normalized;
+}
+
+export function buildDocUrl(docPath: string): string {
+  const normalized = normalizeDocPath(docPath);
+  const base = DOCS_PUBLIC_BASE.endsWith("/") ? DOCS_PUBLIC_BASE : `${DOCS_PUBLIC_BASE}/`;
+  const url = new URL(normalized, base);
+  return url.toString();
+}
+
+export async function readDocContent(docPath: string): Promise<string> {
+  const normalized = normalizeDocPath(docPath);
+  const localPath = path.join(MD_EXPORTS_ROOT, normalized);
+
+  try {
+    const file = await fs.readFile(localPath, "utf8");
+    return file;
+  } catch (localError) {
+    const url = buildDocUrl(normalized);
+
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch doc from ${url}: ${response.status} ${response.statusText}`);
+    }
+
+    return await response.text();
+  }
+}

package.json

@@ -42,6 +42,7 @@
     "@google-cloud/storage": "^7.7.0",
     "@mdx-js/loader": "^3.0.0",
     "@mdx-js/react": "^3.0.0",
+    "@modelcontextprotocol/sdk": "^1.18.1",
     "@pondorasti/remark-img-links": "^1.0.8",
     "@popperjs/core": "^2.11.8",
     "@prettier/plugin-xml": "^3.3.1",
@@ -70,6 +71,7 @@
     "js-cookie": "^3.0.5",
     "js-yaml": "^4.1.0",
     "match-sorter": "^6.3.4",
+    "mcp-handler": "^1.0.2",
     "mdx-bundler": "^10.0.1",
     "mermaid": "^11.11.0",
     "micromark": "^4.0.0",
@@ -110,7 +112,8 @@
     "tailwindcss-scoped-preflight": "^3.0.4",
     "textarea-markdown-editor": "^1.0.4",
     "unified": "^11.0.5",
-    "unist-util-remove": "^4.0.0"
+    "unist-util-remove": "^4.0.0",
+    "zod": "^3.25.76"
   },
   "devDependencies": {
     "@babel/preset-typescript": "^7.15.0",