Skip to content

Commit cc1d366

Browse files
idosalidosal
authored
feat: async process indexing (#85)
1 parent 8f3dcb3 commit cc1d366

File tree

8 files changed

+79
-46
lines changed

8 files changed

+79
-46
lines changed

‎src/api/tools/commonTools.ts‎

Lines changed: 41 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@ import { generateServerName } from "../../shared/nameUtils.js";
1717
export async function fetchDocumentation({
1818
repoData,
1919
env,
20+
ctx,
2021
initiatedFromSearch = false,
2122
}: {
2223
repoData: RepoData;
2324
env: any;
25+
ctx: any;
2426
initiatedFromSearch?: boolean;
2527
}): Promise<{
2628
fileUsed: string;
@@ -247,33 +249,7 @@ export async function fetchDocumentation({
247249

248250
// Store documentation in vector database for later search
249251
if (content && owner && repo) {
250-
try {
251-
// First check if vectors exist in KV cache
252-
let vectorsExist = await getIsIndexedFromCache(owner, repo, env);
253-
254-
// Only store vectors if they don't exist yet
255-
if (!vectorsExist) {
256-
// Pass the Vectorize binding from env
257-
await storeDocumentationVectors(
258-
owner,
259-
repo,
260-
content,
261-
fileUsed,
262-
env.VECTORIZE,
263-
);
264-
265-
// Update the cache to indicate vectors now exist
266-
await cacheIsIndexed(owner, repo, true, env);
267-
console.log(`Stored documentation vectors for ${owner}/${repo}`);
268-
} else {
269-
console.log(
270-
`Documentation vectors already exist for ${owner}/${repo}, skipping indexing`,
271-
);
272-
}
273-
} catch (error) {
274-
console.error(`Failed to store documentation vectors: ${error}`);
275-
// Continue despite vector storage failure
276-
}
252+
ctx.waitUntil(indexDocumentation(owner, repo, content, fileUsed, env));
277253
}
278254
}
279255

@@ -293,6 +269,41 @@ export async function fetchDocumentation({
293269
};
294270
}
295271

272+
async function indexDocumentation(
273+
owner: string,
274+
repo: string,
275+
content: string,
276+
fileUsed: string,
277+
env: any,
278+
) {
279+
try {
280+
// First check if vectors exist in KV cache
281+
let vectorsExist = await getIsIndexedFromCache(owner, repo, env);
282+
283+
// Only store vectors if they don't exist yet
284+
if (!vectorsExist) {
285+
// Pass the Vectorize binding from env
286+
await storeDocumentationVectors(
287+
owner,
288+
repo,
289+
content,
290+
fileUsed,
291+
env.VECTORIZE,
292+
);
293+
294+
// Update the cache to indicate vectors now exist
295+
await cacheIsIndexed(owner, repo, true, env);
296+
console.log(`Stored documentation vectors for ${owner}/${repo}`);
297+
} else {
298+
console.log(
299+
`Documentation vectors already exist for ${owner}/${repo}, skipping indexing`,
300+
);
301+
}
302+
} catch (error) {
303+
console.error(`Failed to store documentation vectors: ${error}`);
304+
// Continue despite vector storage failure
305+
}
306+
}
296307
/**
297308
* Search documentation using vector search
298309
* Will fetch and index documentation if none exists
@@ -302,11 +313,13 @@ export async function searchRepositoryDocumentation({
302313
query,
303314
forceReindex = false,
304315
env,
316+
ctx,
305317
}: {
306318
repoData: RepoData;
307319
query: string;
308320
forceReindex?: boolean;
309321
env: any;
322+
ctx: any;
310323
}): Promise<{
311324
searchQuery: string;
312325
content: { type: "text"; text: string }[];
@@ -346,7 +359,7 @@ export async function searchRepositoryDocumentation({
346359
await cacheIsIndexed(owner, repo, false, env);
347360

348361
// Fetch the documentation - pass env
349-
const docResult = await fetchDocumentation({ repoData, env });
362+
const docResult = await fetchDocumentation({ repoData, env, ctx });
350363
const content = docResult.content[0].text;
351364
const fileUsed = docResult.fileUsed;
352365

‎src/api/tools/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ export function getMcpTools(
66
requestHost: string,
77
requestUrl?: string,
88
env?: any,
9+
ctx?: any,
910
): Array<Tool> {
1011
const repoData = getRepoData({ requestHost, requestUrl });
1112
const handler = getHandlerByRepoData(repoData);
12-
return handler.getTools(repoData, env);
13+
return handler.getTools(repoData, env, ctx);
1314
}

‎src/api/tools/repoHandlers/DefaultRepoHandler.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import type { RepoHandler, Tool } from "./RepoHandler.js";
1616

1717
class DefaultRepoHandler implements RepoHandler {
1818
name = "default";
19-
getTools(repoData: RepoData, env: any): Array<Tool> {
19+
getTools(repoData: RepoData, env: any, ctx: any): Array<Tool> {
2020
// Generate a dynamic description based on the URL
2121
const fetchToolName = generateFetchToolName(repoData);
2222
const fetchToolDescription = generateFetchToolDescription(repoData);
@@ -32,7 +32,7 @@ class DefaultRepoHandler implements RepoHandler {
3232
description: fetchToolDescription,
3333
paramsSchema: undefined,
3434
cb: async () => {
35-
return fetchDocumentation({ repoData, env });
35+
return fetchDocumentation({ repoData, env, ctx });
3636
},
3737
},
3838
{
@@ -48,6 +48,7 @@ class DefaultRepoHandler implements RepoHandler {
4848
repoData,
4949
query,
5050
env,
51+
ctx,
5152
});
5253
},
5354
},
@@ -91,24 +92,28 @@ class DefaultRepoHandler implements RepoHandler {
9192
async fetchDocumentation({
9293
repoData,
9394
env,
95+
ctx,
9496
}: {
9597
repoData: RepoData;
9698
env: any;
99+
ctx: any;
97100
}): Promise<{
98101
fileUsed: string;
99102
content: { type: "text"; text: string }[];
100103
}> {
101-
return await fetchDocumentation({ repoData, env });
104+
return await fetchDocumentation({ repoData, env, ctx });
102105
}
103106

104107
async searchRepositoryDocumentation({
105108
repoData,
106109
query,
107110
env,
111+
ctx,
108112
}: {
109113
repoData: RepoData;
110114
query: string;
111115
env: any;
116+
ctx: any;
112117
}): Promise<{
113118
searchQuery: string;
114119
content: { type: "text"; text: string }[];
@@ -117,6 +122,7 @@ class DefaultRepoHandler implements RepoHandler {
117122
repoData,
118123
query,
119124
env,
125+
ctx,
120126
});
121127
}
122128
}

‎src/api/tools/repoHandlers/GenericRepoHandler.ts‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010

1111
class GenericRepoHandler implements RepoHandler {
1212
name = "generic";
13-
getTools(_: RepoData, env?: any): Array<Tool> {
13+
getTools(_: RepoData, env: any, ctx: any): Array<Tool> {
1414
console.debug("Creating tools for docs page");
1515

1616
return [
@@ -79,7 +79,7 @@ class GenericRepoHandler implements RepoHandler {
7979
urlType: "github",
8080
host: "gitmcp.io",
8181
};
82-
return fetchDocumentation({ repoData, env });
82+
return fetchDocumentation({ repoData, env, ctx });
8383
},
8484
},
8585
{
@@ -102,7 +102,7 @@ class GenericRepoHandler implements RepoHandler {
102102
urlType: "github",
103103
host: "gitmcp.io",
104104
};
105-
return searchRepositoryDocumentation({ repoData, query, env });
105+
return searchRepositoryDocumentation({ repoData, query, env, ctx });
106106
},
107107
},
108108
{
@@ -151,29 +151,33 @@ class GenericRepoHandler implements RepoHandler {
151151
async fetchDocumentation({
152152
repoData,
153153
env,
154+
ctx,
154155
}: {
155156
repoData: RepoData;
156-
env?: any;
157+
env: any;
158+
ctx: any;
157159
}): Promise<{
158160
fileUsed: string;
159161
content: { type: "text"; text: string }[];
160162
}> {
161-
return await fetchDocumentation({ repoData, env });
163+
return await fetchDocumentation({ repoData, env, ctx });
162164
}
163165

164166
async searchRepositoryDocumentation({
165167
repoData,
166168
query,
167169
env,
170+
ctx,
168171
}: {
169172
repoData: RepoData;
170173
query: string;
171-
env?: any;
174+
env: any;
175+
ctx: any;
172176
}): Promise<{
173177
searchQuery: string;
174178
content: { type: "text"; text: string }[];
175179
}> {
176-
return await searchRepositoryDocumentation({ repoData, query, env });
180+
return await searchRepositoryDocumentation({ repoData, query, env, ctx });
177181
}
178182
}
179183

‎src/api/tools/repoHandlers/ReactRouterRepoHandler.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ import { getDefaultRepoHandler } from "./DefaultRepoHandler.js";
55

66
class ReactRouterRepoHandler implements RepoHandler {
77
name = "react-router";
8-
getTools(repoData: RepoData, env: any): Array<Tool> {
9-
const defaultTools = getDefaultRepoHandler().getTools(repoData, env);
8+
getTools(repoData: RepoData, env: any, ctx: any): Array<Tool> {
9+
const defaultTools = getDefaultRepoHandler().getTools(repoData, env, ctx);
1010
const searchToolName = generateSearchToolName(repoData);
1111
// filter out the search tool
1212
const tools = defaultTools.filter((tool) => tool.name !== searchToolName);
@@ -16,31 +16,36 @@ class ReactRouterRepoHandler implements RepoHandler {
1616
async fetchDocumentation({
1717
repoData,
1818
env,
19+
ctx,
1920
}: {
2021
repoData: RepoData;
2122
env: any;
23+
ctx: any;
2224
}): Promise<{
2325
fileUsed: string;
2426
content: { type: "text"; text: string }[];
2527
}> {
26-
return await fetchDocumentation({ repoData, env });
28+
return await fetchDocumentation({ repoData, env, ctx });
2729
}
2830

2931
async searchRepositoryDocumentation({
3032
repoData,
3133
query,
3234
env,
35+
ctx,
3336
}: {
3437
repoData: RepoData;
3538
query: string;
3639
env: any;
40+
ctx: any;
3741
}): Promise<{
3842
searchQuery: string;
3943
content: { type: "text"; text: string }[];
4044
}> {
4145
const fetchResult = await fetchDocumentation({
4246
repoData,
4347
env,
48+
ctx,
4449
});
4550
return {
4651
searchQuery: query,

‎src/api/tools/repoHandlers/RepoHandler.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,17 @@ export interface Tool {
99

1010
export interface RepoHandler {
1111
name: string;
12-
getTools(repoData: RepoData, env: any): Array<Tool>;
12+
getTools(repoData: RepoData, env: any, ctx: any): Array<Tool>;
1313

1414
// For the generic MCP to call
1515
fetchDocumentation({
1616
repoData,
1717
env,
18+
ctx,
1819
}: {
1920
repoData: RepoData;
20-
env?: any;
21+
env: any;
22+
ctx: any;
2123
}): Promise<{
2224
fileUsed: string;
2325
content: { type: "text"; text: string }[];

‎src/api/tools/repoHandlers/ThreejsRepoHandler.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import {
88
} from "./threejs/utils.js";
99
class ThreejsRepoHandler implements RepoHandler {
1010
name = "threejs";
11-
getTools(repoData: RepoData, env?: any): Array<Tool> {
11+
getTools(repoData: RepoData, env: any): Array<Tool> {
1212
return [
1313
{
1414
name: "get_reference_docs_list",
@@ -61,6 +61,7 @@ class ThreejsRepoHandler implements RepoHandler {
6161
}: {
6262
repoData: RepoData;
6363
env: any;
64+
ctx: any;
6465
}): Promise<{
6566
fileUsed: string;
6667
content: { type: "text"; text: string }[];

‎src/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,10 @@ export class MyMCP extends McpAgent {
7676

7777
// Access env from this.env (Cloudflare worker environment is accessible here)
7878
const env = this.env;
79+
const ctx = this.ctx;
7980

8081
// Pass env to getMcpTools
81-
getMcpTools(host, canonicalUrl, env).forEach((tool) => {
82+
getMcpTools(host, canonicalUrl, env, ctx).forEach((tool) => {
8283
this.server.tool(
8384
tool.name,
8485
tool.description,

0 commit comments

Comments
 (0)