Merge pull request #8301 from continuedev/pe/fetch

feat: return fetch errors to agent

Author: RomneyDa

core/context/providers/URLContextProvider.ts

@@ -33,39 +33,40 @@ export async function getUrlContextItems(
   query: string,
   fetchFn: FetchFunction,
 ): Promise<ContextItem[]> {
-  try {
-    const url = new URL(query);
-    const icon = await fetchFavicon(url);
-    const resp = await fetchFn(url);
-    const html = await resp.text();
+  const url = new URL(query);
+  const icon = await fetchFavicon(url);
+  const resp = await fetchFn(url);
 
-    const dom = new JSDOM(html);
-    let reader = new Readability(dom.window.document);
-    let article = reader.parse();
-    const content = article?.content || "";
-    const markdown = NodeHtmlMarkdown.translate(
-      content,
-      {},
-      undefined,
-      undefined,
-    );
+  // Check if the response is not OK
+  if (!resp.ok) {
+    throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
+  }
+
+  const html = await resp.text();
 
-    const title = article?.title || url.pathname;
+  const dom = new JSDOM(html);
+  let reader = new Readability(dom.window.document);
+  let article = reader.parse();
+  const content = article?.content || "";
+  const markdown = NodeHtmlMarkdown.translate(
+    content,
+    {},
+    undefined,
+    undefined,
+  );
 
-    return [
-      {
-        icon,
-        description: url.toString(),
-        content: markdown,
-        name: title,
-        uri: {
-          type: "url",
-          value: url.toString(),
-        },
+  const title = article?.title || url.pathname;
+
+  return [
+    {
+      icon,
+      description: url.toString(),
+      content: markdown,
+      name: title,
+      uri: {
+        type: "url",
+        value: url.toString(),
       },
-    ];
-  } catch (e) {
-    console.log(e);
-    return [];
-  }
+    },
+  ];
 }

core/tools/implementations/fetchUrlContent.vitest.ts

@@ -89,3 +89,13 @@ test("fetchUrlContent should add truncation warning with multiple truncated item
   expect(result[1].content.length).toBe(DEFAULT_FETCH_URL_CHAR_LIMIT);
   expect(result[2].name).toBe("Truncation warning");
 });
+
+test("fetchUrlContent should propagate errors when URL fetch fails", async () => {
+  const errorMessage = "HTTP 404 Not Found";
+
+  (getUrlContextItems as any).mockRejectedValue(new Error(errorMessage));
+
+  await expect(
+    fetchUrlContentImpl({ url: "https://example.com/404" }, mockExtras),
+  ).rejects.toThrow(errorMessage);
+});