better retry util, retry on fail to get full page content in extract

Author: anerli

.changeset/shy-ants-tease.md

@@ -0,0 +1,5 @@
+---
+"magnitude-core": patch
+---
+
+retry get full page content in extract

packages/magnitude-core/src/agent/browserAgent.ts

@@ -9,6 +9,7 @@ import { renderMinimalAccessibilityTree } from "@/web/util";
 import { narrateAgent, narrateBrowserAgent } from "./narrator";
 import { PartitionOptions, partitionHtml, MarkdownSerializerOptions, serializeToMarkdown } from 'magnitude-extract';
 import EventEmitter from "eventemitter3";
+import { retry } from "@/common/retry";
 
 // export interface StartAgentWithWebOptions {
 //     agentBaseOptions?: Partial<AgentOptions>;
@@ -119,7 +120,10 @@ export class BrowserAgent extends Agent {
     async extract<T extends Schema>(instructions: string, schema: T): Promise<z.infer<T>> {
         this.browserAgentEvents.emit('extractStarted', instructions, schema);
         //const htmlContent = await this.page.content();
-        const htmlContent = await getFullPageContent(this.page);
+        const htmlContent = await retry(
+            async () => await getFullPageContent(this.page),
+            { retries: 5, delay: 200, exponential: true }
+        );
         // const accessibilityTree = await this.page.accessibility.snapshot({ interestingOnly: true });
         // const pageRepr = renderMinimalAccessibilityTree(accessibilityTree);
 

packages/magnitude-core/src/common/retry.ts

@@ -0,0 +1,69 @@
+type RetryOptions = {
+    retries?: number;
+    delay?: number;
+    maxDelay?: number;
+    exponential?: boolean | number; // true = 2x, or custom multiplier
+    throwOnExhaustion?: boolean;
+    retryIf?: (error: Error) => boolean;
+    onRetry?: (error: Error, attempt: number) => void;
+};
+
+export async function retry<T>(
+    fn: () => Promise<T>,
+    options: RetryOptions & { throwOnExhaustion: false }
+): Promise<T | null>;
+
+export async function retry<T>(
+    fn: () => Promise<T>,
+    options?: RetryOptions & { throwOnExhaustion?: true }
+): Promise<T>;
+
+export async function retry<T>(
+    fn: () => Promise<T>,
+    options: RetryOptions = {}
+): Promise<T | null> {
+    const {
+        retries = 3,
+        delay = 0,
+        maxDelay = Infinity,
+        exponential = false,
+        throwOnExhaustion = true, // Now defaults to true
+        retryIf = () => true,
+        onRetry
+    } = options;
+
+    const multiplier = exponential === true ? 2 : exponential || 1;
+
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            return await fn();
+        } catch (error) {
+            if (!(error instanceof Error)) {
+                throw new Error(`Non-Error thrown: ${String(error)}`);
+            }
+
+            if (attempt === retries || !retryIf(error)) {
+                if (throwOnExhaustion) {
+                    throw error;
+                }
+                return null;
+            }
+
+            if (onRetry) {
+                onRetry(error, attempt + 1);
+            }
+
+            if (delay > 0) {
+                const currentDelay = Math.min(
+                    delay * Math.pow(multiplier, attempt),
+                    maxDelay
+                );
+                await new Promise(resolve => setTimeout(resolve, currentDelay));
+            }
+        }
+    }
+
+    // Unreachable
+    return null as any;
+}
+