From 99facbb2c42b4d40f378ede3f48cb29aad75970b Mon Sep 17 00:00:00 2001 From: Marty Zhou Date: Fri, 20 Jun 2025 11:00:44 +0300 Subject: [PATCH 1/3] record iframe interactions --- extension/src/entrypoints/content.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/extension/src/entrypoints/content.ts b/extension/src/entrypoints/content.ts index 8f7f233f..8bb72ef3 100644 --- a/extension/src/entrypoints/content.ts +++ b/extension/src/entrypoints/content.ts @@ -544,6 +544,7 @@ function handleBlur(event: FocusEvent) { export default defineContentScript({ matches: [""], + allFrames: true, main(ctx) { // Listener for status updates from the background script chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { From 22bc8a18763e5b0edcfe126d56fed0508cfe5635 Mon Sep 17 00:00:00 2001 From: Marty Zhou Date: Fri, 20 Jun 2025 11:31:37 +0300 Subject: [PATCH 2/3] matchAboutBlank iframes --- extension/src/entrypoints/content.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/extension/src/entrypoints/content.ts b/extension/src/entrypoints/content.ts index 8bb72ef3..bfa81bdf 100644 --- a/extension/src/entrypoints/content.ts +++ b/extension/src/entrypoints/content.ts @@ -545,6 +545,7 @@ function handleBlur(event: FocusEvent) { export default defineContentScript({ matches: [""], allFrames: true, + matchAboutBlank: true, main(ctx) { // Listener for status updates from the background script chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { From 7a2b8b045f3c72f79409971b8d852f89be5c009b Mon Sep 17 00:00:00 2001 From: Marty Zhou Date: Fri, 20 Jun 2025 13:02:32 +0300 Subject: [PATCH 3/3] ignore about blank navigation step --- workflows/workflow_use/builder/service.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/workflows/workflow_use/builder/service.py b/workflows/workflow_use/builder/service.py index b260ce5a..b855ec2a 100644 --- a/workflows/workflow_use/builder/service.py +++ b/workflows/workflow_use/builder/service.py @@ -187,15 +187,20 @@ async def build_workflow( images_used = 0 for step in input_workflow.steps: step_messages: List[Dict[str, Any]] = [] # Messages for this specific step + + step_dict = step.model_dump(mode='json', exclude_none=True) + step_type = getattr(step, 'type', step_dict.get('type')) + step_url = getattr(step, 'url', step_dict.get('url', '')) + # Skip steps to avoid processing empty or irrelevant navigation steps, mostly from iframes. + if step_type == 'navigation' and step_url == 'about:blank': + continue # 1. Text representation (JSON dump) - step_dict = step.model_dump(mode='json', exclude_none=True) screenshot_data = step_dict.pop('screenshot', None) # Pop potential screenshot step_messages.append({'type': 'text', 'text': json.dumps(step_dict, indent=2)}) # 2. Optional screenshot - attach_image = use_screenshots and images_used < max_images - step_type = getattr(step, 'type', step_dict.get('type')) + attach_image = use_screenshots and images_used < max_images if attach_image and step_type != 'input': # Don't attach for inputs # Re-retrieve screenshot data if it wasn't popped (e.g., nested under 'data')