diff --git a/workflows/examples/custom-action/runner.py b/workflows/examples/custom-action/runner.py new file mode 100644 index 00000000..6696cee0 --- /dev/null +++ b/workflows/examples/custom-action/runner.py @@ -0,0 +1,78 @@ + +import os +import asyncio +import requests +from typing import Optional +from pydantic import BaseModel, Field +from langchain_openai import ChatOpenAI +from browser_use.controller.service import Controller +from workflow_use.workflow.service import Workflow +from workflow_use.workflow.service import ActionResult + +controller = Controller() + +llm_instance = None +try: + llm_instance = ChatOpenAI(model='gpt-4o') +except Exception as e: + print(f'Error initializing LLM: {e}. Would you like to set your OPENAI_API_KEY?') + set_openai_api_key = input('Set OPENAI_API_KEY? (y/n): ') + if set_openai_api_key.lower() == 'y': + os.environ['OPENAI_API_KEY'] = input('Enter your OPENAI_API_KEY: ') + llm_instance = ChatOpenAI(model='gpt-4o') + +class NotificationParams(BaseModel): + error: Optional[str] = Field(default=None, description="Error message") + step: Optional[str] = Field(default=None, description="Step name") + step_description: Optional[str] = Field(default=None, description="Step description") + plan: Optional[str] = Field(default=None, description="Agent's plan to resolve the error") + +@controller.registry.action( + 'Notify Discord of Workflow Error', + param_model=NotificationParams, +) +async def notify_discord_of_workflow_error(params: NotificationParams): + print(f"Notifying Discord of workflow error: {params.error}") + webhook_url = os.getenv("WEBHOOK_URL") + if not webhook_url: + raise ValueError("WEBHOOK_URL environment variable is not set") + + message = f''' + Error occurred in workflow: + ``` +{params.error} + +On step: {params.step} + +Description: {params.step_description} + +Plan: {params.plan} + ``` + ''' + try: + response = requests.post(webhook_url, json={"content": message}, timeout=10) + response.raise_for_status() + except Exception as e: + print(f"Failed to notify Discord: {e}") + + return ActionResult(extracted_content=message, include_in_memory=True) + +async def main(): + agent_custom_instructions = "Before attempting to resolve the step, use the notify discord of workflow error action to notify discord of the error." + workflow = Workflow.load_from_file( + 'examples/custom-action/workflow.json', + llm=llm_instance, + agent_controller=controller, + agent_custom_instructions=agent_custom_instructions, + ) + + ticket_number = '123456789' + license_plate = 'ABC123' + + await workflow.run( + inputs={'ticket_number': ticket_number, 'license_plate': license_plate}, + close_browser_at_end=False, + ) + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/workflows/examples/custom-action/workflow.json b/workflows/examples/custom-action/workflow.json new file mode 100644 index 00000000..d6af0b27 --- /dev/null +++ b/workflows/examples/custom-action/workflow.json @@ -0,0 +1,104 @@ +{ + "workflow_analysis": "The workflow is about checking and paying for a parking ticket using the Philadelphia Parking Authority's online portal. The process involves navigating to the online services hub, accepting the terms of use, searching for a ticket using a ticket number and vehicle details, and viewing details of a ticket. The workflow requires dynamic input such as ticket number and vehicle license plate to search for the ticket.", + "name": "Pay Parking Ticket", + "description": "A workflow to check and pay for a parking ticket through the Philadelphia Parking Authority's online portal.", + "version": "1.0", + "steps": [ + { + "description": "Navigate to the PPA Online Service Hub for parking tickets.", + "output": null, + "timestamp": null, + "tabId": null, + "type": "navigation", + "url": "https://onlineserviceshub.com/ParkingPortal/Philadelphia" + }, + { + "description": "Agree to the PPA Online Violation Payment Services Terms of Use Agreement.", + "output": null, + "timestamp": 1747416499198, + "tabId": 249360593, + "type": "click", + "cssSelector": "XXXXXXXlabel.checkBoxLabel[for=\"termsAgreement\"]", + "xpath": "XXXXXXXid(\"modalContent\")/form[1]/div[1]/div[2]/div[2]/table[1]/tbody[1]/tr[1]/td[2]/label[1]", + "elementTag": "XXXXXXXLABEL", + "elementText": "XXXXXXXI agree to the PPA Online Violation Payment Services Terms of Use Agreement" + }, + { + "description": "Click the Continue button after agreeing to terms.", + "output": null, + "timestamp": 1747416499932, + "tabId": 249360593, + "type": "click", + "cssSelector": "button.button.button--primary.button--full-width.h-100[type=\"submit\"][id=\"disclaimerSubmitBtn\"]", + "xpath": "id(\"disclaimerSubmitBtn\")", + "elementTag": "BUTTON", + "elementText": "Continue" + }, + { + "description": "Select search by Ticket/Citation Number.", + "output": null, + "timestamp": 1747416504966, + "tabId": 249360593, + "type": "select_change", + "cssSelector": "select.material-input.material-text-field__drop-down[id=\"searchBy\"]", + "selectedText": "Ticket/Citation Number", + "xpath": "id(\"searchBy\")", + "elementTag": "SELECT" + }, + { + "description": "Input the ticket number to search for the parking ticket.", + "output": null, + "timestamp": 1747416525782, + "tabId": 249360593, + "type": "input", + "cssSelector": "input.material-input.material-text-field__input[id=\"otherFirstField\"][name=\"OtherFirstField\"][type=\"text\"]", + "value": "{ticket_number}", + "xpath": "id(\"otherFirstField\")", + "elementTag": "INPUT" + }, + { + "description": "Input the vehicle license plate to assist in searching for the parking ticket.", + "output": null, + "timestamp": 1747416528830, + "tabId": 249360593, + "type": "input", + "cssSelector": "input.material-input.material-text-field__input[id=\"otherSecondField\"][name=\"OtherSecondField\"][type=\"text\"]", + "value": "{license_plate}", + "xpath": "id(\"otherSecondField\")", + "elementTag": "INPUT" + }, + { + "description": "Click the reCAPTCHA checkbox (targeting the checkmark element).", + "output": null, + "timestamp": 1747416532064, + "tabId": 249360593, + "type": "click", + "cssSelector": "#recaptcha-anchor .recaptcha-checkbox-checkmark", + "xpath": "//span[@id='recaptcha-anchor']/div[@class='recaptcha-checkbox-checkmark']", + "elementTag": "DIV" + }, + { + "description": "Click the Search button to find the parking ticket.", + "output": null, + "timestamp": 1747416536531, + "tabId": 249360593, + "type": "click", + "cssSelector": "i.fas.fa-external-link-alt", + "xpath": "id(\"openDetails112650268\")/i[1]", + "elementTag": "I", + "elementText": "" + } + ], + "input_schema": [ + { + "name": "ticket_number", + "type": "string", + "required": true + }, + { + "name": "license_plate", + "type": "string", + "required": true + } + ] + } \ No newline at end of file diff --git a/workflows/workflow_use/workflow/prompts.py b/workflows/workflow_use/workflow/prompts.py index f6fd441b..1dcc40f4 100644 --- a/workflows/workflow_use/workflow/prompts.py +++ b/workflows/workflow_use/workflow/prompts.py @@ -20,4 +20,5 @@ "For example, if a click failed, consider navigating to a URL, inputting text, or selecting an option. " "Once the objective of step {step_index} is reached, call the Done action to complete the step. " "Do not proceed to the next step; focus only on completing step {step_index}." + "The user has also provided the following custom instructions: {agent_custom_instructions}\n" ) \ No newline at end of file diff --git a/workflows/workflow_use/workflow/service.py b/workflows/workflow_use/workflow/service.py index 8aaa0d80..0ec3d6e7 100644 --- a/workflows/workflow_use/workflow/service.py +++ b/workflows/workflow_use/workflow/service.py @@ -11,6 +11,7 @@ from browser_use.agent.views import ActionResult, AgentHistoryList from browser_use.browser.browser import Browser from browser_use.browser.context import BrowserContext +from browser_use.controller.service import Controller from langchain.agents import AgentExecutor, create_tool_calling_agent from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.prompts import ChatPromptTemplate @@ -47,6 +48,8 @@ def __init__( browser: Browser | None = None, llm: BaseChatModel | None = None, fallback_to_agent: bool = True, + agent_controller: Controller | None = None, + agent_custom_instructions: str | None = None, ) -> None: """Initialize a new Workflow instance from a schema object. @@ -71,6 +74,8 @@ def __init__( self.browser = browser or Browser() self.llm = llm self.fallback_to_agent = fallback_to_agent + self.agent_controller = agent_controller or Controller() + self.agent_custom_instructions = agent_custom_instructions self.browser_context = BrowserContext(browser=self.browser, config=self.browser.config.new_context_config) @@ -88,12 +93,14 @@ def load_from_file( controller: WorkflowController | None = None, browser: Browser | None = None, llm: BaseChatModel | None = None, + agent_controller: Controller | None = None, + agent_custom_instructions: str | None = None, ) -> Workflow: """Load a workflow from a file.""" with open(file_path, 'r') as f: data = _json.load(f) workflow_schema = WorkflowDefinitionSchema(**data) - return Workflow(workflow_schema=workflow_schema, controller=controller, browser=browser, llm=llm) + return Workflow(workflow_schema=workflow_schema, controller=controller, browser=browser, llm=llm, agent_controller=agent_controller, agent_custom_instructions=agent_custom_instructions) # --- Runners --- async def _run_deterministic_step(self, step: DeterministicWorkflowStep) -> ActionResult: @@ -124,6 +131,7 @@ async def _run_agent_step(self, step: AgenticWorkflowStep) -> AgentHistoryList | llm=self.llm, browser=self.browser, browser_context=self.browser_context, + controller=self.agent_controller, use_vision=True, # Consider making this configurable via WorkflowStep schema ) return await agent.run(max_steps=max_steps) @@ -190,7 +198,8 @@ async def _fallback_to_agent( action_type=failed_action_name, fail_details=fail_details, failed_value=failed_value, - step_description=step_description + step_description=step_description, + agent_custom_instructions=self.agent_custom_instructions, ) logger.info(f'Agent fallback task: {fallback_task}')