Add ability to define custom agent action and instructions + example

workflows/examples/custom-action/runner.py

@@ -0,0 +1,78 @@
+
+import os
+import asyncio
+import requests
+from typing import Optional
+from pydantic import BaseModel, Field
+from langchain_openai import ChatOpenAI
+from browser_use.controller.service import Controller
+from workflow_use.workflow.service import Workflow
+from workflow_use.workflow.service import ActionResult
+
+controller = Controller()
+
+llm_instance = None
+try:
+    llm_instance = ChatOpenAI(model='gpt-4o')
+except Exception as e:
+    print(f'Error initializing LLM: {e}. Would you like to set your OPENAI_API_KEY?')
+    set_openai_api_key = input('Set OPENAI_API_KEY? (y/n): ')
+    if set_openai_api_key.lower() == 'y':
+        os.environ['OPENAI_API_KEY'] = input('Enter your OPENAI_API_KEY: ')
+        llm_instance = ChatOpenAI(model='gpt-4o')
+
+class NotificationParams(BaseModel):
+    error: Optional[str] = Field(default=None, description="Error message")
+    step: Optional[str] = Field(default=None, description="Step name")
+    step_description: Optional[str] = Field(default=None, description="Step description")
+    plan: Optional[str] = Field(default=None, description="Agent's plan to resolve the error")
+
+@controller.registry.action(
+    'Notify Discord of Workflow Error',
+    param_model=NotificationParams,
+)
+async def notify_discord_of_workflow_error(params: NotificationParams):
+    print(f"Notifying Discord of workflow error: {params.error}")
+    webhook_url = os.getenv("WEBHOOK_URL")
+    if not webhook_url:
+        raise ValueError("WEBHOOK_URL environment variable is not set")
+
+    message = f'''
+    Error occurred in workflow:
+    ```
+{params.error}
+
+On step: {params.step}
+
+Description: {params.step_description}
+
+Plan: {params.plan}
+    ```
+    '''
+    try:
+        response = requests.post(webhook_url, json={"content": message}, timeout=10)
+        response.raise_for_status()
+    except Exception as e:
+        print(f"Failed to notify Discord: {e}")
+
+    return ActionResult(extracted_content=message, include_in_memory=True)
+
+async def main():
+    agent_custom_instructions = "Before attempting to resolve the step, use the notify discord of workflow error action to notify discord of the error."
+    workflow = Workflow.load_from_file(
+        'examples/custom-action/workflow.json', 
+        llm=llm_instance,
+        agent_controller=controller,
+        agent_custom_instructions=agent_custom_instructions,
+    )
+
+    ticket_number = '123456789'
+    license_plate = 'ABC123'
+
+    await workflow.run(
+        inputs={'ticket_number': ticket_number, 'license_plate': license_plate},
+        close_browser_at_end=False,
+    )
+
+if __name__ == '__main__':
+    asyncio.run(main())

workflows/examples/custom-action/workflow.json

@@ -0,0 +1,104 @@
+{
+    "workflow_analysis": "The workflow is about checking and paying for a parking ticket using the Philadelphia Parking Authority's online portal. The process involves navigating to the online services hub, accepting the terms of use, searching for a ticket using a ticket number and vehicle details, and viewing details of a ticket. The workflow requires dynamic input such as ticket number and vehicle license plate to search for the ticket.",
+    "name": "Pay Parking Ticket",
+    "description": "A workflow to check and pay for a parking ticket through the Philadelphia Parking Authority's online portal.",
+    "version": "1.0",
+    "steps": [
+      {
+        "description": "Navigate to the PPA Online Service Hub for parking tickets.",
+        "output": null,
+        "timestamp": null,
+        "tabId": null,
+        "type": "navigation",
+        "url": "https://onlineserviceshub.com/ParkingPortal/Philadelphia"
+      },
+      {
+        "description": "Agree to the PPA Online Violation Payment Services Terms of Use Agreement.",
+        "output": null,
+        "timestamp": 1747416499198,
+        "tabId": 249360593,
+        "type": "click",
+        "cssSelector": "XXXXXXXlabel.checkBoxLabel[for=\"termsAgreement\"]",
+        "xpath": "XXXXXXXid(\"modalContent\")/form[1]/div[1]/div[2]/div[2]/table[1]/tbody[1]/tr[1]/td[2]/label[1]",
+        "elementTag": "XXXXXXXLABEL",
+        "elementText": "XXXXXXXI agree to the PPA Online Violation Payment Services Terms of Use Agreement"
+      },
+      {
+        "description": "Click the Continue button after agreeing to terms.",
+        "output": null,
+        "timestamp": 1747416499932,
+        "tabId": 249360593,
+        "type": "click",
+        "cssSelector": "button.button.button--primary.button--full-width.h-100[type=\"submit\"][id=\"disclaimerSubmitBtn\"]",
+        "xpath": "id(\"disclaimerSubmitBtn\")",
+        "elementTag": "BUTTON",
+        "elementText": "Continue"
+      },
+      {
+        "description": "Select search by Ticket/Citation Number.",
+        "output": null,
+        "timestamp": 1747416504966,
+        "tabId": 249360593,
+        "type": "select_change",
+        "cssSelector": "select.material-input.material-text-field__drop-down[id=\"searchBy\"]",
+        "selectedText": "Ticket/Citation Number",
+        "xpath": "id(\"searchBy\")",
+        "elementTag": "SELECT"
+      },
+      {
+        "description": "Input the ticket number to search for the parking ticket.",
+        "output": null,
+        "timestamp": 1747416525782,
+        "tabId": 249360593,
+        "type": "input",
+        "cssSelector": "input.material-input.material-text-field__input[id=\"otherFirstField\"][name=\"OtherFirstField\"][type=\"text\"]",
+        "value": "{ticket_number}",
+        "xpath": "id(\"otherFirstField\")",
+        "elementTag": "INPUT"
+      },
+      {
+        "description": "Input the vehicle license plate to assist in searching for the parking ticket.",
+        "output": null,
+        "timestamp": 1747416528830,
+        "tabId": 249360593,
+        "type": "input",
+        "cssSelector": "input.material-input.material-text-field__input[id=\"otherSecondField\"][name=\"OtherSecondField\"][type=\"text\"]",
+        "value": "{license_plate}",
+        "xpath": "id(\"otherSecondField\")",
+        "elementTag": "INPUT"
+      },
+      {
+        "description": "Click the reCAPTCHA checkbox (targeting the checkmark element).",
+        "output": null,
+        "timestamp": 1747416532064,
+        "tabId": 249360593,
+        "type": "click",
+        "cssSelector": "#recaptcha-anchor .recaptcha-checkbox-checkmark",
+        "xpath": "//span[@id='recaptcha-anchor']/div[@class='recaptcha-checkbox-checkmark']",
+        "elementTag": "DIV"
+      },
+      {
+        "description": "Click the Search button to find the parking ticket.",
+        "output": null,
+        "timestamp": 1747416536531,
+        "tabId": 249360593,
+        "type": "click",
+        "cssSelector": "i.fas.fa-external-link-alt",
+        "xpath": "id(\"openDetails112650268\")/i[1]",
+        "elementTag": "I",
+        "elementText": ""
+      }
+    ],
+    "input_schema": [
+      {
+        "name": "ticket_number",
+        "type": "string",
+        "required": true
+      },
+      {
+        "name": "license_plate",
+        "type": "string",
+        "required": true
+      }
+    ]
+  }

workflows/workflow_use/workflow/prompts.py

@@ -20,4 +20,5 @@
     "For example, if a click failed, consider navigating to a URL, inputting text, or selecting an option. "
     "Once the objective of step {step_index} is reached, call the Done action to complete the step. "
     "Do not proceed to the next step; focus only on completing step {step_index}."
+    "The user has also provided the following custom instructions: {agent_custom_instructions}\n"
 )

workflows/workflow_use/workflow/service.py

@@ -11,6 +11,7 @@
 from browser_use.agent.views import ActionResult, AgentHistoryList
 from browser_use.browser.browser import Browser
 from browser_use.browser.context import BrowserContext
+from browser_use.controller.service import Controller
 from langchain.agents import AgentExecutor, create_tool_calling_agent
 from langchain_core.language_models.chat_models import BaseChatModel
 from langchain_core.prompts import ChatPromptTemplate
@@ -47,6 +48,8 @@ def __init__(
 		browser: Browser | None = None,
 		llm: BaseChatModel | None = None,
 		fallback_to_agent: bool = True,
+		agent_controller: Controller | None = None,
+		agent_custom_instructions: str | None = None,
 	) -> None:
 		"""Initialize a new Workflow instance from a schema object.
 
@@ -71,6 +74,8 @@ def __init__(
 		self.browser = browser or Browser()
 		self.llm = llm
 		self.fallback_to_agent = fallback_to_agent
+		self.agent_controller = agent_controller or Controller()
+		self.agent_custom_instructions = agent_custom_instructions
 
 		self.browser_context = BrowserContext(browser=self.browser, config=self.browser.config.new_context_config)
 
@@ -88,12 +93,14 @@ def load_from_file(
 		controller: WorkflowController | None = None,
 		browser: Browser | None = None,
 		llm: BaseChatModel | None = None,
+		agent_controller: Controller | None = None,
+		agent_custom_instructions: str | None = None,
 	) -> Workflow:
 		"""Load a workflow from a file."""
 		with open(file_path, 'r') as f:
 			data = _json.load(f)
 		workflow_schema = WorkflowDefinitionSchema(**data)
-		return Workflow(workflow_schema=workflow_schema, controller=controller, browser=browser, llm=llm)
+		return Workflow(workflow_schema=workflow_schema, controller=controller, browser=browser, llm=llm, agent_controller=agent_controller, agent_custom_instructions=agent_custom_instructions)
 
 	# --- Runners ---
 	async def _run_deterministic_step(self, step: DeterministicWorkflowStep) -> ActionResult:
@@ -124,6 +131,7 @@ async def _run_agent_step(self, step: AgenticWorkflowStep) -> AgentHistoryList |
 			llm=self.llm,
 			browser=self.browser,
 			browser_context=self.browser_context,
+			controller=self.agent_controller,
 			use_vision=True,  # Consider making this configurable via WorkflowStep schema
 		)
 		return await agent.run(max_steps=max_steps)
@@ -190,7 +198,8 @@ async def _fallback_to_agent(
 			action_type=failed_action_name,
 			fail_details=fail_details,
 			failed_value=failed_value,
-			step_description=step_description
+			step_description=step_description,
+			agent_custom_instructions=self.agent_custom_instructions,
 		)
 		logger.info(f'Agent fallback task: {fallback_task}')