Den 418 improve cache retrieval and testing (#67)

* Added impossible as status and improved speed and reliability by checking caches for extract and get_element first. Also tracks if url changes with helps avoid sending wrong page information to the server due to race conditions. Fixed issue in screenshot manager and improved prompts for click and fill.

Finally, refactored get_element and extract_mixins.

* small refactor

* Added missing DTO to async api

* add docstring to extract

* update mixin to not include util functions

* fix use async instead of sync DTO

* refactor browserless config to remote module

---------

Co-authored-by: Arian Hanifi <[email protected]>

Author: charlesmaddock

dendrite_sdk/async_api/_api/browser_api_client.py

@@ -1,6 +1,12 @@
 from typing import Optional
 
 from loguru import logger
+from dendrite_sdk.async_api._api.response.cache_extract_response import (
+    CacheExtractResponse,
+)
+from dendrite_sdk.async_api._api.response.selector_cache_response import (
+    SelectorCacheResponse,
+)
 from dendrite_sdk.async_api._core.models.authentication import AuthSession
 from dendrite_sdk.async_api._api.response.get_element_response import GetElementResponse
 from dendrite_sdk.async_api._api.dto.ask_page_dto import AskPageDTO
@@ -19,6 +25,7 @@
 from dendrite_sdk._common._exceptions.dendrite_exception import (
     InvalidAuthSessionError,
 )
+from dendrite_sdk.async_api._api.dto.get_elements_dto import CheckSelectorCacheDTO
 
 
 class BrowserAPIClient(HTTPClient):
@@ -38,6 +45,14 @@ async def upload_auth_session(self, dto: UploadAuthSessionDTO):
             "actions/upload-auth-session", data=dto.dict(), method="POST"
         )
 
+    async def check_selector_cache(
+        self, dto: CheckSelectorCacheDTO
+    ) -> SelectorCacheResponse:
+        res = await self.send_request(
+            "actions/check-selector-cache", data=dto.dict(), method="POST"
+        )
+        return SelectorCacheResponse(**res.json())
+
     async def get_interactions_selector(
         self, dto: GetElementsDTO
     ) -> GetElementResponse:
@@ -55,6 +70,12 @@ async def make_interaction(self, dto: MakeInteractionDTO) -> InteractionResponse
             status=res_dict["status"], message=res_dict["message"]
         )
 
+    async def check_extract_cache(self, dto: ExtractDTO) -> CacheExtractResponse:
+        res = await self.send_request(
+            "actions/check-extract-cache", data=dto.dict(), method="POST"
+        )
+        return CacheExtractResponse(**res.json())
+
     async def extract(self, dto: ExtractDTO) -> ExtractResponse:
         res = await self.send_request(
             "actions/extract-page", data=dto.dict(), method="POST"

dendrite_sdk/async_api/_api/dto/get_elements_dto.py

@@ -5,9 +5,15 @@
 from dendrite_sdk.async_api._core.models.page_information import PageInformation
 
 
+class CheckSelectorCacheDTO(BaseModel):
+    url: str
+    prompt: Union[str, Dict[str, str]]
+
+
 class GetElementsDTO(BaseModel):
     page_information: PageInformation
     prompt: Union[str, Dict[str, str]]
     api_config: APIConfig
     use_cache: bool = True
     only_one: bool
+    force_use_cache: bool = False

dendrite_sdk/async_api/_api/response/cache_extract_response.py

@@ -0,0 +1,5 @@
+from pydantic import BaseModel
+
+
+class CacheExtractResponse(BaseModel):
+    exists: bool

dendrite_sdk/async_api/_api/response/get_element_response.py

@@ -9,3 +9,4 @@ class GetElementResponse(BaseModel):
     status: Status
     selectors: Optional[Union[List[str], Dict[str, List[str]]]] = None
     message: str = ""
+    used_cache: bool = False

dendrite_sdk/async_api/_api/response/selector_cache_response.py

@@ -0,0 +1,5 @@
+from pydantic import BaseModel
+
+
+class SelectorCacheResponse(BaseModel):
+    exists: bool

dendrite_sdk/async_api/_common/status.py

@@ -1,4 +1,4 @@
 from typing import Literal
 
 
-Status = Literal["success", "failed", "loading"]
+Status = Literal["success", "failed", "loading", "impossible"]

dendrite_sdk/async_api/_core/_impl_mapping.py

@@ -4,10 +4,9 @@
 
 from dendrite_sdk.async_api._ext_impl.browserbase._impl import BrowserBaseImpl
 from dendrite_sdk.async_api._ext_impl.browserless._impl import BrowserlessImpl
-from dendrite_sdk.async_api._ext_impl.browserless._settings import BrowserlessConfig
-from dendrite_sdk.remote import Providers
+from dendrite_sdk.remote.browserless_config import BrowserlessConfig
 from dendrite_sdk.remote.browserbase_config import BrowserbaseConfig
-
+from dendrite_sdk.remote import Providers
 
 IMPL_MAPPING: Dict[Type[Providers], Type[ImplBrowser]] = {
     BrowserbaseConfig: BrowserBaseImpl,

dendrite_sdk/async_api/_core/_managers/navigation_tracker.py

@@ -0,0 +1,78 @@
+import asyncio
+import time
+
+from typing import TYPE_CHECKING, Dict, Optional
+
+if TYPE_CHECKING:
+    from dendrite_sdk.async_api._core.dendrite_page import AsyncPage
+
+
+class NavigationTracker:
+    def __init__(
+        self,
+        page: "AsyncPage",
+    ):
+        self.playwright_page = page.playwright_page
+        self._nav_start_timestamp: Optional[float] = None
+
+        # Track all navigation-related events
+        self.playwright_page.on("framenavigated", self._on_frame_navigated)
+        self.playwright_page.on("popup", self._on_popup)
+
+        # Store last event times
+        self._last_events: Dict[str, Optional[float]] = {
+            "framenavigated": None,
+            "popup": None,
+        }
+
+    def _on_frame_navigated(self, frame):
+        self._last_events["framenavigated"] = time.time()
+        if frame is self.playwright_page.main_frame:
+            self._last_main_frame_url = frame.url
+            self._last_frame_navigated_timestamp = time.time()
+
+    def _on_popup(self, page):
+        self._last_events["popup"] = time.time()
+
+    def start_nav_tracking(self):
+        """Call this just before performing an action that might trigger navigation"""
+        self._nav_start_timestamp = time.time()
+        # Reset event timestamps
+        for event in self._last_events:
+            self._last_events[event] = None
+
+    def get_nav_events_since_start(self):
+        """
+        Returns which events have fired since start_nav_tracking() was called
+        and how long after the start they occurred
+        """
+        if self._nav_start_timestamp is None:
+            return "Navigation tracking not started. Call start_nav_tracking() first."
+
+        results = {}
+        for event, timestamp in self._last_events.items():
+            if timestamp is not None:
+                delay = timestamp - self._nav_start_timestamp
+                results[event] = f"{delay:.3f}s"
+            else:
+                results[event] = "not fired"
+
+        return results
+
+    async def has_navigated_since_start(self):
+        """Returns True if any navigation event has occurred since start_nav_tracking()"""
+        if self._nav_start_timestamp is None:
+            return False
+
+        start_time = time.time()
+        max_wait = 1.0  # Maximum wait time in seconds
+
+        while time.time() - start_time < max_wait:
+            if any(
+                timestamp is not None and timestamp > self._nav_start_timestamp
+                for timestamp in self._last_events.values()
+            ):
+                return True
+            await asyncio.sleep(0.1)
+
+        return False

dendrite_sdk/async_api/_core/_managers/screenshot_manager.py

@@ -16,15 +16,14 @@ async def take_full_page_screenshot(self) -> str:
             # Check the page height
             scroll_height = await self.page.evaluate(
                 """
-                Math.max(
-                    document.body.scrollHeight,
-                    document.documentElement.scrollHeight,
-                    document.body.offsetHeight,
-                    document.documentElement.offsetHeight,
-                    document.body.clientHeight,
-                    document.documentElement.clientHeight
-                )
-            """
+                () => {
+                    const body = document.body;
+                    if (!body) {
+                        return 0;  // Return 0 if body is null
+                    }
+                    return body.scrollHeight || 0;
+                }
+                """
             )
 
             if scroll_height > 30000:

dendrite_sdk/async_api/_core/dendrite_element.py

@@ -13,6 +13,7 @@
 
 if TYPE_CHECKING:
     from dendrite_sdk.async_api._core.dendrite_browser import AsyncDendrite
+from dendrite_sdk.async_api._core._managers.navigation_tracker import NavigationTracker
 from dendrite_sdk.async_api._core.models.page_diff_information import (
     PageDiffInformation,
 )
@@ -46,10 +47,6 @@ async def wrapper(
         ) -> InteractionResponse:
             expected_outcome: Optional[str] = kwargs.pop("expected_outcome", None)
 
-            logger.info(
-                f'Performing action "{interaction_type}" | element: d_id:"{self.dendrite_id}" {self.locator}'
-            )
-
             if not expected_outcome:
                 await func(self, *args, **kwargs)
                 return InteractionResponse(status="success", message="")
@@ -146,7 +143,11 @@ async def screenshot(self) -> str:
 
     @perform_action("click")
     async def click(
-        self, expected_outcome: Optional[str] = None, *args, **kwargs
+        self,
+        expected_outcome: Optional[str] = None,
+        wait_for_navigation: bool = True,
+        *args,
+        **kwargs,
     ) -> InteractionResponse:
         """
         Click the element.
@@ -163,6 +164,10 @@ async def click(
         timeout = kwargs.pop("timeout", 2000)
         force = kwargs.pop("force", False)
 
+        page = await self._dendrite_browser.get_active_page()
+        navigation_tracker = NavigationTracker(page)
+        navigation_tracker.start_nav_tracking()
+
         try:
             await self.locator.click(timeout=timeout, force=force, *args, **kwargs)
         except Exception as e:
@@ -171,15 +176,26 @@ async def click(
             except Exception as e:
                 await self.locator.dispatch_event("click", timeout=2000)
 
+        if wait_for_navigation:
+            has_navigated = await navigation_tracker.has_navigated_since_start()
+            if has_navigated:
+                try:
+                    start_time = time.time()
+                    await page.playwright_page.wait_for_load_state("load", timeout=2000)
+                    wait_duration = time.time() - start_time
+                    print(f"Waited {wait_duration:.2f} seconds for load state")
+                except Exception as e:
+                    print(f"Page navigated but failed to wait for load state: {e}")
+
         return InteractionResponse(status="success", message="")
 
     @perform_action("fill")
     async def fill(
         self, value: str, expected_outcome: Optional[str] = None, *args, **kwargs
     ) -> InteractionResponse:
         """
-        Fill the element with a value. If an expected outcome is provided, the LLM will be used to verify the outcome and raise an exception if the outcome is not as expected.
-        All additional arguments are passed to the Playwright fill method.
+        Fill the element with a value. If the element itself is not fillable,
+        it attempts to find and fill a fillable child element.
 
         Args:
             value (str): The value to fill the element with.
@@ -192,7 +208,15 @@ async def fill(
         """
 
         timeout = kwargs.pop("timeout", 2000)
-        await self.locator.fill(value, timeout=timeout, *args, **kwargs)
+        try:
+            # First, try to fill the element directly
+            await self.locator.fill(value, timeout=timeout, *args, **kwargs)
+        except Exception as e:
+            # If direct fill fails, try to find a fillable child element
+            fillable_child = self.locator.locator(
+                'input, textarea, [contenteditable="true"]'
+            ).first
+            await fillable_child.fill(value, timeout=timeout, *args, **kwargs)
 
         return InteractionResponse(status="success", message="")