Den 231 improve fast selector agent (#39)

* update wait_for to loading for Status

* switch to take image before getting html

* update timeout for get_element to use increasing intervals

Author: coffecup25

dendrite_sdk/_common/status.py

@@ -1,4 +1,4 @@
 from typing import Literal
 
 
-Status = Literal["success", "failed", "wait_for"]
+Status = Literal["success", "failed", "loading"]

dendrite_sdk/_core/dendrite_page.py

@@ -214,11 +214,11 @@ async def _get_page_information(self) -> PageInformation:
         Returns:
             PageInformation: An object containing the page's URL, raw HTML, and a screenshot in base64 format.
         """
-        soup = await self._get_soup()
 
         base64 = await self.screenshot_manager.take_full_page_screenshot(
             self.playwright_page
         )
+        soup = await self._get_soup()
 
         return PageInformation(
             url=self.playwright_page.url,

dendrite_sdk/_core/mixin/get_element.py

@@ -1,4 +1,5 @@
 import asyncio
+import time
 from typing import Dict, List, Literal, Optional, Union, overload
 
 from loguru import logger
@@ -10,6 +11,10 @@
 from dendrite_sdk._exceptions.dendrite_exception import DendriteException
 
 
+# The timeout interval between retries in milliseconds
+TIMEOUT_INTERVAL = [150, 450, 900]
+
+
 class GetElementMixin(DendritePageProtocol):
     @overload
     async def get_elements(
@@ -240,7 +245,19 @@ async def _get_element(
         """
 
         llm_config = self.dendrite_browser.llm_config
+        attempt_start = time.time()
         for attempt in range(max_retries):
+            current_timeout = (
+                TIMEOUT_INTERVAL[attempt]
+                if len(TIMEOUT_INTERVAL) > attempt
+                else timeout
+            )
+            elapsed_time = time.time() - attempt_start
+            if (current_timeout * 0.001 - elapsed_time) > 0:
+                await asyncio.sleep(current_timeout * 0.001 - elapsed_time)
+
+            attempt_start = time.time()
+
             is_last_attempt = attempt == max_retries - 1
             force_not_use_cache = is_last_attempt
 
@@ -282,7 +299,4 @@ async def _get_element(
                         f"Attempt {attempt + 1}: Failed to get elements from selector, trying again "
                     )
 
-            if not is_last_attempt:
-                await asyncio.sleep(timeout * 0.001)
-
         return None