include request_url

Author: aldogonzalez8

airbyte_cdk/sources/streams/http/http_client.py

@@ -259,7 +259,7 @@ def _send_with_retry(
         max_tries = max(0, max_retries) + 1
         max_time = self._max_time
 
-        user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries, max_time=max_time)(
+        user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries, max_time=max_time, request_url=request.url)(
             self._send
         )
         rate_limit_backoff_handler = rate_limit_default_backoff_handler(max_tries=max_tries)

airbyte_cdk/sources/streams/http/rate_limiting.py

@@ -101,7 +101,7 @@ def should_give_up(exc: Exception) -> bool:
 
 
 def user_defined_backoff_handler(
-    max_tries: Optional[int], max_time: Optional[int] = None, **kwargs: Any
+    max_tries: Optional[int], max_time: Optional[int] = None, request_url: Optional[str] = None, **kwargs: Any
 ) -> Callable[[SendRequestCallableType], SendRequestCallableType]:
     def sleep_on_ratelimit(details: Mapping[str, Any]) -> None:
         _, exc, _ = sys.exc_info()
@@ -112,7 +112,7 @@ def sleep_on_ratelimit(details: Mapping[str, Any]) -> None:
                 )
             retry_after = exc.backoff
             # include logging og the current time to help with debugging
-            logger.info(f"Retrying. Sleeping for {retry_after} seconds at {time.strftime('%Y-%m-%d %H:%M:%S')}")
+            logger.info(f"Retrying. Sleeping for {retry_after} seconds at {time.strftime('%Y-%m-%d %H:%M:%S')} for URL: {request_url or 'unknown'}")
             time.sleep(retry_after + 1)  # extra second to cover any fractions of second
 
     def log_give_up(details: Mapping[str, Any]) -> None: