fix: url and path joining for cursor paginators (#695)

Author: lmossman

airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -168,7 +168,13 @@ def _get_url(
             next_page_token=next_page_token,
         )
 
-        full_url = self._join_url(url_base, path) if url_base else url + path if path else url
+        full_url = (
+            self._join_url(url_base, path)
+            if url_base
+            else self._join_url(url, path)
+            if path
+            else url
+        )
 
         return full_url
 

airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -159,14 +159,7 @@ def path(
     ) -> Optional[str]:
         token = next_page_token.get("next_page_token") if next_page_token else None
         if token and self.page_token_option and isinstance(self.page_token_option, RequestPath):
-            # make additional interpolation context
-            interpolation_context = get_interpolation_context(
-                stream_state=stream_state,
-                stream_slice=stream_slice,
-                next_page_token=next_page_token,
-            )
-            # Replace url base to only return the path
-            return str(token).replace(self.url_base.eval(self.config, **interpolation_context), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
+            return str(token)
         else:
             return None
 

unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py

@@ -36,7 +36,7 @@
         (
             RequestPath(parameters={}),
             None,
-            "/next_url",
+            "https://airbyte.io/next_url",
             {"limit": 2},
             {},
             {},
@@ -128,7 +128,7 @@
         (
             RequestPath(parameters={}),
             None,
-            "/next_url",
+            "https://airbyte.io/next_url",
             {"limit": 2},
             {},
             {},
@@ -539,42 +539,3 @@ def test_path_returns_none_when_option_not_request_path() -> None:
     )
     result = paginator.path(next_page_token)
     assert result is None
-
-
-def test_path_with_additional_interpolation_context() -> None:
-    page_token_option = RequestPath(parameters={})
-    paginator = DefaultPaginator(
-        pagination_strategy=Mock(),
-        config={},
-        url_base="https://api.domain.com/{{ stream_slice['campaign_id'] }}",
-        parameters={},
-        page_token_option=page_token_option,
-    )
-    # define stream_state here
-    stream_state = {"state": "state_value"}
-    # define stream_slice here
-    stream_slice = StreamSlice(
-        partition={
-            "campaign_id": "123_abcd",
-        },
-        cursor_slice={
-            "start": "A",
-            "end": "B",
-        },
-        extra_fields={
-            "extra_field_A": "value_A",
-            "extra_field_B": "value_B",
-        },
-    )
-    # define next_page_token here
-    next_page_token = {
-        "next_page_token": "https://api.domain.com/123_abcd/some_next_page_token_here"
-    }
-
-    expected_after_interpolation = "/some_next_page_token_here"
-
-    assert expected_after_interpolation == paginator.path(
-        next_page_token=next_page_token,
-        stream_state=stream_state,
-        stream_slice=stream_slice,
-    )