From 033b88f15c7e3877bbb099b83a08366905428b87 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 21 Mar 2025 23:40:26 +0100 Subject: [PATCH 1/2] Add pagination to http resolver unit tests --- .../test_http_components_resolver.py | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/unit_tests/sources/declarative/resolvers/test_http_components_resolver.py b/unit_tests/sources/declarative/resolvers/test_http_components_resolver.py index 09d069bff..2f59a6668 100644 --- a/unit_tests/sources/declarative/resolvers/test_http_components_resolver.py +++ b/unit_tests/sources/declarative/resolvers/test_http_components_resolver.py @@ -281,9 +281,21 @@ def to_configured_catalog( }, "record_selector": { "type": "RecordSelector", - "extractor": {"type": "DpathExtractor", "field_path": []}, + "extractor": {"type": "DpathExtractor", "field_path": ["data"]}, + }, + "paginator": { + "type": "DefaultPaginator", + "page_token_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "page_cursor" + }, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": "{{ response.get('next_cursor') }}", + "stop_condition": "{{ not response.get('has_more', False) }}" + } }, - "paginator": {"type": "NoPagination"}, "partition_router": { "type": "SubstreamPartitionRouter", "parent_stream_configs": [ @@ -543,10 +555,26 @@ def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_str HttpRequest(url=f"https://api.test.com/parent/{parent_id}/items"), HttpResponse( body=json.dumps( - [ - {"id": 1, "name": "item_1"}, - {"id": 2, "name": "item_2"}, - ] + { + "data": [ + {"id": 1, "name": "item_1"}, + ], + "has_more": True, + "next_cursor": 1 + } + ) + ), + ) + http_mocker.get( + HttpRequest(url=f"https://api.test.com/parent/{parent_id}/items?page_cursor=1"), + HttpResponse( + body=json.dumps( + { + "data": [ + {"id": 2, "name": "item_2"}, + ], + "has_more": False + } ) ), ) From ab532091b56694343955e8e6520a17598710c64d Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Fri, 21 Mar 2025 22:45:18 +0000 Subject: [PATCH 2/2] Auto-fix lint and format issues --- .../resolvers/test_http_components_resolver.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/unit_tests/sources/declarative/resolvers/test_http_components_resolver.py b/unit_tests/sources/declarative/resolvers/test_http_components_resolver.py index 2f59a6668..3ced049a9 100644 --- a/unit_tests/sources/declarative/resolvers/test_http_components_resolver.py +++ b/unit_tests/sources/declarative/resolvers/test_http_components_resolver.py @@ -288,13 +288,13 @@ def to_configured_catalog( "page_token_option": { "type": "RequestOption", "inject_into": "request_parameter", - "field_name": "page_cursor" + "field_name": "page_cursor", }, "pagination_strategy": { "type": "CursorPagination", "cursor_value": "{{ response.get('next_cursor') }}", - "stop_condition": "{{ not response.get('has_more', False) }}" - } + "stop_condition": "{{ not response.get('has_more', False) }}", + }, }, "partition_router": { "type": "SubstreamPartitionRouter", @@ -560,7 +560,7 @@ def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_str {"id": 1, "name": "item_1"}, ], "has_more": True, - "next_cursor": 1 + "next_cursor": 1, } ) ), @@ -573,7 +573,7 @@ def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_str "data": [ {"id": 2, "name": "item_2"}, ], - "has_more": False + "has_more": False, } ) ),