Skip to content

Commit e7864a8

Browse files
brianjlaibrianjlai
committed
fix tests that now receive DefaultStream instead of DeclarativeStream
1 parent 16e186f commit e7864a8

File tree

1 file changed

+28
-149
lines changed

1 file changed

+28
-149
lines changed

‎unit_tests/sources/declarative/test_concurrent_declarative_source.py‎

Lines changed: 28 additions & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -2391,8 +2391,8 @@ def test_valid_manifest(self):
23912391

23922392
streams = source.streams({})
23932393
assert len(streams) == 2
2394-
assert isinstance(streams[0], DeclarativeStream)
2395-
assert isinstance(streams[1], DeclarativeStream)
2394+
assert isinstance(streams[0], DefaultStream)
2395+
assert isinstance(streams[1], DefaultStream)
23962396
assert (
23972397
source.resolved_manifest["description"]
23982398
== "This is a sample source connector that is very valid."
@@ -2992,140 +2992,6 @@ def test_source_with_no_external_spec_and_no_in_yaml_spec_fails(self):
29922992
with pytest.raises(FileNotFoundError):
29932993
source.spec(logger)
29942994

2995-
@patch("airbyte_cdk.sources.concurrent_source.concurrent_source.ConcurrentSource.read")
2996-
def test_given_debug_when_read_then_set_log_level(self, declarative_source_read):
2997-
any_valid_manifest = {
2998-
"version": "0.29.3",
2999-
"definitions": {
3000-
"schema_loader": {
3001-
"type": "InlineSchemaLoader",
3002-
"schema": {
3003-
"type": "object",
3004-
"properties": {"id": {"type": "string"}},
3005-
},
3006-
},
3007-
"retriever": {
3008-
"paginator": {
3009-
"type": "DefaultPaginator",
3010-
"page_size": 10,
3011-
"page_size_option": {
3012-
"type": "RequestOption",
3013-
"inject_into": "request_parameter",
3014-
"field_name": "page_size",
3015-
},
3016-
"page_token_option": {"type": "RequestPath"},
3017-
"pagination_strategy": {
3018-
"type": "CursorPagination",
3019-
"cursor_value": "{{ response._metadata.next }}",
3020-
},
3021-
},
3022-
"requester": {
3023-
"path": "/v3/marketing/lists",
3024-
"authenticator": {
3025-
"type": "BearerAuthenticator",
3026-
"api_token": "{{ config.apikey }}",
3027-
},
3028-
"request_parameters": {"page_size": "10"},
3029-
},
3030-
"record_selector": {"extractor": {"field_path": ["result"]}},
3031-
},
3032-
},
3033-
"streams": [
3034-
{
3035-
"type": "DeclarativeStream",
3036-
"$parameters": {
3037-
"name": "lists",
3038-
"primary_key": "id",
3039-
"url_base": "https://api.sendgrid.com",
3040-
},
3041-
"schema_loader": {
3042-
"type": "InlineSchemaLoader",
3043-
"schema": {
3044-
"type": "object",
3045-
"properties": {"id": {"type": "string"}},
3046-
},
3047-
},
3048-
"retriever": {
3049-
"paginator": {
3050-
"type": "DefaultPaginator",
3051-
"page_size": 10,
3052-
"page_size_option": {
3053-
"type": "RequestOption",
3054-
"inject_into": "request_parameter",
3055-
"field_name": "page_size",
3056-
},
3057-
"page_token_option": {"type": "RequestPath"},
3058-
"pagination_strategy": {
3059-
"type": "CursorPagination",
3060-
"cursor_value": "{{ response._metadata.next }}",
3061-
"page_size": 10,
3062-
},
3063-
},
3064-
"requester": {
3065-
"path": "/v3/marketing/lists",
3066-
"authenticator": {
3067-
"type": "BearerAuthenticator",
3068-
"api_token": "{{ config.apikey }}",
3069-
},
3070-
"request_parameters": {"page_size": "{{ 10 }}"},
3071-
},
3072-
"record_selector": {"extractor": {"field_path": ["result"]}},
3073-
},
3074-
},
3075-
{
3076-
"type": "DeclarativeStream",
3077-
"$parameters": {
3078-
"name": "stream_with_custom_requester",
3079-
"primary_key": "id",
3080-
"url_base": "https://api.sendgrid.com",
3081-
},
3082-
"schema_loader": {
3083-
"type": "InlineSchemaLoader",
3084-
"schema": {
3085-
"type": "object",
3086-
"properties": {"id": {"type": "string"}},
3087-
},
3088-
},
3089-
"retriever": {
3090-
"paginator": {
3091-
"type": "DefaultPaginator",
3092-
"page_size": 10,
3093-
"page_size_option": {
3094-
"type": "RequestOption",
3095-
"inject_into": "request_parameter",
3096-
"field_name": "page_size",
3097-
},
3098-
"page_token_option": {"type": "RequestPath"},
3099-
"pagination_strategy": {
3100-
"type": "CursorPagination",
3101-
"cursor_value": "{{ response._metadata.next }}",
3102-
"page_size": 10,
3103-
},
3104-
},
3105-
"requester": {
3106-
"type": "CustomRequester",
3107-
"class_name": "unit_tests.sources.declarative.external_component.SampleCustomComponent",
3108-
"path": "/v3/marketing/lists",
3109-
"custom_request_parameters": {"page_size": 10},
3110-
},
3111-
"record_selector": {"extractor": {"field_path": ["result"]}},
3112-
},
3113-
},
3114-
],
3115-
"check": {"type": "CheckStream", "stream_names": ["lists"]},
3116-
}
3117-
3118-
catalog = create_catalog("lists")
3119-
3120-
source = ConcurrentDeclarativeSource(
3121-
source_config=any_valid_manifest, config={}, catalog=catalog, state=None, debug=True
3122-
)
3123-
3124-
debug_logger = logging.getLogger("logger.debug")
3125-
list(source.read(debug_logger, {}, catalog, None))
3126-
3127-
assert debug_logger.isEnabledFor(logging.DEBUG)
3128-
31292995
@pytest.mark.parametrize(
31302996
"is_sandbox, expected_stream_count",
31312997
[
@@ -3280,13 +3146,13 @@ def test_conditional_streams_manifest(self, is_sandbox, expected_stream_count):
32803146

32813147
actual_streams = source.streams(config=config)
32823148
assert len(actual_streams) == expected_stream_count
3283-
assert isinstance(actual_streams[0], DeclarativeStream)
3149+
assert isinstance(actual_streams[0], DefaultStream)
32843150
assert actual_streams[0].name == "students"
32853151

32863152
if is_sandbox:
3287-
assert isinstance(actual_streams[1], DeclarativeStream)
3153+
assert isinstance(actual_streams[1], DefaultStream)
32883154
assert actual_streams[1].name == "classrooms"
3289-
assert isinstance(actual_streams[2], DeclarativeStream)
3155+
assert isinstance(actual_streams[2], DefaultStream)
32903156
assert actual_streams[2].name == "clubs"
32913157

32923158
assert (
@@ -4161,24 +4027,37 @@ def test_only_parent_streams_use_cache():
41614027
assert len(streams) == 3
41624028

41634029
# Main stream with caching (parent for substream `applications_interviews`)
4164-
assert streams[0].name == "applications"
4165-
assert streams[0].retriever.requester.use_cache
4030+
stream_0 = streams[0]
4031+
assert stream_0.name == "applications"
4032+
assert isinstance(stream_0, DefaultStream)
4033+
assert stream_0._stream_partition_generator._partition_factory._retriever.requester.use_cache
41664034

41674035
# Substream
4168-
assert streams[1].name == "applications_interviews"
4169-
assert not streams[1].retriever.requester.use_cache
4036+
stream_1 = streams[1]
4037+
assert stream_1.name == "applications_interviews"
4038+
assert isinstance(stream_1, DefaultStream)
4039+
assert (
4040+
not stream_1._stream_partition_generator._partition_factory._retriever.requester.use_cache
4041+
)
41704042

41714043
# Parent stream created for substream
4172-
assert streams[1].retriever.stream_slicer.parent_stream_configs[0].stream.name == "applications"
41734044
assert (
4174-
streams[1]
4175-
.retriever.stream_slicer.parent_stream_configs[0]
4176-
.stream.retriever.requester.use_cache
4045+
stream_1._stream_partition_generator._partition_factory._retriever.stream_slicer.parent_stream_configs[
4046+
0
4047+
].stream.name
4048+
== "applications"
41774049
)
4050+
assert stream_1._stream_partition_generator._partition_factory._retriever.stream_slicer.parent_stream_configs[
4051+
0
4052+
].stream.retriever.requester.use_cache
41784053

41794054
# Main stream without caching
4180-
assert streams[2].name == "jobs"
4181-
assert not streams[2].retriever.requester.use_cache
4055+
stream_2 = streams[2]
4056+
assert stream_2.name == "jobs"
4057+
assert isinstance(stream_2, DefaultStream)
4058+
assert (
4059+
not stream_2._stream_partition_generator._partition_factory._retriever.requester.use_cache
4060+
)
41824061

41834062

41844063
def _run_read(manifest: Mapping[str, Any], stream_name: str) -> List[AirbyteMessage]:

0 commit comments

Comments
 (0)