Skip to content

Commit 925dc88

Browse files
aldogonzalez8aldogonzalez8
committed
ruff format
1 parent a53540d commit 925dc88

File tree

1 file changed

+154
-182
lines changed

1 file changed

+154
-182
lines changed

‎unit_tests/connector_builder/test_property_chunking.py‎

Lines changed: 154 additions & 182 deletions
Original file line numberDiff line numberDiff line change
@@ -29,170 +29,131 @@
2929
FREEZE_DATE = "2025-05-23"
3030

3131
PROPERTY_KEY = "test"
32-
PROPERTY_LIST = [
33-
"one",
34-
"two",
35-
"three",
36-
"four"
37-
]
32+
PROPERTY_LIST = ["one", "two", "three", "four"]
3833

3934
MANIFEST = {
40-
"version": "6.48.15",
41-
"type": "DeclarativeSource",
42-
"check": {
43-
"type": "CheckStream",
44-
"stream_names": [
45-
"Rates"
46-
]
47-
},
48-
"streams": [
49-
{
50-
"type": "DeclarativeStream",
51-
"name": "Rates",
52-
"retriever": {
53-
"type": "SimpleRetriever",
54-
"requester": {
55-
"type": "HttpRequester",
56-
"path": "/exchangerates_data/{{stream_interval.start_time}}",
57-
"url_base": "https://api.apilayer.com",
58-
"http_method": "GET",
59-
"authenticator": {
60-
"type": "ApiKeyAuthenticator",
61-
"api_token": "{{ config['api_key'] }}",
62-
"inject_into": {
63-
"type": "RequestOption",
64-
"field_name": "apikey",
65-
"inject_into": "header"
66-
}
67-
},
68-
"request_parameters": {
69-
"base": "{{ config['base'] }}",
70-
PROPERTY_KEY: {
71-
"type": "QueryProperties",
72-
"property_list": PROPERTY_LIST,
73-
"property_chunking": {
74-
"type": "PropertyChunking",
75-
"property_limit_type": "property_count",
76-
"property_limit": 2
77-
}
78-
}
79-
}
80-
},
81-
"record_selector": {
82-
"type": "RecordSelector",
83-
"extractor": {
84-
"type": "DpathExtractor",
85-
"field_path": []
86-
}
87-
}
88-
},
89-
"primary_key": [],
90-
"schema_loader": {
91-
"type": "InlineSchemaLoader",
92-
"schema": {
93-
"type": "object",
94-
"$schema": "http://json-schema.org/schema#",
95-
"properties": {
96-
"base": {
97-
"type": "string"
98-
},
99-
"date": {
100-
"type": "string"
101-
},
102-
"rates": {
103-
"type": "object",
104-
"properties": {
105-
"fake_currency": {
106-
"type": "number"
107-
}
108-
}
35+
"version": "6.48.15",
36+
"type": "DeclarativeSource",
37+
"check": {"type": "CheckStream", "stream_names": ["Rates"]},
38+
"streams": [
39+
{
40+
"type": "DeclarativeStream",
41+
"name": "Rates",
42+
"retriever": {
43+
"type": "SimpleRetriever",
44+
"requester": {
45+
"type": "HttpRequester",
46+
"path": "/exchangerates_data/{{stream_interval.start_time}}",
47+
"url_base": "https://api.apilayer.com",
48+
"http_method": "GET",
49+
"authenticator": {
50+
"type": "ApiKeyAuthenticator",
51+
"api_token": "{{ config['api_key'] }}",
52+
"inject_into": {
53+
"type": "RequestOption",
54+
"field_name": "apikey",
55+
"inject_into": "header",
56+
},
57+
},
58+
"request_parameters": {
59+
"base": "{{ config['base'] }}",
60+
PROPERTY_KEY: {
61+
"type": "QueryProperties",
62+
"property_list": PROPERTY_LIST,
63+
"property_chunking": {
64+
"type": "PropertyChunking",
65+
"property_limit_type": "property_count",
66+
"property_limit": 2,
67+
},
68+
},
69+
},
70+
},
71+
"record_selector": {
72+
"type": "RecordSelector",
73+
"extractor": {"type": "DpathExtractor", "field_path": []},
74+
},
10975
},
110-
"success": {
111-
"type": "boolean"
76+
"primary_key": [],
77+
"schema_loader": {
78+
"type": "InlineSchemaLoader",
79+
"schema": {
80+
"type": "object",
81+
"$schema": "http://json-schema.org/schema#",
82+
"properties": {
83+
"base": {"type": "string"},
84+
"date": {"type": "string"},
85+
"rates": {
86+
"type": "object",
87+
"properties": {"fake_currency": {"type": "number"}},
88+
},
89+
"success": {"type": "boolean"},
90+
"timestamp": {"type": "number"},
91+
"historical": {"type": "boolean"},
92+
},
93+
},
11294
},
113-
"timestamp": {
114-
"type": "number"
95+
"transformations": [],
96+
"incremental_sync": {
97+
"type": "DatetimeBasedCursor",
98+
"step": "P1D",
99+
"cursor_field": "date",
100+
"end_datetime": {
101+
"type": "MinMaxDatetime",
102+
"datetime": "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}",
103+
"datetime_format": "%Y-%m-%dT%H:%M:%SZ",
104+
},
105+
"start_datetime": {
106+
"type": "MinMaxDatetime",
107+
"datetime": "{{ config['start_date'] }}",
108+
"datetime_format": "%Y-%m-%dT%H:%M:%SZ",
109+
},
110+
"datetime_format": "%Y-%m-%d",
111+
"cursor_granularity": "P1D",
112+
"cursor_datetime_formats": ["%Y-%m-%d"],
115113
},
116-
"historical": {
117-
"type": "boolean"
118-
}
119-
}
114+
"state_migrations": [],
120115
}
121-
},
122-
"transformations": [],
123-
"incremental_sync": {
124-
"type": "DatetimeBasedCursor",
125-
"step": "P1D",
126-
"cursor_field": "date",
127-
"end_datetime": {
128-
"type": "MinMaxDatetime",
129-
"datetime": "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}",
130-
"datetime_format": "%Y-%m-%dT%H:%M:%SZ"
131-
},
132-
"start_datetime": {
133-
"type": "MinMaxDatetime",
134-
"datetime": "{{ config['start_date'] }}",
135-
"datetime_format": "%Y-%m-%dT%H:%M:%SZ"
136-
},
137-
"datetime_format": "%Y-%m-%d",
138-
"cursor_granularity": "P1D",
139-
"cursor_datetime_formats": [
140-
"%Y-%m-%d"
141-
]
142-
},
143-
"state_migrations": []
144-
}
145-
],
146-
"spec": {
147-
"type": "Spec",
148-
"documentation_url": "https://example.org",
149-
"connection_specification": {
150-
"type": "object",
151-
"$schema": "http://json-schema.org/draft-07/schema#",
152-
"required": [
153-
"start_date",
154-
"api_key",
155-
"base"
156-
],
157-
"properties": {
158-
"base": {
159-
"type": "string",
160-
"order": 2,
161-
"title": "Base"
116+
],
117+
"spec": {
118+
"type": "Spec",
119+
"documentation_url": "https://example.org",
120+
"connection_specification": {
121+
"type": "object",
122+
"$schema": "http://json-schema.org/draft-07/schema#",
123+
"required": ["start_date", "api_key", "base"],
124+
"properties": {
125+
"base": {"type": "string", "order": 2, "title": "Base"},
126+
"api_key": {
127+
"type": "string",
128+
"order": 1,
129+
"title": "API Key",
130+
"airbyte_secret": True,
131+
},
132+
"start_date": {
133+
"type": "string",
134+
"order": 0,
135+
"title": "Start date",
136+
"format": "date-time",
137+
"pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
138+
},
139+
},
140+
"additionalProperties": True,
162141
},
163-
"api_key": {
164-
"type": "string",
165-
"order": 1,
166-
"title": "API Key",
167-
"airbyte_secret": True
142+
},
143+
"metadata": {
144+
"testedStreams": {
145+
"Rates": {
146+
"hasRecords": False,
147+
"streamHash": "4dce031d602258dd3bcc478731d6862a5cdeb70f",
148+
"hasResponse": False,
149+
"primaryKeysAreUnique": False,
150+
"primaryKeysArePresent": False,
151+
"responsesAreSuccessful": False,
152+
}
168153
},
169-
"start_date": {
170-
"type": "string",
171-
"order": 0,
172-
"title": "Start date",
173-
"format": "date-time",
174-
"pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
175-
}
176-
},
177-
"additionalProperties": True
178-
}
179-
},
180-
"metadata": {
181-
"testedStreams": {
182-
"Rates": {
183-
"hasRecords": False,
184-
"streamHash": "4dce031d602258dd3bcc478731d6862a5cdeb70f",
185-
"hasResponse": False,
186-
"primaryKeysAreUnique": False,
187-
"primaryKeysArePresent": False,
188-
"responsesAreSuccessful": False
189-
}
154+
"autoImportSchema": {"Rates": True},
190155
},
191-
"autoImportSchema": {
192-
"Rates": True
193-
}
194-
},
195-
"dynamic_streams": []
156+
"dynamic_streams": [],
196157
}
197158

198159
_stream_name = "Rates"
@@ -232,6 +193,7 @@
232193
]
233194
}
234195

196+
235197
@freezegun.freeze_time(f"{FREEZE_DATE}T00:00:00Z")
236198
def test_read():
237199
conversion_base = "USD"
@@ -240,32 +202,42 @@ def test_read():
240202
config["base"] = conversion_base
241203
config["api_key"] = "test_api_key"
242204

243-
stream_url = f'{BASE_URL}{FREEZE_DATE}?base={conversion_base}&{PROPERTY_KEY}='
205+
stream_url = f"{BASE_URL}{FREEZE_DATE}?base={conversion_base}&{PROPERTY_KEY}="
244206

245-
with HttpMocker() as http_mocker:
246-
source = ManifestDeclarativeSource(source_config=MANIFEST, emit_connector_builder_messages=True)
247-
limits = TestLimits()
207+
with HttpMocker() as http_mocker:
208+
source = ManifestDeclarativeSource(
209+
source_config=MANIFEST, emit_connector_builder_messages=True
210+
)
211+
limits = TestLimits()
248212

249-
http_mocker.get(
250-
HttpRequest(url=f"{stream_url}{PROPERTY_LIST[0]}%2C{PROPERTY_LIST[1]}"),
251-
HttpResponse(json.dumps(find_template("declarative/property_chunking/rates_one_two", __file__)), 200),
252-
)
253-
http_mocker.get(
254-
HttpRequest(url=f"{stream_url}{PROPERTY_LIST[2]}%2C{PROPERTY_LIST[3]}"),
255-
HttpResponse(json.dumps(find_template("declarative/property_chunking/rates_three_four", __file__)), 200),
256-
)
257-
output_record = handle_connector_builder_request(
258-
source,
259-
"test_read",
260-
config,
261-
ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG),
262-
_A_STATE,
263-
limits,
264-
)
265-
# for connector build we get each record in a single page
266-
assert len(output_record.record.data["slices"][0]["pages"]) == 2
267-
for current_log in output_record.record.data["logs"]:
268-
assert not "Something went wrong in the connector" in current_log["message"]
269-
assert not current_log["internal_message"]
270-
assert not current_log["level"] == Level.ERROR
271-
assert not current_log["stacktrace"]
213+
http_mocker.get(
214+
HttpRequest(url=f"{stream_url}{PROPERTY_LIST[0]}%2C{PROPERTY_LIST[1]}"),
215+
HttpResponse(
216+
json.dumps(find_template("declarative/property_chunking/rates_one_two", __file__)),
217+
200,
218+
),
219+
)
220+
http_mocker.get(
221+
HttpRequest(url=f"{stream_url}{PROPERTY_LIST[2]}%2C{PROPERTY_LIST[3]}"),
222+
HttpResponse(
223+
json.dumps(
224+
find_template("declarative/property_chunking/rates_three_four", __file__)
225+
),
226+
200,
227+
),
228+
)
229+
output_record = handle_connector_builder_request(
230+
source,
231+
"test_read",
232+
config,
233+
ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG),
234+
_A_STATE,
235+
limits,
236+
)
237+
# for connector build we get each record in a single page
238+
assert len(output_record.record.data["slices"][0]["pages"]) == 2
239+
for current_log in output_record.record.data["logs"]:
240+
assert not "Something went wrong in the connector" in current_log["message"]
241+
assert not current_log["internal_message"]
242+
assert not current_log["level"] == Level.ERROR
243+
assert not current_log["stacktrace"]

0 commit comments

Comments
 (0)