Skip to content

Commit 480f5f7

Browse files
author
Oleksandr Bazarnov
committed
updated request_body_* migration
1 parent c418561 commit 480f5f7

File tree

2 files changed

+58
-27
lines changed

2 files changed

+58
-27
lines changed

‎airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@ class HttpRequesterRequestBodyJsonDataToRequestBody(ManifestMigration):
1818
"""
1919

2020
component_type = "HttpRequester"
21-
original_keys = ["request_body_json", "request_body_data"]
21+
22+
body_json_key = "request_body_json"
23+
body_data_key = "request_body_data"
24+
original_keys = (body_json_key, body_data_key)
25+
2226
replacement_key = "request_body"
2327

2428
def should_migrate(self, manifest: ManifestType) -> bool:
@@ -28,14 +32,20 @@ def should_migrate(self, manifest: ManifestType) -> bool:
2832

2933
def migrate(self, manifest: ManifestType) -> None:
3034
for key in self.original_keys:
31-
if key in manifest:
32-
manifest[self.replacement_key] = manifest[key]
35+
if key == self.body_json_key and key in manifest:
36+
manifest[self.replacement_key] = {
37+
"type": "RequestBodyJson",
38+
"value": manifest[key],
39+
}
40+
manifest.pop(key, None)
41+
elif key == self.body_data_key and key in manifest:
42+
manifest[self.replacement_key] = {
43+
"type": "RequestBodyData",
44+
"value": manifest[key],
45+
}
3346
manifest.pop(key, None)
3447

3548
def validate(self, manifest: ManifestType) -> bool:
36-
"""
37-
Validate the migration by checking if the `request_body` key is present and none of the original keys are.
38-
"""
3949
return self.replacement_key in manifest and all(
4050
key not in manifest for key in self.original_keys
4151
)

‎unit_tests/manifest_migrations/conftest.py‎

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -837,9 +837,12 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
837837
"http_method": "GET",
838838
"url": "https://example.com/v1/path_to_A",
839839
"request_body": {
840-
"test_key": "{{ config['config_key'] }}",
841-
"test_key_2": "test_value_2",
842-
"test_key_3": 123,
840+
"type": "RequestBodyData",
841+
"value": {
842+
"test_key": "{{ config['config_key'] }}",
843+
"test_key_2": "test_value_2",
844+
"test_key_3": 123,
845+
},
843846
},
844847
},
845848
"record_selector": {
@@ -867,9 +870,12 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
867870
"http_method": "GET",
868871
"url": "https://example.com/v1/path_to_A",
869872
"request_body": {
870-
"test_key": "{{ config['config_key'] }}",
871-
"test_key_2": "test_value_2",
872-
"test_key_3": 123,
873+
"type": "RequestBodyData",
874+
"value": {
875+
"test_key": "{{ config['config_key'] }}",
876+
"test_key_2": "test_value_2",
877+
"test_key_3": 123,
878+
},
873879
},
874880
},
875881
"record_selector": {
@@ -897,9 +903,12 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
897903
"http_method": "GET",
898904
"url": "https://example.com/v2/path_to_B",
899905
"request_body": {
900-
"reportType": "test_report",
901-
"groupBy": "GROUP",
902-
"metrics": "{{ ','.join( ['a-b','cd','e-f-g-h'] ) }}",
906+
"type": "RequestBodyJson",
907+
"value": {
908+
"reportType": "test_report",
909+
"groupBy": "GROUP",
910+
"metrics": "{{ ','.join( ['a-b','cd','e-f-g-h'] ) }}",
911+
},
903912
},
904913
},
905914
"record_selector": {
@@ -972,9 +981,12 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
972981
"type": "HttpRequester",
973982
"url": "https://example.com/v1/",
974983
"request_body": {
975-
"test_key": "{{ config['config_key'] }}",
976-
"test_key_2": "test_value_2",
977-
"test_key_3": 123,
984+
"type": "RequestBodyData",
985+
"value": {
986+
"test_key": "{{ config['config_key'] }}",
987+
"test_key_2": "test_value_2",
988+
"test_key_3": 123,
989+
},
978990
},
979991
},
980992
"requester_B": {"type": "HttpRequester", "url": "https://example.com/v2/"},
@@ -990,9 +1002,12 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
9901002
"http_method": "GET",
9911003
"url": "https://example.com/v1/path_to_A",
9921004
"request_body": {
993-
"test_key": "{{ config['config_key'] }}",
994-
"test_key_2": "test_value_2",
995-
"test_key_3": 123,
1005+
"type": "RequestBodyData",
1006+
"value": {
1007+
"test_key": "{{ config['config_key'] }}",
1008+
"test_key_2": "test_value_2",
1009+
"test_key_3": 123,
1010+
},
9961011
},
9971012
},
9981013
"record_selector": {
@@ -1020,9 +1035,12 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
10201035
"http_method": "GET",
10211036
"url": "https://example.com/v1/path_to_A",
10221037
"request_body": {
1023-
"test_key": "{{ config['config_key'] }}",
1024-
"test_key_2": "test_value_2",
1025-
"test_key_3": 123,
1038+
"type": "RequestBodyData",
1039+
"value": {
1040+
"test_key": "{{ config['config_key'] }}",
1041+
"test_key_2": "test_value_2",
1042+
"test_key_3": 123,
1043+
},
10261044
},
10271045
},
10281046
"record_selector": {
@@ -1050,9 +1068,12 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
10501068
"http_method": "GET",
10511069
"url": "https://example.com/v2/path_to_B",
10521070
"request_body": {
1053-
"reportType": "test_report",
1054-
"groupBy": "GROUP",
1055-
"metrics": "{{ ','.join( ['a-b','cd','e-f-g-h'] ) }}",
1071+
"type": "RequestBodyJson",
1072+
"value": {
1073+
"reportType": "test_report",
1074+
"groupBy": "GROUP",
1075+
"metrics": "{{ ','.join( ['a-b','cd','e-f-g-h'] ) }}",
1076+
},
10561077
},
10571078
},
10581079
"record_selector": {

0 commit comments

Comments
 (0)