Skip to content

Commit 427875b

Browse files
author
Oleksandr Bazarnov
committed
adjusted the migration logic
1 parent 939e9f3 commit 427875b

File tree

4 files changed

+101
-59
lines changed

4 files changed

+101
-59
lines changed

‎airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,44 @@ def should_migrate(self, manifest: ManifestType) -> bool:
3333
def migrate(self, manifest: ManifestType) -> None:
3434
for key in self.original_keys:
3535
if key == self.body_json_key and key in manifest:
36-
manifest[self.replacement_key] = {
37-
"type": "RequestBodyJson",
38-
"value": manifest[key],
39-
}
40-
manifest.pop(key, None)
36+
self._migrate_body_json(manifest, key)
4137
elif key == self.body_data_key and key in manifest:
42-
manifest[self.replacement_key] = {
43-
"type": "RequestBodyData",
44-
"value": manifest[key],
45-
}
46-
manifest.pop(key, None)
38+
self._migrate_body_data(manifest, key)
4739

4840
def validate(self, manifest: ManifestType) -> bool:
4941
return self.replacement_key in manifest and all(
5042
key not in manifest for key in self.original_keys
5143
)
44+
45+
def _migrate_body_json(self, manifest: ManifestType, key: str) -> None:
46+
"""
47+
Migrate the value of the request_body_json.
48+
"""
49+
query_key = "query"
50+
text_type = "RequestBodyPlainText"
51+
graph_ql_type = "RequestBodyGraphQL"
52+
json_object_type = "RequestBodyJsonObject"
53+
54+
if isinstance(manifest[key], str):
55+
self._migrate_value(manifest, key, text_type)
56+
elif isinstance(manifest[key], dict):
57+
if manifest[key].get(query_key) is not None:
58+
self._migrate_value(manifest, key, graph_ql_type)
59+
else:
60+
self._migrate_value(manifest, key, json_object_type)
61+
62+
def _migrate_body_data(self, manifest: ManifestType, key: str) -> None:
63+
"""
64+
Migrate the value of the request_body_data.
65+
"""
66+
self._migrate_value(manifest, key, "RequestBodyUrlEncodedForm")
67+
68+
def _migrate_value(self, manifest: ManifestType, key: str, type: str) -> None:
69+
"""
70+
Migrate the value of the key to a specific type and update the manifest.
71+
"""
72+
manifest[self.replacement_key] = {
73+
"type": type,
74+
"value": manifest[key],
75+
}
76+
manifest.pop(key, None)

‎airbyte_cdk/manifest_migrations/migrations/registry.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
#
44

55
manifest_migrations:
6-
- version: 6.48.2
6+
- version: 6.48.3
77
migrations:
88
- name: http_requester_url_base_to_url
99
order: 1

‎airbyte_cdk/sources/declarative/declarative_component_schema.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4106,7 +4106,7 @@ definitions:
41064106
type: string
41074107
RequestBodyJsonObject:
41084108
title: Json Object Body
4109-
description: Request body value is converted into a url-encoded form
4109+
description: Request body value converted into a JSON object
41104110
type: object
41114111
required:
41124112
- type
@@ -4120,7 +4120,7 @@ definitions:
41204120
additionalProperties: true
41214121
RequestBodyGraphQL:
41224122
title: GraphQL Body
4123-
description: Request body is a GraphQL query object
4123+
description: Request body value converted into a GraphQL query object
41244124
type: object
41254125
required:
41264126
- type

‎unit_tests/manifest_migrations/conftest.py‎

Lines changed: 63 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ def manifest_with_url_base_to_migrate_to_url() -> Dict[str, Any]:
197197
@pytest.fixture
198198
def expected_manifest_with_url_base_migrated_to_url() -> Dict[str, Any]:
199199
return {
200-
"version": "6.48.2",
200+
"version": "6.48.3",
201201
"type": "DeclarativeSource",
202202
"check": {"type": "CheckStream", "stream_names": ["A"]},
203203
"definitions": {
@@ -494,13 +494,13 @@ def expected_manifest_with_url_base_migrated_to_url() -> Dict[str, Any]:
494494
"applied_migrations": [
495495
{
496496
"from_version": "0.0.0",
497-
"to_version": "6.48.2",
497+
"to_version": "6.48.3",
498498
"migration": "HttpRequesterUrlBaseToUrl",
499499
"migrated_at": "2025-04-01T00:00:00+00:00", # time freezed in the test
500500
},
501501
{
502502
"from_version": "0.0.0",
503-
"to_version": "6.48.2",
503+
"to_version": "6.48.3",
504504
"migration": "HttpRequesterPathToUrl",
505505
"migrated_at": "2025-04-01T00:00:00+00:00", # time freezed in the test
506506
},
@@ -512,7 +512,7 @@ def expected_manifest_with_url_base_migrated_to_url() -> Dict[str, Any]:
512512
@pytest.fixture
513513
def manifest_with_migrated_url_base_and_path_is_joined_to_url() -> Dict[str, Any]:
514514
return {
515-
"version": "6.48.2",
515+
"version": "6.48.3",
516516
"type": "DeclarativeSource",
517517
"check": {"type": "CheckStream", "stream_names": ["A"]},
518518
"definitions": {},
@@ -636,6 +636,13 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic
636636
"$ref": "#/definitions/requester_A",
637637
"path": "/path_to_A",
638638
"http_method": "GET",
639+
# this requester has a `request_body_data` key,
640+
# to be migrated to the `request_body` key
641+
"request_body_data": {
642+
"test_key": "{{ config['config_key'] }}",
643+
"test_key_2": "test_value_2",
644+
"test_key_3": 123,
645+
},
639646
},
640647
"record_selector": {
641648
"type": "RecordSelector",
@@ -656,6 +663,9 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic
656663
"$ref": "#/definitions/requester_A",
657664
"path": "path_to_A",
658665
"http_method": "GET",
666+
# this requester has a `request_body_data` key,
667+
# to be migrated to the `request_body` key
668+
"request_body_data": "&test_key=TestValue&test_key_2=test_value_2",
659669
},
660670
"record_selector": {
661671
"type": "RecordSelector",
@@ -703,6 +713,8 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic
703713
# ! the double-slash is intentional here for the test.
704714
"path": "//path_to_B",
705715
"http_method": "GET",
716+
# the `request_body_json` is expected to be migrated to the `request_body` key
717+
"request_body_json": """{"nested": { "key": "{{ config['option'] }}" }}""",
706718
},
707719
"record_selector": {
708720
"type": "RecordSelector",
@@ -723,6 +735,14 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic
723735
"$ref": "#/definitions/requester_B",
724736
"path": "/path_to_B",
725737
"http_method": "GET",
738+
# the `request_body_json` is expected to be migrated to the `request_body` key,
739+
# this example holds the GraphQL query object.
740+
"request_body_json": {
741+
"query": {
742+
"field": "{{ config['query_field'] }}",
743+
"value": "{{ config['query_value'] }}",
744+
}
745+
},
726746
},
727747
"record_selector": {
728748
"type": "RecordSelector",
@@ -741,20 +761,10 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic
741761
"requester_A": {
742762
"type": "HttpRequester",
743763
"url_base": "https://example.com/v1/",
744-
# this requester has a `request_body_json` key,
745-
# to be migrated to the `request_body` key
746-
"request_body_data": {
747-
"test_key": "{{ config['config_key'] }}",
748-
"test_key_2": "test_value_2",
749-
"test_key_3": 123,
750-
},
751764
},
752765
"requester_B": {
753766
"type": "HttpRequester",
754767
"url_base": "https://example.com/v2/",
755-
# for this requester, the `request_body_json` key is not present,
756-
# but the `request_body_data` key is present in the stream `C` itself.
757-
# it should also be migrated to the `request_body` key
758768
},
759769
},
760770
"streams": [
@@ -822,7 +832,7 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic
822832
@pytest.fixture
823833
def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
824834
return {
825-
"version": "6.48.2",
835+
"version": "6.48.3",
826836
"type": "DeclarativeSource",
827837
"check": {"type": "CheckStream", "stream_names": ["A"]},
828838
"definitions": {
@@ -837,7 +847,7 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
837847
"http_method": "GET",
838848
"url": "https://example.com/v1/path_to_A",
839849
"request_body": {
840-
"type": "RequestBodyData",
850+
"type": "RequestBodyUrlEncodedForm",
841851
"value": {
842852
"test_key": "{{ config['config_key'] }}",
843853
"test_key_2": "test_value_2",
@@ -870,12 +880,8 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
870880
"http_method": "GET",
871881
"url": "https://example.com/v1/path_to_A",
872882
"request_body": {
873-
"type": "RequestBodyData",
874-
"value": {
875-
"test_key": "{{ config['config_key'] }}",
876-
"test_key_2": "test_value_2",
877-
"test_key_3": 123,
878-
},
883+
"type": "RequestBodyUrlEncodedForm",
884+
"value": "&test_key=TestValue&test_key_2=test_value_2",
879885
},
880886
},
881887
"record_selector": {
@@ -903,7 +909,7 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
903909
"http_method": "GET",
904910
"url": "https://example.com/v2/path_to_B",
905911
"request_body": {
906-
"type": "RequestBodyJson",
912+
"type": "RequestBodyJsonObject",
907913
"value": {
908914
"reportType": "test_report",
909915
"groupBy": "GROUP",
@@ -935,6 +941,10 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
935941
"type": "HttpRequester",
936942
"http_method": "GET",
937943
"url": "https://example.com/v2/path_to_B",
944+
"request_body": {
945+
"type": "RequestBodyPlainText",
946+
"value": '{"nested": { "key": "{{ config[\'option\'] }}" }}',
947+
},
938948
},
939949
"record_selector": {
940950
"type": "RecordSelector",
@@ -960,6 +970,15 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
960970
"type": "HttpRequester",
961971
"http_method": "GET",
962972
"url": "https://example.com/v2/path_to_B",
973+
"request_body": {
974+
"type": "RequestBodyGraphQL",
975+
"value": {
976+
"query": {
977+
"field": "{{ config['query_field'] }}",
978+
"value": "{{ config['query_value'] }}",
979+
}
980+
},
981+
},
963982
},
964983
"record_selector": {
965984
"type": "RecordSelector",
@@ -977,18 +996,7 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
977996
},
978997
},
979998
},
980-
"requester_A": {
981-
"type": "HttpRequester",
982-
"url": "https://example.com/v1/",
983-
"request_body": {
984-
"type": "RequestBodyData",
985-
"value": {
986-
"test_key": "{{ config['config_key'] }}",
987-
"test_key_2": "test_value_2",
988-
"test_key_3": 123,
989-
},
990-
},
991-
},
999+
"requester_A": {"type": "HttpRequester", "url": "https://example.com/v1/"},
9921000
"requester_B": {"type": "HttpRequester", "url": "https://example.com/v2/"},
9931001
},
9941002
"streams": [
@@ -1002,7 +1010,7 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
10021010
"http_method": "GET",
10031011
"url": "https://example.com/v1/path_to_A",
10041012
"request_body": {
1005-
"type": "RequestBodyData",
1013+
"type": "RequestBodyUrlEncodedForm",
10061014
"value": {
10071015
"test_key": "{{ config['config_key'] }}",
10081016
"test_key_2": "test_value_2",
@@ -1035,12 +1043,8 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
10351043
"http_method": "GET",
10361044
"url": "https://example.com/v1/path_to_A",
10371045
"request_body": {
1038-
"type": "RequestBodyData",
1039-
"value": {
1040-
"test_key": "{{ config['config_key'] }}",
1041-
"test_key_2": "test_value_2",
1042-
"test_key_3": 123,
1043-
},
1046+
"type": "RequestBodyUrlEncodedForm",
1047+
"value": "&test_key=TestValue&test_key_2=test_value_2",
10441048
},
10451049
},
10461050
"record_selector": {
@@ -1068,7 +1072,7 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
10681072
"http_method": "GET",
10691073
"url": "https://example.com/v2/path_to_B",
10701074
"request_body": {
1071-
"type": "RequestBodyJson",
1075+
"type": "RequestBodyJsonObject",
10721076
"value": {
10731077
"reportType": "test_report",
10741078
"groupBy": "GROUP",
@@ -1100,6 +1104,10 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
11001104
"type": "HttpRequester",
11011105
"http_method": "GET",
11021106
"url": "https://example.com/v2/path_to_B",
1107+
"request_body": {
1108+
"type": "RequestBodyPlainText",
1109+
"value": '{"nested": { "key": "{{ config[\'option\'] }}" }}',
1110+
},
11031111
},
11041112
"record_selector": {
11051113
"type": "RecordSelector",
@@ -1125,6 +1133,15 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
11251133
"type": "HttpRequester",
11261134
"http_method": "GET",
11271135
"url": "https://example.com/v2/path_to_B",
1136+
"request_body": {
1137+
"type": "RequestBodyGraphQL",
1138+
"value": {
1139+
"query": {
1140+
"field": "{{ config['query_field'] }}",
1141+
"value": "{{ config['query_value'] }}",
1142+
}
1143+
},
1144+
},
11281145
},
11291146
"record_selector": {
11301147
"type": "RecordSelector",
@@ -1178,19 +1195,19 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
11781195
"applied_migrations": [
11791196
{
11801197
"from_version": "0.0.0",
1181-
"to_version": "6.48.2",
1198+
"to_version": "6.48.3",
11821199
"migration": "HttpRequesterUrlBaseToUrl",
11831200
"migrated_at": "2025-04-01T00:00:00+00:00",
11841201
},
11851202
{
11861203
"from_version": "0.0.0",
1187-
"to_version": "6.48.2",
1204+
"to_version": "6.48.3",
11881205
"migration": "HttpRequesterPathToUrl",
11891206
"migrated_at": "2025-04-01T00:00:00+00:00",
11901207
},
11911208
{
11921209
"from_version": "0.0.0",
1193-
"to_version": "6.48.2",
1210+
"to_version": "6.48.3",
11941211
"migration": "HttpRequesterRequestBodyJsonDataToRequestBody",
11951212
"migrated_at": "2025-04-01T00:00:00+00:00",
11961213
},

0 commit comments

Comments
 (0)