add request_body_json/data deprecation + tests the new RequestBody provides the same functionality

Author: Oleksandr Bazarnov

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2000,6 +2000,8 @@ definitions:
           - "$ref": "#/definitions/CustomAuthenticator"
           - "$ref": "#/definitions/LegacySessionTokenAuthenticator"
       request_body_data:
+        deprecated: true
+        deprecation_message: "Use `request_body` field instead."
         title: Request Body Payload (Non-JSON)
         description: Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.
         anyOf:
@@ -2018,6 +2020,8 @@ definitions:
                 [{"value": {{ stream_interval['start_time'] | int * 1000 }} }]
               }, "orderBy": 1, "columnName": "Timestamp"}]/
       request_body_json:
+        deprecated: true
+        deprecation_message: "Use `request_body` field instead."
         title: Request Body JSON Payload
         description: Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.
         anyOf:
@@ -2036,6 +2040,35 @@ definitions:
           - sort:
               field: "updated_at"
               order: "ascending"
+      request_body:
+        title: Request Body Payload to be send as a part of the API request.
+        description: Specifies how to populate the body of the request with a payload. Can contain nested objects.
+        anyOf:
+          - "$ref": "#/definitions/RequestBody"
+        interpolation_context:
+          - next_page_token
+          - stream_interval
+          - stream_partition
+          - stream_slice
+        examples:
+          - type: RequestBodyJson
+            value:
+              sort_order: "ASC"
+              sort_field: "CREATED_AT"
+          - type: RequestBodyJson
+            value:
+              key: "{{ config['value'] }}"
+          - type: RequestBodyJson
+            value:
+              sort:
+                field: "updated_at"
+                order: "ascending"
+          - type: RequestBodyData
+            value: "plain_text_body"
+          - type: RequestBodyData
+            value:
+              param1: "value1"
+              param2: "{{ config['param2_value'] }}"
       request_headers:
         title: Request Headers
         description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.
@@ -4036,6 +4069,27 @@ definitions:
       - type
       - stream_template
       - components_resolver
+  RequestBody:
+    type: object
+    description: The request body payload. Can be either URL encoded data or JSON.
+    properties:
+      type:
+        anyOf:
+          - type: string
+            enum: [RequestBodyData]
+          - type: string
+            enum: [RequestBodyJson]
+      value:
+        anyOf:
+          - type: string
+            description: The request body payload as a string.
+          - type: object
+            description: The request body payload as a Non-JSON object (url-encoded data).
+            additionalProperties:
+              type: string
+          - type: object
+            description: The request body payload as a JSON object (json-encoded data).
+            additionalProperties: true
 interpolation:
   variables:
     - title: config

airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py

@@ -12,7 +12,7 @@
 NestedMappingEntry = Union[
     dict[str, "NestedMapping"], list["NestedMapping"], str, int, float, bool, None
 ]
-NestedMapping = Union[dict[str, NestedMappingEntry], str]
+NestedMapping = Union[dict[str, NestedMappingEntry], str, dict[str, Any]]
 
 
 @dataclass

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1499,6 +1499,11 @@ class ConfigComponentsResolver(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class RequestBody(BaseModel):
+    type: Optional[Union[Literal["RequestBodyData"], Literal["RequestBodyJson"]]] = None
+    value: Optional[Union[str, Dict[str, str], Dict[str, Any]]] = None
+
+
 class AddedFieldDefinition(BaseModel):
     type: Literal["AddedFieldDefinition"]
     path: List[str] = Field(
@@ -2273,6 +2278,8 @@ class HttpRequester(BaseModelWithDeprecations):
     )
     request_body_data: Optional[Union[Dict[str, str], str]] = Field(
         None,
+        deprecated=True,
+        deprecation_message="Use `request_body` field instead.",
         description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.",
         examples=[
             '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n    [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n  }, "orderBy": 1, "columnName": "Timestamp"}]/\n'
@@ -2281,6 +2288,8 @@ class HttpRequester(BaseModelWithDeprecations):
     )
     request_body_json: Optional[Union[Dict[str, Any], str]] = Field(
         None,
+        deprecated=True,
+        deprecation_message="Use `request_body` field instead.",
         description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.",
         examples=[
             {"sort_order": "ASC", "sort_field": "CREATED_AT"},
@@ -2289,6 +2298,27 @@ class HttpRequester(BaseModelWithDeprecations):
         ],
         title="Request Body JSON Payload",
     )
+    request_body: Optional[RequestBody] = Field(
+        None,
+        description="Specifies how to populate the body of the request with a payload. Can contain nested objects.",
+        examples=[
+            {
+                "type": "RequestBodyJson",
+                "value": {"sort_order": "ASC", "sort_field": "CREATED_AT"},
+            },
+            {"type": "RequestBodyJson", "value": {"key": "{{ config['value'] }}"}},
+            {
+                "type": "RequestBodyJson",
+                "value": {"sort": {"field": "updated_at", "order": "ascending"}},
+            },
+            {"type": "RequestBodyData", "value": "plain_text_body"},
+            {
+                "type": "RequestBodyData",
+                "value": {"param1": "value1", "param2": "{{ config['param2_value'] }}"},
+            },
+        ],
+        title="Request Body Payload to be send as a part of the API request.",
+    )
     request_headers: Optional[Union[Dict[str, str], str]] = Field(
         None,
         description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -2239,6 +2239,7 @@ def create_http_requester(
             request_parameters = model.request_parameters
 
         request_options_provider = InterpolatedRequestOptionsProvider(
+            request_body=model.request_body,
             request_body_data=model.request_body_data,
             request_body_json=model.request_body_json,
             request_headers=model.request_headers,

airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py

@@ -6,6 +6,9 @@
 from typing import Any, List, Mapping, MutableMapping, Optional, Union
 
 from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import NestedMapping
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    RequestBody,
+)
 from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import (
     InterpolatedNestedRequestInputProvider,
 )
@@ -38,6 +41,7 @@ class InterpolatedRequestOptionsProvider(RequestOptionsProvider):
     config: Config = field(default_factory=dict)
     request_parameters: Optional[RequestInput] = None
     request_headers: Optional[RequestInput] = None
+    request_body: Optional[RequestBody] = None
     request_body_data: Optional[RequestInput] = None
     request_body_json: Optional[NestedMapping] = None
     query_properties_key: Optional[str] = None
@@ -47,16 +51,19 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
             self.request_parameters = {}
         if self.request_headers is None:
             self.request_headers = {}
+        # resolve the request body to either data or json
+        self._resolve_request_body()
+        # If request_body is not provided, set request_body_data and request_body_json to empty dicts
         if self.request_body_data is None:
             self.request_body_data = {}
         if self.request_body_json is None:
             self.request_body_json = {}
-
+        # If both request_body_data and request_body_json are provided, raise an error
         if self.request_body_json and self.request_body_data:
             raise ValueError(
                 "RequestOptionsProvider should only contain either 'request_body_data' or 'request_body_json' not both"
             )
-
+        # set interpolators
         self._parameter_interpolator = InterpolatedRequestInputProvider(
             config=self.config, request_inputs=self.request_parameters, parameters=parameters
         )
@@ -70,6 +77,21 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
             config=self.config, request_inputs=self.request_body_json, parameters=parameters
         )
 
+    def _resolve_request_body(self) -> None:
+        """
+        Resolves the request body configuration by setting either `request_body_data` or `request_body_json`
+        based on the type specified in `self.request_body`. If neither is provided, both are initialized as empty
+        dictionaries. Raises a ValueError if both `request_body_data` and `request_body_json` are set simultaneously.
+        Raises:
+            ValueError: If both `request_body_data` and `request_body_json` are provided.
+        """
+        # Resolve the request body to either data or json
+        if self.request_body is not None and self.request_body.type is not None:
+            if self.request_body.type == "RequestBodyData":
+                self.request_body_data = self.request_body.value
+            elif self.request_body.type == "RequestBodyJson":
+                self.request_body_json = self.request_body.value
+
     def get_request_params(
         self,
         *,

unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py

@@ -4,6 +4,7 @@
 
 import pytest
 
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestBody
 from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import (
     InterpolatedRequestOptionsProvider,
 )
@@ -131,6 +132,71 @@ def test_interpolated_request_json(test_name, input_request_json, expected_reque
     assert actual_request_json == expected_request_json
 
 
+@pytest.mark.parametrize(
+    "test_name, input_request_json, expected_request_json",
+    [
+        (
+            "test_static_json",
+            {"a_static_request_param": "a_static_value"},
+            {"a_static_request_param": "a_static_value"},
+        ),
+        (
+            "test_value_depends_on_stream_slice",
+            {"read_from_slice": "{{ stream_slice['start_date'] }}"},
+            {"read_from_slice": "2020-01-01"},
+        ),
+        (
+            "test_value_depends_on_next_page_token",
+            {"read_from_token": "{{ next_page_token['offset'] }}"},
+            {"read_from_token": 12345},
+        ),
+        (
+            "test_value_depends_on_config",
+            {"read_from_config": "{{ config['option'] }}"},
+            {"read_from_config": "OPTION"},
+        ),
+        (
+            "test_interpolated_keys",
+            {"{{ stream_interval['start_date'] }}": 123, "{{ config['option'] }}": "ABC"},
+            {"2020-01-01": 123, "OPTION": "ABC"},
+        ),
+        ("test_boolean_false_value", {"boolean_false": "{{ False }}"}, {"boolean_false": False}),
+        ("test_integer_falsy_value", {"integer_falsy": "{{ 0 }}"}, {"integer_falsy": 0}),
+        ("test_number_falsy_value", {"number_falsy": "{{ 0.0 }}"}, {"number_falsy": 0.0}),
+        ("test_string_falsy_value", {"string_falsy": "{{ '' }}"}, {}),
+        ("test_none_value", {"none_value": "{{ None }}"}, {}),
+        (
+            "test_string",
+            """{"nested": { "key": "{{ config['option'] }}" }}""",
+            {"nested": {"key": "OPTION"}},
+        ),
+        (
+            "test_nested_objects",
+            {"nested": {"key": "{{ config['option'] }}"}},
+            {"nested": {"key": "OPTION"}},
+        ),
+        (
+            "test_nested_objects_interpolated keys",
+            {"nested": {"{{ stream_interval['start_date'] }}": "{{ config['option'] }}"}},
+            {"nested": {"2020-01-01": "OPTION"}},
+        ),
+    ],
+)
+def test_interpolated_request_json_using_request_body(
+    test_name, input_request_json, expected_request_json
+):
+    provider = InterpolatedRequestOptionsProvider(
+        config=config,
+        request_body=RequestBody(type="RequestBodyJson", value=input_request_json),
+        parameters={},
+    )
+    actual_request_json = provider.get_request_body_json(
+        stream_state=state, stream_slice=stream_slice, next_page_token=next_page_token
+    )
+
+    assert actual_request_json == expected_request_json
+
+
 @pytest.mark.parametrize(
     "test_name, input_request_data, expected_request_data",
     [
@@ -169,6 +235,48 @@ def test_interpolated_request_data(test_name, input_request_data, expected_reque
     assert actual_request_data == expected_request_data
 
 
+@pytest.mark.parametrize(
+    "test_name, input_request_data, expected_request_data",
+    [
+        (
+            "test_static_map_data",
+            {"a_static_request_param": "a_static_value"},
+            {"a_static_request_param": "a_static_value"},
+        ),
+        (
+            "test_map_depends_on_stream_slice",
+            {"read_from_slice": "{{ stream_slice['start_date'] }}"},
+            {"read_from_slice": "2020-01-01"},
+        ),
+        (
+            "test_map_depends_on_config",
+            {"read_from_config": "{{ config['option'] }}"},
+            {"read_from_config": "OPTION"},
+        ),
+        ("test_defaults_to_empty_dict", None, {}),
+        (
+            "test_interpolated_keys",
+            {"{{ stream_interval['start_date'] }} - {{ next_page_token['offset'] }}": "ABC"},
+            {"2020-01-01 - 12345": "ABC"},
+        ),
+    ],
+)
+def test_interpolated_request_data_using_request_body(
+    test_name, input_request_data, expected_request_data
+):
+    provider = InterpolatedRequestOptionsProvider(
+        config=config,
+        request_body=RequestBody(type="RequestBodyData", value=input_request_data),
+        parameters={},
+    )
+
+    actual_request_data = provider.get_request_body_data(
+        stream_state=state, stream_slice=stream_slice, next_page_token=next_page_token
+    )
+
+    assert actual_request_data == expected_request_data
+
+
 def test_error_on_create_for_both_request_json_and_data():
     request_json = {"body_key": "{{ stream_slice['start_date'] }}"}
     request_data = "interpolate_me=5&invalid={{ config['option'] }}"