added HttpRequester deprecations

Author: Oleksandr Bazarnov

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1863,14 +1863,16 @@ definitions:
     type: object
     required:
       - type
-      - url_base
     properties:
       type:
         type: string
         enum: [HttpRequester]
       url_base:
+        deprecated: true
+        deprecation_message: "Use `url` field instead."
+        sharable: true
         title: API Base URL
-        description: Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
         type: string
         interpolation_context:
           - config
@@ -1886,9 +1888,30 @@ definitions:
           - "{{ config['base_url'] or 'https://app.posthog.com'}}/api"
           - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups"
           - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}"
+      url:
+        sharable: true
+        title: API URL
+        description: The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        type: string
+        interpolation_context:
+          - config
+          - next_page_token
+          - stream_interval
+          - stream_partition
+          - stream_slice
+          - creation_response
+          - polling_response
+          - download_target
+        examples:
+          - "https://connect.squareup.com/v2"
+          - "{{ config['url'] or 'https://app.posthog.com'}}/api"
+          - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups"
+          - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}"
       path:
+        deprecated: true
+        deprecation_message: "Use `url` field instead."
         title: URL Path
-        description: Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        description: Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
         type: string
         interpolation_context:
           - config
@@ -4158,4 +4181,4 @@ interpolation:
         regex: The regular expression to search for. It must include a capture group.
       return_type: str
       examples:
-        - '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"'
+        - '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"'

airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py

@@ -46,15 +46,39 @@ class Config:
 
         extra = "allow"
 
-    def __init__(self, **data: Any) -> None:
+    def __init__(self, **model_data: Any) -> None:
         """
         Show warnings for deprecated fields during component initialization.
         """
-        # placeholder for deprecation logs
+
+        # call the parent constructor first to initialize Pydantic internals
+        super().__init__(**model_data)
+
+        # set the placeholder for the deprecation logs
         self._deprecation_logs: List[AirbyteLogMessage] = []
 
+        # process deprecated fields, if present
+        self._process_fields(model_data)
+
+        # set the deprecation logs attribute to the model
+        self._set_deprecation_logs_attr_to_model()
+
+    def _process_fields(self, model_data: Any) -> None:
+        """
+        Processes the fields in the provided model data, checking for deprecated fields.
+
+        For each field in the input `model_data`, this method checks if the field exists in the model's defined fields.
+        If the field is marked as deprecated (using the `DEPRECATED` flag in its metadata), it triggers a deprecation warning
+        by calling the `_deprecated_warning` method with the field name and an optional deprecation message.
+
+        Args:
+            model_data (Any): The data containing fields to be processed.
+
+        Returns:
+            None
+        """
         model_fields = self.__fields__
-        for field_name in data:
+        for field_name in model_data.keys():
             if field_name in model_fields:
                 is_deprecated_field = model_fields[field_name].field_info.extra.get(
                     DEPRECATED, False
@@ -65,28 +89,18 @@ def __init__(self, **data: Any) -> None:
                     )
                     self._deprecated_warning(field_name, deprecation_message)
 
-        # Call the parent constructor
-        super().__init__(**data)
-
-    def __getattribute__(self, name: str) -> Any:
-        """
-        Show warnings for deprecated fields during field usage.
+    def _set_deprecation_logs_attr_to_model(self) -> None:
         """
+        Sets the deprecation logs attribute on the model instance.
 
-        value = super().__getattribute__(name)
-        try:
-            model_fields = super().__getattribute__(FIELDS_TAG)
-            field_info = model_fields.get(name)
-            is_deprecated_field = (
-                field_info.field_info.extra.get(DEPRECATED, False) if field_info else False
-            )
-            if is_deprecated_field:
-                deprecation_message = field_info.field_info.extra.get(DEPRECATION_MESSAGE, "")
-                self._deprecated_warning(name, deprecation_message)
-        except (AttributeError, KeyError):
-            pass
-
-        return value
+        This method attaches the current instance's deprecation logs to the model by setting
+        an attribute named by `DEPRECATION_LOGS_TAG` to the value of `self._deprecation_logs`.
+        This is typically used to track or log deprecated features or configurations within the model.
+
+        Returns:
+            None
+        """
+        setattr(self, DEPRECATION_LOGS_TAG, self._deprecation_logs)
 
     def _deprecated_warning(self, field_name: str, message: str) -> None:
         """
@@ -97,12 +111,12 @@ def _deprecated_warning(self, field_name: str, message: str) -> None:
         """
 
         message = f"Component type: `{self.__class__.__name__}`. Field '{field_name}' is deprecated. {message}"
-
         # Emit a warning message for deprecated fields (to stdout) (Python Default behavior)
         warnings.warn(message, DeprecationWarning)
-
+        # Create an Airbyte deprecation log message
+        deprecation_log_message = AirbyteLogMessage(level=Level.WARN, message=message)
         # Add the deprecation message to the Airbyte log messages,
         # this logs are displayed in the Connector Builder.
-        self._deprecation_logs.append(
-            AirbyteLogMessage(level=Level.WARN, message=message),
-        )
+        if deprecation_log_message not in self._deprecation_logs:
+            # Avoid duplicates in the deprecation logs
+            self._deprecation_logs.append(deprecation_log_message)

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1,5 +1,3 @@
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-
 # generated by datamodel-codegen:
 #   filename:  declarative_component_schema.yaml
 
@@ -10,6 +8,10 @@
 
 from pydantic.v1 import BaseModel, Extra, Field
 
+from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import (
+    BaseModelWithDeprecations,
+)
+
 
 class AuthFlowType(Enum):
     oauth2_0 = "oauth2.0"
@@ -880,20 +882,17 @@ class FlattenFields(BaseModel):
 
 
 class KeyTransformation(BaseModel):
-    prefix: Optional[Union[str, None]] = Field(
+    type: Literal["KeyTransformation"]
+    prefix: Optional[str] = Field(
         None,
         description="Prefix to add for object keys. If not provided original keys remain unchanged.",
-        examples=[
-            "flattened_",
-        ],
+        examples=["flattened_"],
         title="Key Prefix",
     )
-    suffix: Optional[Union[str, None]] = Field(
+    suffix: Optional[str] = Field(
         None,
         description="Suffix to add for object keys. If not provided original keys remain unchanged.",
-        examples=[
-            "_flattened",
-        ],
+        examples=["_flattened"],
         title="Key Suffix",
     )
 
@@ -916,7 +915,7 @@ class DpathFlattenFields(BaseModel):
         description="Whether to replace the origin record or not. Default is False.",
         title="Replace Origin Record",
     )
-    key_transformation: Optional[Union[KeyTransformation, None]] = Field(
+    key_transformation: Optional[KeyTransformation] = Field(
         None,
         description="Transformation for object keys. If not provided, original key will be used.",
         title="Key transformation",
@@ -2171,11 +2170,13 @@ class SessionTokenAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class HttpRequester(BaseModel):
+class HttpRequester(BaseModelWithDeprecations):
     type: Literal["HttpRequester"]
-    url_base: str = Field(
-        ...,
-        description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+    url_base: Optional[str] = Field(
+        None,
+        deprecated=True,
+        deprecation_message="Use `url` field instead.",
+        description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "https://connect.squareup.com/v2",
             "{{ config['base_url'] or 'https://app.posthog.com'}}/api",
@@ -2184,9 +2185,22 @@ class HttpRequester(BaseModel):
         ],
         title="API Base URL",
     )
+    url: Optional[str] = Field(
+        None,
+        description="The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+        examples=[
+            "https://connect.squareup.com/v2",
+            "{{ config['url'] or 'https://app.posthog.com'}}/api",
+            "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups",
+            "https://example.com/api/v1/resource/{{ next_page_token['id'] }}",
+        ],
+        title="API URL",
+    )
     path: Optional[str] = Field(
         None,
-        description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+        deprecated=True,
+        deprecation_message="Use `url` field instead.",
+        description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "/products",
             "/quotes/{{ stream_partition['id'] }}/quote_line_groups",

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -2194,7 +2194,7 @@ def create_http_requester(
             self._create_component_from_model(
                 model=model.authenticator,
                 config=config,
-                url_base=model.url_base,
+                url_base=model.url or model.url_base,
                 name=name,
                 decoder=decoder,
             )
@@ -2231,6 +2231,7 @@ def create_http_requester(
 
         return HttpRequester(
             name=name,
+            url=model.url,
             url_base=model.url_base,
             path=model.path,
             authenticator=authenticator,
@@ -2928,6 +2929,25 @@ def create_simple_retriever(
         use_cache: Optional[bool] = None,
         **kwargs: Any,
     ) -> SimpleRetriever:
+        def _get_url() -> str:
+            """
+            Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever.
+            This is needed because the URL is not set until the requester is created.
+            """
+
+            _url = (
+                model.requester.url
+                if hasattr(model.requester, "url") and model.requester.url is not None
+                else requester.get_url()
+            )
+            _url_base = (
+                model.requester.url_base
+                if hasattr(model.requester, "url_base") and model.requester.url_base is not None
+                else requester.get_url_base()
+            )
+
+            return _url or _url_base
+
         decoder = (
             self._create_component_from_model(model=model.decoder, config=config)
             if model.decoder
@@ -2992,11 +3012,6 @@ def create_simple_retriever(
             use_cache=use_cache,
             config=config,
         )
-        url_base = (
-            model.requester.url_base
-            if hasattr(model.requester, "url_base")
-            else requester.get_url_base()
-        )
 
         # Define cursor only if per partition or common incremental support is needed
         cursor = stream_slicer if isinstance(stream_slicer, DeclarativeCursor) else None
@@ -3020,7 +3035,7 @@ def create_simple_retriever(
             self._create_component_from_model(
                 model=model.paginator,
                 config=config,
-                url_base=url_base,
+                url_base=_get_url(),
                 extractor_model=model.record_selector.extractor,
                 decoder=decoder,
                 cursor_used_for_stop_condition=cursor_used_for_stop_condition,