Auto-fix lint and format issues

Author: octavia-squidington-iii

airbyte_cdk/sources/declarative/extractors/__init__.py

@@ -2,10 +2,10 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
-from airbyte_cdk.sources.declarative.extractors.key_value_extractor import KeyValueExtractor
 from airbyte_cdk.sources.declarative.extractors.combined_extractor import CombinedExtractor
+from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
 from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
+from airbyte_cdk.sources.declarative.extractors.key_value_extractor import KeyValueExtractor
 from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
 from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
 from airbyte_cdk.sources.declarative.extractors.response_to_file_extractor import (

airbyte_cdk/sources/declarative/extractors/combined_extractor.py

@@ -6,6 +6,7 @@
 from typing import Any, Iterable, List, MutableMapping
 
 import requests
+
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 
 
@@ -41,4 +42,4 @@ def extract_records(self, response: requests.Response) -> Iterable[MutableMappin
             merged = {}
             for record in records:
                 merged.update(record)  # merge all fields
-            yield merged
+            yield merged

airbyte_cdk/sources/declarative/extractors/key_value_extractor.py

@@ -3,10 +3,11 @@
 #
 
 from dataclasses import dataclass
+from itertools import islice
 from typing import Any, Iterable, MutableMapping
 
 import requests
-from itertools import islice
+
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 
 

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -618,9 +618,7 @@ class OAuthAuthenticator(BaseModel):
     scopes: Optional[List[str]] = Field(
         None,
         description="List of scopes that should be granted to the access token.",
-        examples=[
-            ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]
-        ],
+        examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
         title="Scopes",
     )
     token_expiry_date: Optional[str] = Field(
@@ -1126,28 +1124,24 @@ class OAuthConfigSpecification(BaseModel):
     class Config:
         extra = Extra.allow
 
-    oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = (
-        Field(
-            None,
-            description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n  oauth_user_input_from_connector_config_specification={\n    app_id: {\n      type: string\n      path_in_connector_config: ['app_id']\n    }\n  }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n  oauth_user_input_from_connector_config_specification={\n    app_id: {\n      type: string\n      path_in_connector_config: ['info', 'app_id']\n    }\n  }",
-            examples=[
-                {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
-                {
-                    "app_id": {
-                        "type": "string",
-                        "path_in_connector_config": ["info", "app_id"],
-                    }
-                },
-            ],
-            title="OAuth user input",
-        )
+    oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field(
+        None,
+        description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n  oauth_user_input_from_connector_config_specification={\n    app_id: {\n      type: string\n      path_in_connector_config: ['app_id']\n    }\n  }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n  oauth_user_input_from_connector_config_specification={\n    app_id: {\n      type: string\n      path_in_connector_config: ['info', 'app_id']\n    }\n  }",
+        examples=[
+            {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
+            {
+                "app_id": {
+                    "type": "string",
+                    "path_in_connector_config": ["info", "app_id"],
+                }
+            },
+        ],
+        title="OAuth user input",
     )
-    oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = (
-        Field(
-            None,
-            description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n  + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n  + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n  + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{{ auth_code_key }}": "{{ auth_code_value }}",\n              "{{ client_id_key }}": "{{ client_id_value }}",\n              "{{ client_secret_key }}": "{{ client_secret_value }}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
-            title="DeclarativeOAuth Connector Specification",
-        )
+    oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
+        None,
+        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n  + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n  + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n  + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{{ auth_code_key }}": "{{ auth_code_value }}",\n              "{{ client_id_key }}": "{{ client_id_value }}",\n              "{{ client_secret_key }}": "{{ client_secret_value }}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
+        title="DeclarativeOAuth Connector Specification",
     )
     complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
         None,
@@ -1165,9 +1159,7 @@ class Config:
     complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field(
         None,
         description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n    complete_oauth_server_input_specification={\n      client_id: {\n        type: string\n      },\n      client_secret: {\n        type: string\n      }\n    }",
-        examples=[
-            {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}
-        ],
+        examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}],
         title="OAuth input specification",
     )
     complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field(
@@ -1498,9 +1490,7 @@ class StreamConfig(BaseModel):
         examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]],
         title="Configs Pointer",
     )
-    default_values: Optional[List] = Field(
-        None, description="placeholder", title="Default Values"
-    )
+    default_values: Optional[List] = Field(None, description="placeholder", title="Default Values")
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -1868,9 +1858,7 @@ class KeyValueExtractor(BaseModel):
 class CombinedExtractor(BaseModel):
     type: Literal["CombinedExtractor"]
     extractors: List[
-        Union[
-            DpathExtractor, CombinedExtractor, KeyValueExtractor, CustomRecordExtractor
-        ]
+        Union[DpathExtractor, CombinedExtractor, KeyValueExtractor, CustomRecordExtractor]
     ] = Field(..., description="placeholder")
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
@@ -1912,17 +1900,13 @@ class ListPartitionRouter(BaseModel):
 
 class RecordSelector(BaseModel):
     type: Literal["RecordSelector"]
-    extractor: Union[
-        DpathExtractor, CombinedExtractor, KeyValueExtractor, CustomRecordExtractor
-    ]
+    extractor: Union[DpathExtractor, CombinedExtractor, KeyValueExtractor, CustomRecordExtractor]
     record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field(
         None,
         description="Responsible for filtering records to be emitted by the Source.",
         title="Record Filter",
     )
-    schema_normalization: Optional[
-        Union[SchemaNormalization, CustomSchemaNormalization]
-    ] = Field(
+    schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
         None,
         description="Responsible for normalization according to the schema.",
         title="Schema Normalization",
@@ -2168,9 +2152,7 @@ class Config:
         extra = Extra.allow
 
     type: Literal["DeclarativeStream"]
-    name: Optional[str] = Field(
-        "", description="The stream name.", example=["Users"], title="Name"
-    )
+    name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
     retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field(
         ...,
         description="Component used to coordinate how records are extracted across stream slices and request pages.",
@@ -2348,20 +2330,18 @@ class HttpRequester(BaseModelWithDeprecations):
         description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.",
         title="Fetch Properties from Endpoint",
     )
-    request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = (
-        Field(
-            None,
-            description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
-            examples=[
-                {"unit": "day"},
-                {
-                    "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
-                },
-                {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
-                {"sort_by[asc]": "updated_at"},
-            ],
-            title="Query Parameters",
-        )
+    request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field(
+        None,
+        description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
+        examples=[
+            {"unit": "day"},
+            {
+                "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
+            },
+            {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
+            {"sort_by[asc]": "updated_at"},
+        ],
+        title="Query Parameters",
     )
     request_headers: Optional[Union[Dict[str, str], str]] = Field(
         None,
@@ -2554,9 +2534,7 @@ class QueryProperties(BaseModel):
 
 class StateDelegatingStream(BaseModel):
     type: Literal["StateDelegatingStream"]
-    name: str = Field(
-        ..., description="The stream name.", example=["Users"], title="Name"
-    )
+    name: str = Field(..., description="The stream name.", example=["Users"], title="Name")
     full_refresh_stream: DeclarativeStream = Field(
         ...,
         description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.",
@@ -2645,9 +2623,7 @@ class AsyncRetriever(BaseModel):
     )
     download_extractor: Optional[
         Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor]
-    ] = Field(
-        None, description="Responsible for fetching the records from provided urls."
-    )
+    ] = Field(None, description="Responsible for fetching the records from provided urls.")
     creation_requester: Union[HttpRequester, CustomRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.",
@@ -2787,12 +2763,10 @@ class DynamicDeclarativeStream(BaseModel):
     stream_template: DeclarativeStream = Field(
         ..., description="Reference to the stream template.", title="Stream Template"
     )
-    components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = (
-        Field(
-            ...,
-            description="Component resolve and populates stream templates with components values.",
-            title="Components Resolver",
-        )
+    components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field(
+        ...,
+        description="Component resolve and populates stream templates with components values.",
+        title="Components Resolver",
     )
 
 

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -83,9 +83,9 @@
     Parser,
 )
 from airbyte_cdk.sources.declarative.extractors import (
+    CombinedExtractor,
     DpathExtractor,
     KeyValueExtractor,
-    CombinedExtractor,
     RecordFilter,
     RecordSelector,
     ResponseToFileExtractor,
@@ -144,6 +144,9 @@
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CheckStream as CheckStreamModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    CombinedExtractor as CombinedExtractorModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ComplexFieldType as ComplexFieldTypeModel,
 )
@@ -225,12 +228,6 @@
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DpathExtractor as DpathExtractorModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    KeyValueExtractor as KeyValueExtractorModel,
-)
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    CombinedExtractor as CombinedExtractorModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DpathFlattenFields as DpathFlattenFieldsModel,
 )
@@ -312,6 +309,9 @@
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     KeysToSnakeCase as KeysToSnakeCaseModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    KeyValueExtractor as KeyValueExtractorModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel,
 )
@@ -2252,9 +2252,7 @@ def create_combined_extractor(
         **kwargs: Any,
     ) -> CombinedExtractor:
         extractors = [
-            self._create_component_from_model(
-                model=extractor, decoder=decoder, config=config
-            )
+            self._create_component_from_model(model=extractor, decoder=decoder, config=config)
             for extractor in model.extractors
         ]
 
@@ -3697,12 +3695,16 @@ def create_stream_config(
     def create_config_components_resolver(
         self, model: ConfigComponentsResolverModel, config: Config
     ) -> Any:
+        model_stream_configs = (
+            model.stream_config if isinstance(model.stream_config, list) else [model.stream_config]
+        )
 
-        model_stream_configs = model.stream_config if isinstance(model.stream_config, list) else [model.stream_config]
-
-        stream_configs = [self._create_component_from_model(
-            stream_config, config=config, parameters=model.parameters or {}
-        ) for stream_config in model_stream_configs]
+        stream_configs = [
+            self._create_component_from_model(
+                stream_config, config=config, parameters=model.parameters or {}
+            )
+            for stream_config in model_stream_configs
+        ]
 
         components_mapping = [
             self._create_component_from_model(