Skip to content

Commit 1444335

Browse files
committed
feat: support custom config transformations
1 parent 940e1fc commit 1444335

File tree

4 files changed

+97
-2
lines changed

4 files changed

+97
-2
lines changed

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3852,6 +3852,7 @@ definitions:
38523852
- "$ref": "#/definitions/ConfigRemapField"
38533853
- "$ref": "#/definitions/ConfigAddFields"
38543854
- "$ref": "#/definitions/ConfigRemoveFields"
3855+
- "$ref": "#/definitions/CustomConfigTransformation"
38553856
default: []
38563857
validations:
38573858
title: Validations
@@ -3885,6 +3886,7 @@ definitions:
38853886
- "$ref": "#/definitions/ConfigRemapField"
38863887
- "$ref": "#/definitions/ConfigAddFields"
38873888
- "$ref": "#/definitions/ConfigRemoveFields"
3889+
- "$ref": "#/definitions/CustomConfigTransformation"
38883890
default: []
38893891
SubstreamPartitionRouter:
38903892
title: Substream Partition Router
@@ -4556,6 +4558,25 @@ definitions:
45564558
- "{{ property is integer }}"
45574559
- "{{ property|length > 5 }}"
45584560
- "{{ property == 'some_string_to_match' }}"
4561+
CustomConfigTransformation:
4562+
title: Custom Config Transformation
4563+
description: A custom config transformation that can be used to transform the connector configuration.
4564+
type: object
4565+
required:
4566+
- type
4567+
- class_name
4568+
properties:
4569+
type:
4570+
type: string
4571+
enum: [CustomConfigTransformation]
4572+
class_name:
4573+
type: string
4574+
description: Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.
4575+
examples:
4576+
- "source_declarative_manifest.components.MyCustomConfigTransformation"
4577+
parameters:
4578+
type: object
4579+
description: Additional parameters to be passed to the custom config transformation.
45594580
interpolation:
45604581
variables:
45614582
- title: config

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,20 @@ class Config:
160160
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
161161

162162

163+
class CustomConfigTransformation(BaseModel):
164+
class Config:
165+
extra = Extra.allow
166+
167+
type: Literal["CustomConfigTransformation"]
168+
class_name: str = Field(
169+
...,
170+
description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.",
171+
examples=["source_declarative_manifest.components.MyCustomConfigTransformation"],
172+
title="Class Name",
173+
)
174+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
175+
176+
163177
class CustomErrorHandler(BaseModel):
164178
class Config:
165179
extra = Extra.allow
@@ -2149,7 +2163,7 @@ class ConfigMigration(BaseModel):
21492163
description: Optional[str] = Field(
21502164
None, description="The description/purpose of the config migration."
21512165
)
2152-
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field(
2166+
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]] = Field(
21532167
...,
21542168
description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.",
21552169
title="Transformations",
@@ -2166,7 +2180,7 @@ class Config:
21662180
title="Config Migrations",
21672181
)
21682182
transformations: Optional[
2169-
List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]]
2183+
List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields, CustomConfigTransformation]]
21702184
] = Field(
21712185
[],
21722186
description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.",

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,9 @@
225225
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
226226
CustomValidationStrategy as CustomValidationStrategyModel,
227227
)
228+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
229+
CustomConfigTransformation as CustomConfigTransformationModel,
230+
)
228231
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
229232
DatetimeBasedCursor as DatetimeBasedCursorModel,
230233
)
@@ -687,6 +690,7 @@ def _init_mappings(self) -> None:
687690
CustomPartitionRouterModel: self.create_custom_component,
688691
CustomTransformationModel: self.create_custom_component,
689692
CustomValidationStrategyModel: self.create_custom_component,
693+
CustomConfigTransformationModel: self.create_custom_component,
690694
DatetimeBasedCursorModel: self.create_datetime_based_cursor,
691695
DeclarativeStreamModel: self.create_declarative_stream,
692696
DefaultErrorHandlerModel: self.create_default_error_handler,
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#
2+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
3+
#
4+
5+
from typing import Any, Dict, MutableMapping, Optional
6+
7+
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
8+
ConfigTransformation,
9+
)
10+
11+
12+
class MockCustomConfigTransformation(ConfigTransformation):
13+
"""
14+
A mock custom config transformation for testing purposes.
15+
This simulates what a real custom transformation would look like.
16+
"""
17+
18+
def __init__(self, parameters: Optional[Dict[str, Any]] = None) -> None:
19+
self.parameters = parameters or {}
20+
21+
def transform(self, config: MutableMapping[str, Any]) -> None:
22+
"""
23+
Transform the config by adding a test field.
24+
This simulates the behavior of a real custom transformation.
25+
"""
26+
# Only modify user config keys, avoid framework-injected keys
27+
# Check if there are any user keys (not starting with __)
28+
has_user_keys = any(not key.startswith('__') for key in config.keys())
29+
if has_user_keys:
30+
config['transformed_field'] = 'transformed_value'
31+
if self.parameters.get('additional_field'):
32+
config['additional_field'] = self.parameters['additional_field']
33+
34+
35+
def test_given_valid_config_when_transform_then_config_is_transformed():
36+
"""Test that a custom config transformation properly transforms the config."""
37+
transformation = MockCustomConfigTransformation()
38+
config = {"original_field": "original_value"}
39+
40+
transformation.transform(config)
41+
42+
assert config["original_field"] == "original_value"
43+
assert config["transformed_field"] == "transformed_value"
44+
45+
46+
def test_given_config_with_parameters_when_transform_then_parameters_are_applied():
47+
"""Test that custom config transformation respects parameters."""
48+
parameters = {"additional_field": "parameter_value"}
49+
transformation = MockCustomConfigTransformation(parameters=parameters)
50+
config = {"original_field": "original_value"}
51+
52+
transformation.transform(config)
53+
54+
assert config["original_field"] == "original_value"
55+
assert config["transformed_field"] == "transformed_value"
56+
assert config["additional_field"] == "parameter_value"

0 commit comments

Comments
 (0)