Skip to content

Commit 3983495

Browse files
committed
update per comments
1 parent 3638129 commit 3983495

File tree

6 files changed

+378
-50
lines changed

6 files changed

+378
-50
lines changed

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3814,6 +3814,7 @@ definitions:
38143814
type: array
38153815
items:
38163816
"$ref": "#/definitions/ConfigMigration"
3817+
default: []
38173818
transformations:
38183819
title: Transformations
38193820
description: The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.
@@ -3823,6 +3824,7 @@ definitions:
38233824
- "$ref": "#/definitions/ConfigRemapField"
38243825
- "$ref": "#/definitions/ConfigAddFields"
38253826
- "$ref": "#/definitions/ConfigRemoveFields"
3827+
default: []
38263828
validations:
38273829
title: Validations
38283830
description: The list of validations that will be performed on the incoming config at the start of each sync.
@@ -3831,13 +3833,18 @@ definitions:
38313833
anyOf:
38323834
- "$ref": "#/definitions/DpathValidator"
38333835
- "$ref": "#/definitions/PredicateValidator"
3836+
default: []
38343837
ConfigMigration:
38353838
title: Config Migration
38363839
description: A config migration that will be applied on the incoming config at the start of a sync.
38373840
type: object
38383841
required:
3842+
- type
38393843
- transformations
38403844
properties:
3845+
type:
3846+
type: string
3847+
enum: [ConfigMigration]
38413848
description:
38423849
type: string
38433850
description: The description/purpose of the config migration.
@@ -4238,7 +4245,7 @@ definitions:
42384245
- ["data", "{{ parameters.name }}"]
42394246
- ["data", "*", "record"]
42404247
validation_strategy:
4241-
title: Validation Stragey
4248+
title: Validation Strategy
42424249
description: The condition that the specified config value will be evaluated against
42434250
anyOf:
42444251
- "$ref": "#/definitions/ValidateAdheresToSchema"

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
2+
13
# generated by datamodel-codegen:
24
# filename: declarative_component_schema.yaml
35

@@ -1995,7 +1997,7 @@ class DpathValidator(BaseModel):
19951997
validation_strategy: ValidateAdheresToSchema = Field(
19961998
...,
19971999
description="The condition that the specified config value will be evaluated against",
1998-
title="Validation Stragey",
2000+
title="Validation Strategy",
19992001
)
20002002

20012003

@@ -2094,6 +2096,7 @@ class Config:
20942096

20952097

20962098
class ConfigMigration(BaseModel):
2099+
type: Literal["ConfigMigration"]
20972100
description: Optional[str] = Field(
20982101
None, description="The description/purpose of the config migration."
20992102
)
@@ -2109,19 +2112,19 @@ class Config:
21092112
extra = Extra.forbid
21102113

21112114
config_migrations: Optional[List[ConfigMigration]] = Field(
2112-
None,
2115+
[],
21132116
description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.",
21142117
title="Config Migrations",
21152118
)
21162119
transformations: Optional[
21172120
List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]]
21182121
] = Field(
2119-
None,
2122+
[],
21202123
description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.",
21212124
title="Transformations",
21222125
)
21232126
validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field(
2124-
None,
2127+
[],
21252128
description="The list of validations that will be performed on the incoming config at the start of each sync.",
21262129
title="Validations",
21272130
)

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

Lines changed: 41 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -499,7 +499,7 @@
499499
RequestOptionsProvider,
500500
)
501501
from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
502-
from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
502+
from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester
503503
from airbyte_cdk.sources.declarative.resolvers import (
504504
ComponentMappingDefinition,
505505
ConfigComponentsResolver,
@@ -542,6 +542,9 @@
542542
ConfigRemapField,
543543
ConfigRemoveFields,
544544
)
545+
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
546+
ConfigTransformation,
547+
)
545548
from airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields import (
546549
DpathFlattenFields,
547550
KeyTransformation,
@@ -823,9 +826,10 @@ def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None:
823826
def create_config_migration(
824827
self, model: ConfigMigrationModel, config: Config
825828
) -> ConfigMigration:
826-
transformations = []
827-
for transformation in model.transformations:
828-
transformations.append(self._create_component_from_model(transformation, config))
829+
transformations: List[ConfigTransformation] = [
830+
self._create_component_from_model(transformation, config)
831+
for transformation in model.transformations
832+
]
829833

830834
return ConfigMigration(
831835
description=model.description,
@@ -3604,24 +3608,39 @@ def _get_job_timeout() -> datetime.timedelta:
36043608
)
36053609

36063610
def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec:
3607-
config_migrations = []
3608-
config_transformations = []
3609-
config_validations = []
3610-
3611-
if model.config_normalization_rules:
3612-
if model.config_normalization_rules.config_migrations:
3613-
for migration in model.config_normalization_rules.config_migrations:
3614-
config_migrations.append(self._create_component_from_model(migration, config))
3615-
3616-
if model.config_normalization_rules.transformations:
3617-
for transformation in model.config_normalization_rules.transformations:
3618-
config_transformations.append(
3619-
self._create_component_from_model(transformation, config)
3620-
)
3621-
3622-
if model.config_normalization_rules.validations:
3623-
for validation in model.config_normalization_rules.validations:
3624-
config_validations.append(self._create_component_from_model(validation, config))
3611+
config_migrations = [
3612+
self._create_component_from_model(migration, config)
3613+
for migration in (
3614+
model.config_normalization_rules.config_migrations
3615+
if (
3616+
model.config_normalization_rules
3617+
and model.config_normalization_rules.config_migrations
3618+
)
3619+
else []
3620+
)
3621+
]
3622+
config_transformations = [
3623+
self._create_component_from_model(transformation, config)
3624+
for transformation in (
3625+
model.config_normalization_rules.transformations
3626+
if (
3627+
model.config_normalization_rules
3628+
and model.config_normalization_rules.transformations
3629+
)
3630+
else []
3631+
)
3632+
]
3633+
config_validations = [
3634+
self._create_component_from_model(validation, config)
3635+
for validation in (
3636+
model.config_normalization_rules.validations
3637+
if (
3638+
model.config_normalization_rules
3639+
and model.config_normalization_rules.validations
3640+
)
3641+
else []
3642+
)
3643+
]
36253644

36263645
return Spec(
36273646
connection_specification=model.connection_specification,

airbyte_cdk/sources/declarative/spec/spec.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,19 +98,14 @@ def migrate_config(
9898
for message in self.message_repository.consume_queue():
9999
print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
100100

101-
def transform_config(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
101+
def transform_config(self, config: MutableMapping[str, Any]) -> None:
102102
"""
103103
Apply all config transformations to the provided config.
104104
105105
:param config: The user-provided configuration
106-
:return: The transformed configuration
107106
"""
108-
mutable_config = dict(config)
109-
110107
for transformation in self.config_transformations:
111-
transformation.transform(mutable_config)
112-
113-
return mutable_config
108+
transformation.transform(config)
114109

115110
def validate_config(self, config: MutableMapping[str, Any]) -> None:
116111
"""

0 commit comments

Comments
 (0)