Skip to content

Commit ce44cc5

Browse files
pnilanpnilan
authored
feat: extend spec class for config migrations (#538)
1 parent d458e8f commit ce44cc5

File tree

11 files changed

+1216
-236
lines changed

11 files changed

+1216
-236
lines changed

‎airbyte_cdk/sources/declarative/declarative_component_schema.yaml‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3803,6 +3803,61 @@ definitions:
38033803
title: Advanced Auth
38043804
description: Advanced specification for configuring the authentication flow.
38053805
"$ref": "#/definitions/AuthFlow"
3806+
config_normalization_rules:
3807+
title: Config Normalization Rules
3808+
type: object
3809+
additionalProperties: false
3810+
properties:
3811+
config_migrations:
3812+
title: Config Migrations
3813+
description: The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.
3814+
type: array
3815+
items:
3816+
"$ref": "#/definitions/ConfigMigration"
3817+
default: []
3818+
transformations:
3819+
title: Transformations
3820+
description: The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.
3821+
type: array
3822+
items:
3823+
anyOf:
3824+
- "$ref": "#/definitions/ConfigRemapField"
3825+
- "$ref": "#/definitions/ConfigAddFields"
3826+
- "$ref": "#/definitions/ConfigRemoveFields"
3827+
default: []
3828+
validations:
3829+
title: Validations
3830+
description: The list of validations that will be performed on the incoming config at the start of each sync.
3831+
type: array
3832+
items:
3833+
anyOf:
3834+
- "$ref": "#/definitions/DpathValidator"
3835+
- "$ref": "#/definitions/PredicateValidator"
3836+
default: []
3837+
ConfigMigration:
3838+
title: Config Migration
3839+
description: A config migration that will be applied on the incoming config at the start of a sync.
3840+
type: object
3841+
required:
3842+
- type
3843+
- transformations
3844+
properties:
3845+
type:
3846+
type: string
3847+
enum: [ConfigMigration]
3848+
description:
3849+
type: string
3850+
description: The description/purpose of the config migration.
3851+
transformations:
3852+
title: Transformations
3853+
description: The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.
3854+
type: array
3855+
items:
3856+
anyOf:
3857+
- "$ref": "#/definitions/ConfigRemapField"
3858+
- "$ref": "#/definitions/ConfigAddFields"
3859+
- "$ref": "#/definitions/ConfigRemoveFields"
3860+
default: []
38063861
SubstreamPartitionRouter:
38073862
title: Substream Partition Router
38083863
description: Partition router that is used to retrieve records that have been partitioned according to records from the specified parent streams. An example of a parent stream is automobile brands and the substream would be the various car models associated with each branch.
@@ -4164,6 +4219,224 @@ definitions:
41644219
description: The GraphQL query to be executed
41654220
default: {}
41664221
additionalProperties: true
4222+
DpathValidator:
4223+
title: Dpath Validator
4224+
description: Validator that extracts the value located at a given field path.
4225+
type: object
4226+
required:
4227+
- type
4228+
- field_path
4229+
- validation_strategy
4230+
properties:
4231+
type:
4232+
type: string
4233+
enum: [DpathValidator]
4234+
field_path:
4235+
title: Field Path
4236+
description: List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.
4237+
type: array
4238+
items:
4239+
type: string
4240+
interpolation_context:
4241+
- config
4242+
examples:
4243+
- ["data"]
4244+
- ["data", "records"]
4245+
- ["data", "{{ parameters.name }}"]
4246+
- ["data", "*", "record"]
4247+
validation_strategy:
4248+
title: Validation Strategy
4249+
description: The condition that the specified config value will be evaluated against
4250+
anyOf:
4251+
- "$ref": "#/definitions/ValidateAdheresToSchema"
4252+
PredicateValidator:
4253+
title: Predicate Validator
4254+
description: Validator that applies a validation strategy to a specified value.
4255+
type: object
4256+
required:
4257+
- type
4258+
- value
4259+
- validation_strategy
4260+
properties:
4261+
type:
4262+
type: string
4263+
enum: [PredicateValidator]
4264+
value:
4265+
title: Value
4266+
description: The value to be validated. Can be a literal value or interpolated from configuration.
4267+
type:
4268+
- string
4269+
- number
4270+
- object
4271+
- array
4272+
- boolean
4273+
- "null"
4274+
interpolation_context:
4275+
- config
4276+
examples:
4277+
- "test-value"
4278+
- "{{ config['api_version'] }}"
4279+
- "{{ config['tenant_id'] }}"
4280+
- 123
4281+
validation_strategy:
4282+
title: Validation Strategy
4283+
description: The validation strategy to apply to the value.
4284+
anyOf:
4285+
- "$ref": "#/definitions/ValidateAdheresToSchema"
4286+
ValidateAdheresToSchema:
4287+
title: Validate Adheres To Schema
4288+
description: Validates that a user-provided schema adheres to a specified JSON schema.
4289+
type: object
4290+
required:
4291+
- type
4292+
- base_schema
4293+
properties:
4294+
type:
4295+
type: string
4296+
enum: [ValidateAdheresToSchema]
4297+
base_schema:
4298+
title: Base JSON Schema
4299+
description: The base JSON schema against which the user-provided schema will be validated.
4300+
type:
4301+
- string
4302+
- object
4303+
interpolation_context:
4304+
- config
4305+
examples:
4306+
- "{{ config['report_validation_schema'] }}"
4307+
- |
4308+
'{
4309+
"$schema": "http://json-schema.org/draft-07/schema#",
4310+
"title": "Person",
4311+
"type": "object",
4312+
"properties": {
4313+
"name": {
4314+
"type": "string",
4315+
"description": "The person's name"
4316+
},
4317+
"age": {
4318+
"type": "integer",
4319+
"minimum": 0,
4320+
"description": "The person's age"
4321+
}
4322+
},
4323+
"required": ["name", "age"]
4324+
}'
4325+
- $schema: "http://json-schema.org/draft-07/schema#"
4326+
title: Person
4327+
type: object
4328+
properties:
4329+
name:
4330+
type: string
4331+
description: "The person's name"
4332+
age:
4333+
type: integer
4334+
minimum: 0
4335+
description: "The person's age"
4336+
required:
4337+
- name
4338+
- age
4339+
ConfigRemapField:
4340+
title: Remap Field
4341+
description: Transformation that remaps a field's value to another value based on a static map.
4342+
type: object
4343+
required:
4344+
- type
4345+
- map
4346+
- field_path
4347+
properties:
4348+
type:
4349+
type: string
4350+
enum: [ConfigRemapField]
4351+
map:
4352+
title: Value Mapping
4353+
description: A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.
4354+
interpolation_context:
4355+
- config
4356+
type:
4357+
- object
4358+
- string
4359+
additionalProperties: true
4360+
examples:
4361+
- pending: "in_progress"
4362+
done: "completed"
4363+
cancelled: "terminated"
4364+
- "{{ config['status_mapping'] }}"
4365+
field_path:
4366+
title: Field Path
4367+
description: The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.
4368+
interpolation_context:
4369+
- config
4370+
type: array
4371+
items:
4372+
type: string
4373+
examples:
4374+
- ["status"]
4375+
- ["data", "status"]
4376+
- ["data", "{{ config.name }}", "status"]
4377+
- ["data", "*", "status"]
4378+
ConfigAddFields:
4379+
title: Config Add Fields
4380+
description: Transformation that adds fields to a config. The path of the added field can be nested.
4381+
type: object
4382+
required:
4383+
- type
4384+
- fields
4385+
properties:
4386+
type:
4387+
type: string
4388+
enum: [ConfigAddFields]
4389+
fields:
4390+
title: Fields
4391+
description: A list of transformations (path and corresponding value) that will be added to the config.
4392+
type: array
4393+
items:
4394+
"$ref": "#/definitions/AddedFieldDefinition"
4395+
condition:
4396+
description: Fields will be added if expression is evaluated to True.
4397+
type: string
4398+
default: ""
4399+
interpolation_context:
4400+
- config
4401+
- property
4402+
examples:
4403+
- "{{ config['environemnt'] == 'sandbox' }}"
4404+
- "{{ property is integer }}"
4405+
- "{{ property|length > 5 }}"
4406+
- "{{ property == 'some_string_to_match' }}"
4407+
ConfigRemoveFields:
4408+
title: Config Remove Fields
4409+
description: Transformation that removes a field from the config.
4410+
type: object
4411+
required:
4412+
- type
4413+
- field_pointers
4414+
properties:
4415+
type:
4416+
type: string
4417+
enum: [ConfigRemoveFields]
4418+
field_pointers:
4419+
title: Field Pointers
4420+
description: A list of field pointers to be removed from the config.
4421+
type: array
4422+
items:
4423+
items:
4424+
type: string
4425+
examples:
4426+
- ["tags"]
4427+
- [["content", "html"], ["content", "plain_text"]]
4428+
condition:
4429+
description: Fields will be removed if expression is evaluated to True.
4430+
type: string
4431+
default: ""
4432+
interpolation_context:
4433+
- config
4434+
- property
4435+
examples:
4436+
- "{{ config['environemnt'] == 'sandbox' }}"
4437+
- "{{ property is integer }}"
4438+
- "{{ property|length > 5 }}"
4439+
- "{{ property == 'some_string_to_match' }}"
41674440
interpolation:
41684441
variables:
41694442
- title: config

0 commit comments

Comments
 (0)