You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
description: The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.
913
+
type: array
914
+
items:
915
+
type: string
916
+
examples:
917
+
- "%Y-%m-%dT%H:%M:%S.%f%z"
918
+
- "%Y-%m-%d"
919
+
- "%s"
920
+
start_datetime:
921
+
title: Start Datetime
922
+
description: The datetime that determines the earliest record that should be synced.
923
+
anyOf:
924
+
- "$ref": "#/definitions/MinMaxDatetime"
925
+
- type: string
926
+
title: Interpolated Value
927
+
interpolation_context:
928
+
- config
929
+
examples:
930
+
- "2020-01-1T00:00:00Z"
931
+
- "{{ config['start_time'] }}"
932
+
start_time_option:
933
+
title: Inject Start Time Into Outgoing HTTP Request
934
+
description: Optionally configures how the start datetime will be sent in requests to the source API.
935
+
"$ref": "#/definitions/RequestOption"
936
+
end_datetime:
937
+
title: End Datetime
938
+
description: The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.
939
+
anyOf:
940
+
- "$ref": "#/definitions/MinMaxDatetime"
941
+
- type: string
942
+
title: Interpolated Value
943
+
interpolation_context:
944
+
- config
945
+
examples:
946
+
- "2021-01-1T00:00:00Z"
947
+
- "{{ now_utc() }}"
948
+
- "{{ day_delta(-1) }}"
949
+
end_time_option:
950
+
title: Inject End Time Into Outgoing HTTP Request
951
+
description: Optionally configures how the end datetime will be sent in requests to the source API.
952
+
"$ref": "#/definitions/RequestOption"
910
953
datetime_format:
911
954
title: Outgoing Datetime Format
912
955
description: |
@@ -948,27 +991,6 @@ definitions:
948
991
- "%s"
949
992
- "%ms"
950
993
- "%s_as_float"
951
-
start_datetime:
952
-
title: Start Datetime
953
-
description: The datetime that determines the earliest record that should be synced.
954
-
anyOf:
955
-
- type: string
956
-
- "$ref": "#/definitions/MinMaxDatetime"
957
-
interpolation_context:
958
-
- config
959
-
examples:
960
-
- "2020-01-1T00:00:00Z"
961
-
- "{{ config['start_time'] }}"
962
-
cursor_datetime_formats:
963
-
title: Cursor Datetime Formats
964
-
description: The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.
965
-
type: array
966
-
items:
967
-
type: string
968
-
examples:
969
-
- "%Y-%m-%dT%H:%M:%S.%f%z"
970
-
- "%Y-%m-%d"
971
-
- "%s"
972
994
cursor_granularity:
973
995
title: Cursor Granularity
974
996
description:
@@ -977,22 +999,6 @@ definitions:
977
999
type: string
978
1000
examples:
979
1001
- "PT1S"
980
-
end_datetime:
981
-
title: End Datetime
982
-
description: The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.
983
-
anyOf:
984
-
- type: string
985
-
- "$ref": "#/definitions/MinMaxDatetime"
986
-
interpolation_context:
987
-
- config
988
-
examples:
989
-
- "2021-01-1T00:00:00Z"
990
-
- "{{ now_utc() }}"
991
-
- "{{ day_delta(-1) }}"
992
-
end_time_option:
993
-
title: Inject End Time Into Outgoing HTTP Request
994
-
description: Optionally configures how the end datetime will be sent in requests to the source API.
995
-
"$ref": "#/definitions/RequestOption"
996
1002
is_data_feed:
997
1003
title: Whether the target API is formatted as a data feed
998
1004
description: A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.
@@ -1032,10 +1038,6 @@ definitions:
1032
1038
type: string
1033
1039
examples:
1034
1040
- "starting_time"
1035
-
start_time_option:
1036
-
title: Inject Start Time Into Outgoing HTTP Request
1037
-
description: Optionally configures how the start datetime will be sent in requests to the source API.
1038
-
"$ref": "#/definitions/RequestOption"
1039
1041
step:
1040
1042
title: Step
1041
1043
description: The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.
@@ -1429,14 +1431,15 @@ definitions:
1429
1431
- "$ref": "#/definitions/InlineSchemaLoader"
1430
1432
- "$ref": "#/definitions/DynamicSchemaLoader"
1431
1433
- "$ref": "#/definitions/JsonFileSchemaLoader"
1432
-
- "$ref": "#/definitions/CustomSchemaLoader"
1433
-
-type: array
1434
+
- title: Multiple Schema Loaders
1435
+
type: array
1434
1436
items:
1435
1437
anyOf:
1436
1438
- "$ref": "#/definitions/InlineSchemaLoader"
1437
1439
- "$ref": "#/definitions/DynamicSchemaLoader"
1438
1440
- "$ref": "#/definitions/JsonFileSchemaLoader"
1439
1441
- "$ref": "#/definitions/CustomSchemaLoader"
1442
+
- "$ref": "#/definitions/CustomSchemaLoader"
1440
1443
# TODO we have move the transformation to the RecordSelector level in the code but kept this here for
1441
1444
# compatibility reason. We should eventually move this to align with the code.
1442
1445
transformations:
@@ -1446,13 +1449,13 @@ definitions:
1446
1449
items:
1447
1450
anyOf:
1448
1451
- "$ref": "#/definitions/AddFields"
1449
-
- "$ref": "#/definitions/CustomTransformation"
1450
1452
- "$ref": "#/definitions/RemoveFields"
1451
1453
- "$ref": "#/definitions/KeysToLower"
1452
1454
- "$ref": "#/definitions/KeysToSnakeCase"
1453
1455
- "$ref": "#/definitions/FlattenFields"
1454
1456
- "$ref": "#/definitions/DpathFlattenFields"
1455
1457
- "$ref": "#/definitions/KeysReplace"
1458
+
- "$ref": "#/definitions/CustomTransformation"
1456
1459
state_migrations:
1457
1460
title: State Migrations
1458
1461
description: Array of state migrations to be applied on the input state
description: If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.
@@ -3999,6 +4004,7 @@ definitions:
3999
4004
additionalProperties: true
4000
4005
HttpComponentsResolver:
4001
4006
type: object
4007
+
title: Http Components Resolver
4002
4008
description: (This component is experimental. Use at your own risk.) Component resolve and populates stream templates with components fetched via an HTTP retriever.
4003
4009
properties:
4004
4010
type:
@@ -4008,9 +4014,9 @@ definitions:
4008
4014
title: Retriever
4009
4015
description: Component used to coordinate how records are extracted across stream slices and request pages.
4010
4016
anyOf:
4017
+
- "$ref": "#/definitions/SimpleRetriever"
4011
4018
- "$ref": "#/definitions/AsyncRetriever"
4012
4019
- "$ref": "#/definitions/CustomRetriever"
4013
-
- "$ref": "#/definitions/SimpleRetriever"
4014
4020
components_mapping:
4015
4021
type: array
4016
4022
items:
@@ -4050,6 +4056,7 @@ definitions:
4050
4056
additionalProperties: true
4051
4057
ConfigComponentsResolver:
4052
4058
type: object
4059
+
title: Config Components Resolver
4053
4060
description: (This component is experimental. Use at your own risk.) Resolves and populates stream templates with components fetched from the source config.
0 commit comments