Skip to content

Commit 959cfe0

Browse files
committed
add option to not include parent slice in child stream slice partitions
1 parent 4b73b46 commit 959cfe0

File tree

3 files changed

+18
-4
lines changed

3 files changed

+18
-4
lines changed

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3881,6 +3881,11 @@ definitions:
38813881
type: array
38823882
items:
38833883
"$ref": "#/definitions/ParentStreamConfig"
3884+
include_parent_slice:
3885+
title: Include Parent Slice
3886+
description: If False, the parent stream slice will not be included in the child stream slice.
3887+
type: boolean
3888+
default: true
38843889
$parameters:
38853890
type: object
38863891
additionalProperties: true

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2867,6 +2867,10 @@ class SubstreamPartitionRouter(BaseModel):
28672867
description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.",
28682868
title="Parent Stream Configs",
28692869
)
2870+
include_parent_slice: Optional[bool] = Field(True,
2871+
description="If False, the parent stream slice will not be included in the child stream slice.",
2872+
title="Include Parent Slice",
2873+
)
28702874
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
28712875

28722876

airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ class SubstreamPartitionRouter(PartitionRouter):
8989
parent_stream_configs: List[ParentStreamConfig]
9090
config: Config
9191
parameters: InitVar[Mapping[str, Any]]
92+
include_parent_slice: Optional[bool] = True
9293

9394
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
9495
if not self.parent_stream_configs:
@@ -221,11 +222,15 @@ def stream_slices(self) -> Iterable[StreamSlice]:
221222
**extracted_extra_fields,
222223
}
223224

224-
yield StreamSlice(
225-
partition={
225+
slice_partition = {
226226
partition_field: partition_value,
227-
"parent_slice": parent_partition or {},
228-
},
227+
}
228+
229+
if self.include_parent_slice:
230+
slice_partition["parent_slice"] = parent_partition or {}
231+
232+
yield StreamSlice(
233+
partition=slice_partition,
229234
cursor_slice={},
230235
extra_fields=extracted_extra_fields,
231236
)

0 commit comments

Comments
 (0)