Skip to content
This repository was archived by the owner on Sep 2, 2025. It is now read-only.

Commit 827fc78

Browse files
authored
[Feature] Allow copy_partitions when using microbatch (#1421)
1 parent 4d255b2 commit 827fc78

File tree

4 files changed

+33
-2
lines changed

4 files changed

+33
-2
lines changed
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
kind: Features
2+
body: Allow copy_partitions in microbatch
3+
time: 2024-12-02T22:38:35.479052Z
4+
custom:
5+
Author: borjavb
6+
Issue: "1414"

dbt/include/bigquery/macros/materializations/incremental.sql

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,9 @@
9595

9696
{{ run_hooks(pre_hooks) }}
9797

98-
{% if partition_by.copy_partitions is true and strategy != 'insert_overwrite' %} {#-- We can't copy partitions with merge strategy --#}
98+
{% if partition_by.copy_partitions is true and strategy not in ['insert_overwrite', 'microbatch'] %} {#-- We can't copy partitions with merge strategy --#}
9999
{% set wrong_strategy_msg -%}
100-
The 'copy_partitions' option requires the 'incremental_strategy' option to be set to 'insert_overwrite'.
100+
The 'copy_partitions' option requires the 'incremental_strategy' option to be set to 'insert_overwrite' or 'microbatch'.
101101
{%- endset %}
102102
{% do exceptions.raise_compiler_error(wrong_strategy_msg) %}
103103

tests/functional/adapter/incremental/incremental_strategy_fixtures.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,3 +629,21 @@
629629
}}
630630
select * from {{ ref('input_model') }}
631631
"""
632+
633+
microbatch_model_no_unique_id_copy_partitions_sql = """
634+
{{ config(
635+
materialized='incremental',
636+
incremental_strategy='microbatch',
637+
partition_by={
638+
'field': 'event_time',
639+
'data_type': 'timestamp',
640+
'granularity': 'day',
641+
'copy_partitions': true
642+
},
643+
event_time='event_time',
644+
batch_size='day',
645+
begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)
646+
)
647+
}}
648+
select * from {{ ref('input_model') }}
649+
"""

tests/functional/adapter/incremental/test_incremental_microbatch.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
microbatch_input_sql,
1414
microbatch_model_no_partition_by_sql,
1515
microbatch_model_invalid_partition_by_sql,
16+
microbatch_model_no_unique_id_copy_partitions_sql,
1617
microbatch_input_event_time_date_sql,
1718
microbatch_input_event_time_datetime_sql,
1819
)
@@ -79,3 +80,9 @@ def test_execution_failure_no_partition_by(self, project):
7980
"The 'microbatch' strategy requires a `partition_by` config with the same granularity as its configured `batch_size`"
8081
in stdout
8182
)
83+
84+
85+
class TestBigQueryMicrobatchWithCopyPartitions(BaseMicrobatch):
86+
@pytest.fixture(scope="class")
87+
def microbatch_model_sql(self) -> str:
88+
return microbatch_model_no_unique_id_copy_partitions_sql

0 commit comments

Comments
 (0)