Skip to content

Commit 58e3eca

Browse files
authored
fix: dbt microbatch parameter conversion (#5298)
1 parent e0cd531 commit 58e3eca

File tree

2 files changed

+12
-11
lines changed

2 files changed

+12
-11
lines changed

sqlmesh/dbt/model.py

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -333,15 +333,8 @@ def model_kind(self, context: DbtContext) -> ModelKind:
333333
raise ConfigError(
334334
f"{self.canonical_name(context)}: 'event_time' is required for microbatch incremental strategy."
335335
)
336-
concurrent_batches = self._get_field_value("concurrent_batches")
337-
if concurrent_batches is True:
338-
if incremental_by_kind_kwargs.get("batch_size"):
339-
get_console().log_warning(
340-
f"'concurrent_batches' is set to True and 'batch_size' are defined in '{self.canonical_name(context)}'. The batch size will be set to the value of `batch_size`."
341-
)
342-
incremental_by_kind_kwargs["batch_size"] = incremental_by_kind_kwargs.get(
343-
"batch_size", 1
344-
)
336+
# dbt microbatch always processes batches in a size of 1
337+
incremental_by_kind_kwargs["batch_size"] = 1
345338
else:
346339
if not self.time_column:
347340
raise ConfigError(
@@ -651,6 +644,11 @@ def to_sqlmesh(
651644
)
652645
else:
653646
model_kwargs["start"] = begin
647+
# If user explicitly disables concurrent batches then we want to set depends on past to true which we
648+
# will do by including the model in the depends_on
649+
if self.concurrent_batches is not None and self.concurrent_batches is False:
650+
depends_on = model_kwargs.get("depends_on", set())
651+
depends_on.add(self.canonical_name(context))
654652

655653
model_kwargs["start"] = model_kwargs.get(
656654
"start", context.sqlmesh_config.model_defaults.start

tests/dbt/test_model.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ def test_load_microbatch_all_defined(
219219
column=exp.to_column("ds", quoted=True), format="%Y-%m-%d"
220220
)
221221
assert model.kind.batch_size == 1
222+
assert model.depends_on_self is False
222223

223224

224225
@pytest.mark.slow
@@ -259,7 +260,8 @@ def test_load_microbatch_all_defined_diff_values(
259260
assert model.kind.time_column == TimeColumn(
260261
column=exp.to_column("blah", quoted=True), format="%Y-%m-%d"
261262
)
262-
assert model.kind.batch_size is None
263+
assert model.kind.batch_size == 1
264+
assert model.depends_on_self is True
263265

264266

265267
@pytest.mark.slow
@@ -297,4 +299,5 @@ def test_load_microbatch_required_only(
297299
assert model.kind.time_column == TimeColumn(
298300
column=exp.to_column("ds", quoted=True), format="%Y-%m-%d"
299301
)
300-
assert model.kind.batch_size is None
302+
assert model.kind.batch_size == 1
303+
assert model.depends_on_self is False

0 commit comments

Comments
 (0)