From e05c016caf95480568e6fd2c6976d0aab84de34b Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Thu, 8 Jan 2026 10:11:59 +0000 Subject: [PATCH 01/16] fix bug with v1 materialisation where multipled foreign keys from one table to another aren't retained on an incremental run Signed-off-by: Ewan Keith --- .../incremental/incremental.sql | 4 + .../adapter/incremental/fixtures.py | 61 +++++++++++++ .../test_incremental_constraints.py | 91 +++++++++++++++++++ .../unit/relation_configs/test_constraint.py | 44 +++++++++ 4 files changed, 200 insertions(+) diff --git a/dbt/include/databricks/macros/materializations/incremental/incremental.sql b/dbt/include/databricks/macros/materializations/incremental/incremental.sql index 82148795a..b6bf6be93 100644 --- a/dbt/include/databricks/macros/materializations/incremental/incremental.sql +++ b/dbt/include/databricks/macros/materializations/incremental/incremental.sql @@ -177,6 +177,7 @@ {% set tags = _configuration_changes.changes.get("tags", None) %} {% set tblproperties = _configuration_changes.changes.get("tblproperties", None) %} {% set liquid_clustering = _configuration_changes.changes.get("liquid_clustering") %} + {% set constraints = _configuration_changes.changes.get("constraints") %} {% if tags is not none %} {% do apply_tags(target_relation, tags.set_tags) %} {%- endif -%} @@ -186,6 +187,9 @@ {% if liquid_clustering is not none %} {% do apply_liquid_clustered_cols(target_relation, liquid_clustering) %} {% endif %} + {% if constraints %} + {{ apply_constraints(target_relation, constraints) }} + {% endif %} {%- endif -%} {% do persist_docs(target_relation, model, for_relation=True) %} {%- endif -%} diff --git a/tests/functional/adapter/incremental/fixtures.py b/tests/functional/adapter/incremental/fixtures.py index 10b49a132..5c6d5f019 100644 --- a/tests/functional/adapter/incremental/fixtures.py +++ b/tests/functional/adapter/incremental/fixtures.py @@ -1109,3 +1109,64 @@ def model(dbt, spark): to_columns: [id] warn_unenforced: false """ + +target_of_multiple_fks_sql = """ +{{ config( + materialized='table', +) }} + +SELECT 'a' AS str_key; +""" + +source_of_multiple_fks_sql = """ +{{ config( + materialized='incremental', + unique_key=['fk_1_col'], + incremental_strategy='delete+insert', + on_schema_change='fail' +) }} + +SELECT + 'a' AS fk_1_col, + 'a' AS fk_2_col +""" + +multiple_fks_on_one_target_schema_yml = """ +version: 2 + +models: + - name: target_of_multiple_fks + config: + contract: + enforced: true + columns: + - name: str_key + data_type: string + constraints: + - type: not_null + - type: primary_key + name: pk_target_key + warn_unenforced: false + + - name: source_of_multiple_fks + config: + contract: + enforced: true + columns: + - name: fk_1_col + data_type: string + constraints: + - type: foreign_key + to: ref('target_of_multiple_fks') + to_columns: ["str_key"] + name: fk_1 + warn_unenforced: false + - name: fk_2_col + data_type: string + constraints: + - type: foreign_key + to: ref('target_of_multiple_fks') + to_columns: ["str_key"] + name: fk_2 + warn_unenforced: false +""" diff --git a/tests/functional/adapter/incremental/test_incremental_constraints.py b/tests/functional/adapter/incremental/test_incremental_constraints.py index fd9533fd9..862259290 100644 --- a/tests/functional/adapter/incremental/test_incremental_constraints.py +++ b/tests/functional/adapter/incremental/test_incremental_constraints.py @@ -328,3 +328,94 @@ def test_remove_foreign_key_constraint(self, project): util.run_dbt(["run"]) referential_constraints = project.run_sql(referential_constraint_sql, fetch="all") assert len(referential_constraints) == 0 + + +@pytest.mark.skip_profile("databricks_cluster") +class TestIncrementalMultipleFKsToSameTable: + """ + Test that multiple foreign keys to the same parent table persist correctly + after incremental runs. This succeeds for `use_materialization_v2` at the time + of writing, but there is a bug in the v1 materialization implementation. This + test is included to confirm this. + """ + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": {"use_materialization_v2": True}, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "target_of_multiple_fks.sql": fixtures.target_of_multiple_fks_sql, + "source_of_multiple_fks.sql": fixtures.source_of_multiple_fks_sql, + "schema.yml": fixtures.multiple_fks_on_one_target_schema_yml, + } + + def test_multiple_fks_to_same_table_persist_after_incremental(self, project): + expected_constraints = { + ("fk_1", "pk_target_key"), + ("fk_2", "pk_target_key"), + } + + # Initial run - create tables with constraints + util.run_dbt(["run"]) + + referential_constraints = project.run_sql(referential_constraint_sql, fetch="all") + + constraints = {(row[0], row[1]) for row in referential_constraints} + assert constraints == expected_constraints + + # Incremental run - this should NOT lose any foreign keys + util.run_dbt(["run"]) + + referential_constraints_after = project.run_sql(referential_constraint_sql, fetch="all") + + constraint_names_after = {(row[0], row[1]) for row in referential_constraints_after} + assert constraint_names_after == expected_constraints + + +@pytest.mark.skip_profile("databricks_cluster") +class TestIncrementalMultipleFKsToSameTableNoV2: + """ + Test that multiple foreign keys to the same parent table persist correctly + after incremental runs. This is a very specific test to reproduce a very + specific bug found when not using `use_materialization_v2`. + """ + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": {"use_materialization_v2": False}, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "target_of_multiple_fks.sql": fixtures.target_of_multiple_fks_sql, + "source_of_multiple_fks.sql": fixtures.source_of_multiple_fks_sql, + "schema.yml": fixtures.multiple_fks_on_one_target_schema_yml, + } + + def test_multiple_fks_to_same_table_persist_after_incremental(self, project): + expected_constraints = { + ("fk_1", "pk_target_key"), + ("fk_2", "pk_target_key"), + } + + # Initial run - create tables with constraints + util.run_dbt(["run"]) + + referential_constraints = project.run_sql(referential_constraint_sql, fetch="all") + + constraints = {(row[0], row[1]) for row in referential_constraints} + assert constraints == expected_constraints + + # Incremental run - this should NOT lose any foreign keys + util.run_dbt(["run"]) + + referential_constraints_after = project.run_sql(referential_constraint_sql, fetch="all") + + constraint_names_after = {(row[0], row[1]) for row in referential_constraints_after} + assert constraint_names_after == expected_constraints diff --git a/tests/unit/relation_configs/test_constraint.py b/tests/unit/relation_configs/test_constraint.py index ab7103936..ce5a6c648 100644 --- a/tests/unit/relation_configs/test_constraint.py +++ b/tests/unit/relation_configs/test_constraint.py @@ -352,3 +352,47 @@ def test_get_diff__check_constraints_different_formatting(self): ) diff = config.get_diff(other) assert diff is None + + def test_get_diff__multiple_fks_to_same_table(self): + """Test that multiple FKs to the same parent table are handled correctly.""" + config = ConstraintsConfig( + set_non_nulls=set(), + set_constraints={ + ForeignKeyConstraint( + type=ConstraintType.foreign_key, + name="fk_start_date", + columns=["start_date"], + to="`catalog`.`schema`.`date_dim`", + to_columns=["date_key"], + ), + ForeignKeyConstraint( + type=ConstraintType.foreign_key, + name="fk_end_date", + columns=["end_date"], + to="`catalog`.`schema`.`date_dim`", + to_columns=["date_key"], + ), + }, + ) + other = ConstraintsConfig( + set_non_nulls=set(), + set_constraints={ + ForeignKeyConstraint( + type=ConstraintType.foreign_key, + name="fk_start_date", + columns=["start_date"], + to="`catalog`.`schema`.`date_dim`", + to_columns=["date_key"], + ), + ForeignKeyConstraint( + type=ConstraintType.foreign_key, + name="fk_end_date", + columns=["end_date"], + to="`catalog`.`schema`.`date_dim`", + to_columns=["date_key"], + ), + }, + ) + # Both configs have same constraints, diff should be None + diff = config.get_diff(other) + assert diff is None From 1055cb0d48fca01b93882f1c3891c174d0ffe683 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Thu, 8 Jan 2026 10:21:36 +0000 Subject: [PATCH 02/16] update CHANGELOG.md Signed-off-by: Ewan Keith --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b20896c7..e8189e106 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Fixes - Fix `hard_deletes: invalidate` incorrectly invalidating active records in snapshots (thanks @Zurbste!) ([#1281](https://github.com/databricks/dbt-databricks/issues/1281)) +- Fix multiple foreign-keys from one table to another not being retained after incremental run ### Under the Hood From 41d7de0788cd324d2331b04cc9cadb222f6e0bc5 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Thu, 8 Jan 2026 21:49:28 +0000 Subject: [PATCH 03/16] revert irrelavent unit test --- .../unit/relation_configs/test_constraint.py | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/tests/unit/relation_configs/test_constraint.py b/tests/unit/relation_configs/test_constraint.py index ce5a6c648..ab7103936 100644 --- a/tests/unit/relation_configs/test_constraint.py +++ b/tests/unit/relation_configs/test_constraint.py @@ -352,47 +352,3 @@ def test_get_diff__check_constraints_different_formatting(self): ) diff = config.get_diff(other) assert diff is None - - def test_get_diff__multiple_fks_to_same_table(self): - """Test that multiple FKs to the same parent table are handled correctly.""" - config = ConstraintsConfig( - set_non_nulls=set(), - set_constraints={ - ForeignKeyConstraint( - type=ConstraintType.foreign_key, - name="fk_start_date", - columns=["start_date"], - to="`catalog`.`schema`.`date_dim`", - to_columns=["date_key"], - ), - ForeignKeyConstraint( - type=ConstraintType.foreign_key, - name="fk_end_date", - columns=["end_date"], - to="`catalog`.`schema`.`date_dim`", - to_columns=["date_key"], - ), - }, - ) - other = ConstraintsConfig( - set_non_nulls=set(), - set_constraints={ - ForeignKeyConstraint( - type=ConstraintType.foreign_key, - name="fk_start_date", - columns=["start_date"], - to="`catalog`.`schema`.`date_dim`", - to_columns=["date_key"], - ), - ForeignKeyConstraint( - type=ConstraintType.foreign_key, - name="fk_end_date", - columns=["end_date"], - to="`catalog`.`schema`.`date_dim`", - to_columns=["date_key"], - ), - }, - ) - # Both configs have same constraints, diff should be None - diff = config.get_diff(other) - assert diff is None From e03b4c2a5d1658f96cbcb427679b8a2271b67a0e Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Thu, 8 Jan 2026 21:56:13 +0000 Subject: [PATCH 04/16] ammend integration test to correctly identify inc-fk -> non-inc-pk as the actual case --- .../adapter/incremental/fixtures.py | 29 ++++++--------- .../test_incremental_constraints.py | 36 ++++++------------- 2 files changed, 20 insertions(+), 45 deletions(-) diff --git a/tests/functional/adapter/incremental/fixtures.py b/tests/functional/adapter/incremental/fixtures.py index 5c6d5f019..f0136db70 100644 --- a/tests/functional/adapter/incremental/fixtures.py +++ b/tests/functional/adapter/incremental/fixtures.py @@ -1110,7 +1110,7 @@ def model(dbt, spark): warn_unenforced: false """ -target_of_multiple_fks_sql = """ +non_incremental_target_of_fk = """ {{ config( materialized='table', ) }} @@ -1118,24 +1118,23 @@ def model(dbt, spark): SELECT 'a' AS str_key; """ -source_of_multiple_fks_sql = """ +incremental_fk_sql = """ {{ config( materialized='incremental', - unique_key=['fk_1_col'], + unique_key=['fk_col'], incremental_strategy='delete+insert', on_schema_change='fail' ) }} SELECT - 'a' AS fk_1_col, - 'a' AS fk_2_col + 'a' AS fk_col """ -multiple_fks_on_one_target_schema_yml = """ +incremental_fk_on_non_incremental_target_schema_yml = """ version: 2 models: - - name: target_of_multiple_fks + - name: non_incremental_target_of_fk config: contract: enforced: true @@ -1148,25 +1147,17 @@ def model(dbt, spark): name: pk_target_key warn_unenforced: false - - name: source_of_multiple_fks + - name: incremental_fk config: contract: enforced: true columns: - - name: fk_1_col + - name: fk_col data_type: string constraints: - type: foreign_key - to: ref('target_of_multiple_fks') + to: ref('non_incremental_target_of_fk') to_columns: ["str_key"] - name: fk_1 - warn_unenforced: false - - name: fk_2_col - data_type: string - constraints: - - type: foreign_key - to: ref('target_of_multiple_fks') - to_columns: ["str_key"] - name: fk_2 + name: fk warn_unenforced: false """ diff --git a/tests/functional/adapter/incremental/test_incremental_constraints.py b/tests/functional/adapter/incremental/test_incremental_constraints.py index 862259290..f76776d1a 100644 --- a/tests/functional/adapter/incremental/test_incremental_constraints.py +++ b/tests/functional/adapter/incremental/test_incremental_constraints.py @@ -330,15 +330,7 @@ def test_remove_foreign_key_constraint(self, project): assert len(referential_constraints) == 0 -@pytest.mark.skip_profile("databricks_cluster") -class TestIncrementalMultipleFKsToSameTable: - """ - Test that multiple foreign keys to the same parent table persist correctly - after incremental runs. This succeeds for `use_materialization_v2` at the time - of writing, but there is a bug in the v1 materialization implementation. This - test is included to confirm this. - """ - +class TestIncrementalFkTargetNonIncrementalIsRetained: @pytest.fixture(scope="class") def project_config_update(self): return { @@ -348,15 +340,14 @@ def project_config_update(self): @pytest.fixture(scope="class") def models(self): return { - "target_of_multiple_fks.sql": fixtures.target_of_multiple_fks_sql, - "source_of_multiple_fks.sql": fixtures.source_of_multiple_fks_sql, - "schema.yml": fixtures.multiple_fks_on_one_target_schema_yml, + "non_incremental_target_of_fk.sql": fixtures.non_incremental_target_of_fk, + "incremental_fk.sql": fixtures.incremental_fk_sql, + "schema.yml": fixtures.incremental_fk_on_non_incremental_target_schema_yml, } def test_multiple_fks_to_same_table_persist_after_incremental(self, project): expected_constraints = { - ("fk_1", "pk_target_key"), - ("fk_2", "pk_target_key"), + ("fk", "pk_target_key"), } # Initial run - create tables with constraints @@ -377,13 +368,7 @@ def test_multiple_fks_to_same_table_persist_after_incremental(self, project): @pytest.mark.skip_profile("databricks_cluster") -class TestIncrementalMultipleFKsToSameTableNoV2: - """ - Test that multiple foreign keys to the same parent table persist correctly - after incremental runs. This is a very specific test to reproduce a very - specific bug found when not using `use_materialization_v2`. - """ - +class TestIncrementalFkTargetNonIncrementalIsRetainedNoV2: @pytest.fixture(scope="class") def project_config_update(self): return { @@ -393,15 +378,14 @@ def project_config_update(self): @pytest.fixture(scope="class") def models(self): return { - "target_of_multiple_fks.sql": fixtures.target_of_multiple_fks_sql, - "source_of_multiple_fks.sql": fixtures.source_of_multiple_fks_sql, - "schema.yml": fixtures.multiple_fks_on_one_target_schema_yml, + "non_incremental_target_of_fk.sql": fixtures.non_incremental_target_of_fk, + "incremental_fk.sql": fixtures.incremental_fk_sql, + "schema.yml": fixtures.incremental_fk_on_non_incremental_target_schema_yml, } def test_multiple_fks_to_same_table_persist_after_incremental(self, project): expected_constraints = { - ("fk_1", "pk_target_key"), - ("fk_2", "pk_target_key"), + ("fk", "pk_target_key"), } # Initial run - create tables with constraints From 867102eb5de0d90e0b9e684a6ca9febb856909df Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Thu, 8 Jan 2026 21:57:21 +0000 Subject: [PATCH 05/16] amend changelog entry to reflect actual fix --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8189e106..dce4ca943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ ### Fixes - Fix `hard_deletes: invalidate` incorrectly invalidating active records in snapshots (thanks @Zurbste!) ([#1281](https://github.com/databricks/dbt-databricks/issues/1281)) -- Fix multiple foreign-keys from one table to another not being retained after incremental run +- Fix foreign-key on an incremental table to a primary key on a non-incremental table being lost after incremental run ### Under the Hood From 6bb1b56930eb92861b3b4fbff9969584384bd44c Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Fri, 9 Jan 2026 17:22:39 +0000 Subject: [PATCH 06/16] Skip test for Databricks cluster profile --- .../adapter/incremental/test_incremental_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/incremental/test_incremental_constraints.py b/tests/functional/adapter/incremental/test_incremental_constraints.py index f76776d1a..d8ec0e089 100644 --- a/tests/functional/adapter/incremental/test_incremental_constraints.py +++ b/tests/functional/adapter/incremental/test_incremental_constraints.py @@ -329,7 +329,7 @@ def test_remove_foreign_key_constraint(self, project): referential_constraints = project.run_sql(referential_constraint_sql, fetch="all") assert len(referential_constraints) == 0 - +@pytest.mark.skip_profile("databricks_cluster") class TestIncrementalFkTargetNonIncrementalIsRetained: @pytest.fixture(scope="class") def project_config_update(self): From f9a9fcb12ada350423cb2b58ce5ce4feb47cdcb0 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Mon, 12 Jan 2026 10:10:52 +0000 Subject: [PATCH 07/16] correctly construct FK sql when using expression to specify constraint --- dbt/adapters/databricks/constraints.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/databricks/constraints.py b/dbt/adapters/databricks/constraints.py index bf3b09f2e..9ffe14f06 100644 --- a/dbt/adapters/databricks/constraints.py +++ b/dbt/adapters/databricks/constraints.py @@ -119,11 +119,12 @@ def _validate(self) -> None: ) def _render_suffix(self) -> str: - suffix = f"FOREIGN KEY ({', '.join(self.columns)})" if self.expression: - suffix += f" {self.expression}" - else: - suffix += f" REFERENCES {self.to} ({', '.join(self.to_columns)})" + if self.expression.strip().startswith("("): + return f"FOREIGN KEY {self.expression}" + return f"FOREIGN KEY ({', '.join(self.columns)}) {self.expression}" + suffix = f"FOREIGN KEY ({', '.join(self.columns)})" + suffix += f" REFERENCES {self.to} ({', '.join(self.to_columns)})" return suffix From 90c9d97f3afaed5258faf9432917890837072913 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Mon, 12 Jan 2026 10:16:39 +0000 Subject: [PATCH 08/16] slightly simplify string logic --- dbt/adapters/databricks/constraints.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dbt/adapters/databricks/constraints.py b/dbt/adapters/databricks/constraints.py index 9ffe14f06..49314da34 100644 --- a/dbt/adapters/databricks/constraints.py +++ b/dbt/adapters/databricks/constraints.py @@ -123,9 +123,7 @@ def _render_suffix(self) -> str: if self.expression.strip().startswith("("): return f"FOREIGN KEY {self.expression}" return f"FOREIGN KEY ({', '.join(self.columns)}) {self.expression}" - suffix = f"FOREIGN KEY ({', '.join(self.columns)})" - suffix += f" REFERENCES {self.to} ({', '.join(self.to_columns)})" - return suffix + return f"FOREIGN KEY ({', '.join(self.columns)}) REFERENCES {self.to} ({', '.join(self.to_columns)})" class CheckConstraint(TypedConstraint): From 84584f450ff6f01e8c18627321e8481531653c86 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Thu, 8 Jan 2026 10:21:36 +0000 Subject: [PATCH 09/16] update CHANGELOG.md Signed-off-by: Ewan Keith --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1da78b306..9462c76b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Fix `hard_deletes: invalidate` incorrectly invalidating active records in snapshots (thanks @Zurbste!) ([#1281](https://github.com/databricks/dbt-databricks/issues/1281)) - Fix foreign-key on an incremental table to a primary key on a non-incremental table being lost after incremental run - Fix serverless Python model environment configuration: use `environment_version` instead of deprecated `client` field. Users can now specify custom environment versions via `python_job_config.environments`. ([#1286](https://github.com/databricks/dbt-databricks/pull/1286)) +- Fix multiple foreign-keys from one table to another not being retained after incremental run ### Under the Hood From 4a88022305f824f20a0ac3c1c7b201f9f11c27f2 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Thu, 8 Jan 2026 21:57:21 +0000 Subject: [PATCH 10/16] amend changelog entry to reflect actual fix --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9462c76b6..ec8cfa1e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Fix `hard_deletes: invalidate` incorrectly invalidating active records in snapshots (thanks @Zurbste!) ([#1281](https://github.com/databricks/dbt-databricks/issues/1281)) - Fix foreign-key on an incremental table to a primary key on a non-incremental table being lost after incremental run - Fix serverless Python model environment configuration: use `environment_version` instead of deprecated `client` field. Users can now specify custom environment versions via `python_job_config.environments`. ([#1286](https://github.com/databricks/dbt-databricks/pull/1286)) -- Fix multiple foreign-keys from one table to another not being retained after incremental run +- Fix foreign-key on an incremental table to a primary key on a non-incremental table being lost after incremental run ### Under the Hood From 6c0e22dfdf04328131d072b0b2968a1618d82389 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Mon, 12 Jan 2026 20:16:52 +0000 Subject: [PATCH 11/16] move changelog entry to 1.11.5 release --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec8cfa1e1..985eaa24d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## dbt-databricks 1.11.5 (TBD) + +### Fixes + +- Fix foreign-key on an incremental table to a primary key on a non-incremental table being lost after incremental run + ## dbt-databricks 1.11.4 (Jan 12, 2026) ### Features @@ -7,10 +13,7 @@ ### Fixes - Fix `hard_deletes: invalidate` incorrectly invalidating active records in snapshots (thanks @Zurbste!) ([#1281](https://github.com/databricks/dbt-databricks/issues/1281)) -- Fix foreign-key on an incremental table to a primary key on a non-incremental table being lost after incremental run - Fix serverless Python model environment configuration: use `environment_version` instead of deprecated `client` field. Users can now specify custom environment versions via `python_job_config.environments`. ([#1286](https://github.com/databricks/dbt-databricks/pull/1286)) -- Fix foreign-key on an incremental table to a primary key on a non-incremental table being lost after incremental run - ### Under the Hood From 4432f017a7ee6d615d30e6de25a099ea69a071a2 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Mon, 12 Jan 2026 20:59:06 +0000 Subject: [PATCH 12/16] dont incrementally apply constraints on hive metsatore as it is not supported --- .../macros/materializations/incremental/incremental.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/include/databricks/macros/materializations/incremental/incremental.sql b/dbt/include/databricks/macros/materializations/incremental/incremental.sql index b6bf6be93..77438674c 100644 --- a/dbt/include/databricks/macros/materializations/incremental/incremental.sql +++ b/dbt/include/databricks/macros/materializations/incremental/incremental.sql @@ -187,7 +187,8 @@ {% if liquid_clustering is not none %} {% do apply_liquid_clustered_cols(target_relation, liquid_clustering) %} {% endif %} - {% if constraints %} + {#- Incremental constraint application requires information_schema access (see fetch_*_constraints macros) -#} + {% if constraints and not target_relation.is_hive_metastore() %} {{ apply_constraints(target_relation, constraints) }} {% endif %} {%- endif -%} From 2230aef815c83b3818d420fffb85359ba815c755 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Thu, 8 Jan 2026 10:21:36 +0000 Subject: [PATCH 13/16] update CHANGELOG.md Signed-off-by: Ewan Keith --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 985eaa24d..b8b55ac43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Fix `hard_deletes: invalidate` incorrectly invalidating active records in snapshots (thanks @Zurbste!) ([#1281](https://github.com/databricks/dbt-databricks/issues/1281)) - Fix serverless Python model environment configuration: use `environment_version` instead of deprecated `client` field. Users can now specify custom environment versions via `python_job_config.environments`. ([#1286](https://github.com/databricks/dbt-databricks/pull/1286)) +- Fix multiple foreign-keys from one table to another not being retained after incremental run ### Under the Hood From 130541863a8bb4db4d2a4d7ffc0691decf304140 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Thu, 8 Jan 2026 21:57:21 +0000 Subject: [PATCH 14/16] amend changelog entry to reflect actual fix --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8b55ac43..84aed91ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ - Fix `hard_deletes: invalidate` incorrectly invalidating active records in snapshots (thanks @Zurbste!) ([#1281](https://github.com/databricks/dbt-databricks/issues/1281)) - Fix serverless Python model environment configuration: use `environment_version` instead of deprecated `client` field. Users can now specify custom environment versions via `python_job_config.environments`. ([#1286](https://github.com/databricks/dbt-databricks/pull/1286)) -- Fix multiple foreign-keys from one table to another not being retained after incremental run +- Fix foreign-key on an incremental table to a primary key on a non-incremental table being lost after incremental run ### Under the Hood From 54edbccc9fd58792d0191ce9745a19af38cf45d4 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Fri, 9 Jan 2026 17:19:45 +0000 Subject: [PATCH 15/16] add decorator to skip constraint test for databricks_cluster profile --- .../adapter/incremental/test_incremental_constraints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/adapter/incremental/test_incremental_constraints.py b/tests/functional/adapter/incremental/test_incremental_constraints.py index d8ec0e089..7b33cf58b 100644 --- a/tests/functional/adapter/incremental/test_incremental_constraints.py +++ b/tests/functional/adapter/incremental/test_incremental_constraints.py @@ -329,6 +329,7 @@ def test_remove_foreign_key_constraint(self, project): referential_constraints = project.run_sql(referential_constraint_sql, fetch="all") assert len(referential_constraints) == 0 + @pytest.mark.skip_profile("databricks_cluster") class TestIncrementalFkTargetNonIncrementalIsRetained: @pytest.fixture(scope="class") From b7918b77aa975ddd229b50aa3610ab4679473fc0 Mon Sep 17 00:00:00 2001 From: Ewan Keith Date: Wed, 14 Jan 2026 07:35:32 +0000 Subject: [PATCH 16/16] fix ruff fmt error --- dbt/adapters/databricks/constraints.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/databricks/constraints.py b/dbt/adapters/databricks/constraints.py index 49314da34..112fd77bf 100644 --- a/dbt/adapters/databricks/constraints.py +++ b/dbt/adapters/databricks/constraints.py @@ -123,7 +123,10 @@ def _render_suffix(self) -> str: if self.expression.strip().startswith("("): return f"FOREIGN KEY {self.expression}" return f"FOREIGN KEY ({', '.join(self.columns)}) {self.expression}" - return f"FOREIGN KEY ({', '.join(self.columns)}) REFERENCES {self.to} ({', '.join(self.to_columns)})" + return ( + f"FOREIGN KEY ({', '.join(self.columns)}) REFERENCES " + + f"{self.to} ({', '.join(self.to_columns)})" + ) class CheckConstraint(TypedConstraint):