Merge branch 'main' into 1.10.latest

benc-db · benc-db · commit 5f686f245df4 · 2025-08-20T09:00:47.000-07:00
diff --git a/.github/workflows/ci-pr-linting.yml b/.github/workflows/ci-pr-linting.yml
@@ -0,0 +1,68 @@
+name: Check PR title format
+description: Check if PR title follows conventional-commits format
+
+permissions:
+  pull-requests: write
+
+on:
+  pull_request:
+    types:
+      - opened      # A pull request was created
+      - edited      # The title or body of a pull request was edited.
+      - synchronize # A pull request's head branch was updated. For example, the head branch was updated from the base branch or new commits were pushed to the head branch.
+
+jobs:
+  pr-title:
+    runs-on: linux-ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Install conventional commit parser
+        shell: bash
+        run: npm install --global conventional-commits-parser
+
+      - name: Validate PR title
+        id: pr-format
+        shell: bash
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
+        # language=bash
+        run: |
+          echo "PR title: ${PR_TITLE}"
+                    
+          # check if PR title follows conventional commits format
+          # issue on parser does not support "!" for breaking change (https://github.com/conventional-changelog/conventional-changelog/issues/648)
+          # so we override the regex to support it
+          conventionalCommitResult=$(echo "${PR_TITLE}" | conventional-commits-parser -p "^(\w*)!?(?:\(([\w\$\.\-\* ]*)\))?\: (.*)$" | jq ".[].type")
+          if [[ "${conventionalCommitResult}" != "null" ]]; then
+            echo "Conventional commit type: ${conventionalCommitResult}"
+            exit 0
+          fi
+          
+          echo "Invalid PR title"
+          exit 1
+
+      - name: Add comment to warn user
+        uses: marocchino/sticky-pull-request-comment@efaaab3fd41a9c3de579aba759d2552635e590fd  # v2.8.0
+        if: failure()
+        with:
+          header: pr-title-lint-error
+          message: |
+            Hey there and thank you for opening this pull request! :wave:
+            We require pull request titles to follow the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/).
+            
+            Examples:
+            - `feat(JIRA-123): My awesome feature`
+            - `fix: My awesome fix`
+            - `fix(name-of-impacted-package): My awesome fix`
+
+      - name: Delete a previous comment when the issue has been resolved
+        uses: marocchino/sticky-pull-request-comment@efaaab3fd41a9c3de579aba759d2552635e590fd  # v2.8.0
+        if: success()
+        with:
+          header: pr-title-lint-error
+          delete: true
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,29 +1,49 @@
-## dbt-databricks 1.10.10 (TBD)
+## dbt-databricks 1.10.11 (TBD)
+
+## dbt-databricks 1.10.10 (August 20, 2025)
+
+### Fixes
+
+- Gate column comment syntax on DBR version for better compatibility ([1151](https://github.com/databricks/dbt-databricks/pull/1151))
+
+### Documentation
+
+- Update Databricks Job documentation to match current terminology ([1145](https://github.com/databricks/dbt-databricks/pull/1145))
 
 ## dbt-databricks 1.10.9 (August 7, 2025)
 
 ### Features
+
 - Support column tags for views using `ALTER TABLE`
 
 ### Under the hood
+
 - Revert `REPLACE USING` syntax being used for insert overwrite ([1025](https://github.com/databricks/dbt-databricks/issues/1025))
 
 ## dbt-databricks 1.10.8 (August 4, 2025)
 
 ### Features
+
 - Support insert_overwrite incremental strategy for SQL warehouses ([1025](https://github.com/databricks/dbt-databricks/issues/1025))
 
 ### Fixes
+
 - Add fallback logic for known error types for `DESCRIBE TABLE EXTENDED .. AS JSON` for better reliability ([1128](https://github.com/databricks/dbt-databricks/issues/1128))
 - Fix no-op logic for views that is causing some incremental materializations to be skipped ([1122](https://github.com/databricks/dbt-databricks/issues/1122))
 - Fix check constraints keep getting replaced [issue-1109](https://github.com/databricks/dbt-databricks/issues/1109)
 
 ### Under the Hood
+
 - Simplify connection management to align with base adapter. Connections are no longer cached per-thread
 
 ## dbt-databricks 1.10.7 (July 31, 2025)
 
+### Features
+
+- feat: add pr linting to enforce conventional commits [issue-1111](https://github.com/databricks/dbt-databricks/issues/1083)
+
 ### Fixes
+
 - Do not use `DESCRIBE TABLE EXTENDED .. AS JSON` for STs when DBR version < 17.1. Do not use at all for MVs (not yet supported)
 
 ## dbt-databricks 1.10.6 (July 30, 2025)
@@ -71,6 +91,7 @@
 - Fix column comments for streaming tables and materialized views ([1049](https://github.com/databricks/dbt-databricks/issues/1049))
 
 ### Under the Hood
+
 - Update to dbt-core 1.10.1
 - Update to dbt-common 1.24.0
 
diff --git a/dbt/adapters/databricks/__version__.py b/dbt/adapters/databricks/__version__.py
@@ -1 +1 @@
-version = "1.10.9"
+version = "1.10.10"
diff --git a/dbt/adapters/databricks/catalogs/_hive_metastore.py b/dbt/adapters/databricks/catalogs/_hive_metastore.py
@@ -12,7 +12,7 @@ class HiveMetastoreCatalogIntegration(CatalogIntegration):
 
     def __init__(self, config: CatalogIntegrationConfig) -> None:
         super().__init__(config)
-        self.file_format: str = config.file_format
+        self.file_format: Optional[str] = config.file_format
 
     @property
     def location_root(self) -> Optional[str]:
@@ -34,7 +34,9 @@ def build_relation(self, model: RelationConfig) -> DatabricksCatalogRelation:
         """
         return DatabricksCatalogRelation(
             catalog_type=self.catalog_type,
-            catalog_name=self.catalog_name,
+            catalog_name=self.catalog_name
+            if self.catalog_name != constants.DEFAULT_HIVE_METASTORE_CATALOG.catalog_name
+            else model.database,
             table_format=parse_model.table_format(model) or self.table_format,
             file_format=parse_model.file_format(model) or self.file_format,
             external_volume=parse_model.location_root(model) or self.external_volume,
diff --git a/dbt/adapters/databricks/catalogs/_unity.py b/dbt/adapters/databricks/catalogs/_unity.py
@@ -14,7 +14,7 @@ def __init__(self, config: CatalogIntegrationConfig) -> None:
         super().__init__(config)
         if location_root := config.adapter_properties.get("location_root"):
             self.external_volume: Optional[str] = location_root
-        self.file_format: str = config.file_format
+        self.file_format: Optional[str] = config.file_format
 
     @property
     def location_root(self) -> Optional[str]:
@@ -36,7 +36,9 @@ def build_relation(self, model: RelationConfig) -> DatabricksCatalogRelation:
         """
         return DatabricksCatalogRelation(
             catalog_type=self.catalog_type,
-            catalog_name=self.catalog_name,
+            catalog_name=self.catalog_name
+            if self.catalog_name != constants.DEFAULT_CATALOG.catalog_name
+            else model.database,
             table_format=parse_model.table_format(model) or self.table_format,
             file_format=parse_model.file_format(model) or self.file_format,
             external_volume=parse_model.location_root(model) or self.external_volume,
diff --git a/dbt/adapters/databricks/parse_model.py b/dbt/adapters/databricks/parse_model.py
@@ -1,6 +1,7 @@
 import posixpath
 from typing import Optional
 
+from dbt.adapters.catalogs import CATALOG_INTEGRATION_MODEL_CONFIG_NAME
 from dbt.adapters.contracts.relation import RelationConfig
 from dbt.adapters.databricks import constants
 
@@ -12,7 +13,7 @@ def catalog_name(model: RelationConfig) -> Optional[str]:
     Luckily, all catalog attribution in the legacy behavior is on the model.
     This means we can take a default catalog and rely on the model overrides to supply the rest.
     """
-    return _get(model, "catalog") or constants.DEFAULT_CATALOG.name
+    return _get(model, CATALOG_INTEGRATION_MODEL_CONFIG_NAME) or constants.DEFAULT_CATALOG.name
 
 
 def file_format(model: RelationConfig) -> Optional[str]:
diff --git a/dbt/include/databricks/macros/adapters/persist_docs.sql b/dbt/include/databricks/macros/adapters/persist_docs.sql
@@ -13,7 +13,22 @@
 {% endmacro %}
 
 {% macro comment_on_column_sql(column_path, escaped_comment) %}
-COMMENT ON COLUMN {{ column_path }} IS '{{ escaped_comment }}'
+  {%- if adapter.compare_dbr_version(16, 1) >= 0 -%}
+    COMMENT ON COLUMN {{ column_path }} IS '{{ escaped_comment }}'
+  {%- else -%}
+    {{ alter_table_change_column_comment_sql(column_path, escaped_comment) }}
+  {%- endif -%}
+{% endmacro %}
+
+{% macro alter_table_change_column_comment_sql(column_path, escaped_comment) %}
+  {%- set parts = column_path.split('.') -%}
+  {%- if parts|length >= 4 -%}
+    {%- set table_path = parts[:-1] | join('.') -%}
+    {%- set column_name = parts[-1] -%}
+    ALTER TABLE {{ table_path }} ALTER COLUMN {{ column_name }} COMMENT '{{ escaped_comment }}'
+  {%- else -%}
+    {{ exceptions.raise_compiler_error("Invalid column path: " ~ column_path ~ ". Expected format: database.schema.table.column") }}
+  {%- endif -%}
 {% endmacro %}
 
 {% macro databricks__persist_docs(relation, model, for_relation, for_columns) -%}
diff --git a/dbt/include/databricks/macros/get_custom_name/get_custom_database.sql b/dbt/include/databricks/macros/get_custom_name/get_custom_database.sql
@@ -4,10 +4,17 @@
 #}
 
 {% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}
-    {%- set default_database = target.database -%}
     {%- if custom_database_name is none -%}
-        {{ return(default_database) }}
+         {%- if node is not none -%}
+            {%- set catalog_relation = adapter.build_catalog_relation(node) -%}
+            {{ return(catalog_relation.catalog_name) }}
+        {%- elif 'config' in target -%}
+            {%- set catalog_relation = adapter.build_catalog_relation(target) -%}
+            {{ return(catalog_relation.catalog_name) }}
+        {%- else -%}
+            {{ return(target.database) }}
+        {%- endif -%}
     {%- else -%}
-        {{ return(custom_database_name) }}
+       {{ return(custom_database_name) }}
     {%- endif -%}
 {%- endmacro %}
diff --git a/docs/databricks-jobs.md b/docs/databricks-jobs.md
@@ -1,6 +1,6 @@
-# Running a dbt project as a job in Databricks Workflows
+# Running a dbt project as a job in Databricks Jobs
 
-Databricks Workflows is a highly-reliable, managed orchestrator that lets you author and schedule DAGs of notebooks, Python scripts as well as dbt projects as production jobs.
+Databricks Lakeflow Jobs is a highly-reliable, managed orchestrator that lets you author and schedule DAGs of notebooks, Python scripts as well as dbt projects as production jobs.
 
 In this guide, you will learn how to update an existing dbt project to run as a job, retrieve dbt run artifacts using the Jobs API and debug common issues.
 
@@ -18,7 +18,7 @@ When you run a dbt project as a Databricks Job, the dbt CLI runs on a single-nod
 - Install and configure the [Databricks CLI](https://docs.databricks.com/dev-tools/cli/index.html)
 - Install [jq](https://stedolan.github.io/jq/download/), a popular open source tool for parsing JSON from the command line
 
-Note: previously dbt tasks on Databricks Workflows could target jobs clusters for compute.
+Note: previously dbt tasks on Databricks Jobs could target jobs clusters for compute.
 That is [no longer supported](https://docs.databricks.com/en/workflows/jobs/how-to/use-dbt-in-workflows.html#advanced-run-dbt-with-a-custom-profile).
 Job clusters can only be used for running the dbt-cli.
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -22,12 +22,12 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "databricks-sdk>=0.41, <0.48.0",
-    "databricks-sql-connector[pyarrow]>=4.0.0, <5.0.0",
-    "dbt-adapters>=1.16.0, <2.0",
-    "dbt-common>=1.24.0, <2.0",
-    "dbt-core>=1.10.1, <2.0",
-    "dbt-spark>=1.9.0, <2.0",
+    "databricks-sdk>=0.41, <0.64.0",
+    "databricks-sql-connector[pyarrow]>=4.0.0, <4.0.6",
+    "dbt-adapters>=1.16.0, <1.16.5",
+    "dbt-common>=1.24.0, <1.29.0",
+    "dbt-core>=1.10.1, <1.10.10",
+    "dbt-spark>=1.9.0, <1.9.4",
     "keyring>=23.13.0",
     "pydantic>=1.10.0",
 ]
@@ -64,12 +64,12 @@ check-sdist = [
 
 [tool.hatch.envs.default]
 pre-install-commands = [
-    "pip install git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-adapters",
+    "pip install git+https://github.com/dbt-labs/dbt-common.git",
+    "pip install git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-adapters",
     "pip install git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter",
+    "pip install git+https://github.com/dbt-labs/dbt-core.git@main#subdirectory=core",
 ]
 dependencies = [
-    "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git",
-    "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git@main#subdirectory=core",
     "dbt-spark @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-spark",
     "pytest",
     "pytest-xdist",
diff --git a/tests/functional/adapter/catalog_integrations/test_catalog_integration.py b/tests/functional/adapter/catalog_integrations/test_catalog_integration.py
@@ -0,0 +1,105 @@
+import os
+
+import pytest
+
+from dbt.tests.adapter.catalog_integrations.test_catalog_integration import (
+    BaseCatalogIntegrationValidation,
+)
+from dbt.tests.util import run_dbt
+
+MODEL__BASIC_ICEBERG_TABLE = """
+{{ config(materialized='table', catalog_name='a-different-catalog') }}
+select 1 as id, 'test' as name
+"""
+
+MODEL__ICEBERG_TABLE_WITH_LOCATION = """
+{{ config(
+    materialized='table',
+    catalog_name='a-different-catalog',
+) }}
+select 2 as id, 'location_test' as name
+"""
+
+MODEL__REF_ICEBERG_TABLE = """
+{{ config(materialized='table') }}
+select * from {{ ref('basic_iceberg_table') }}
+where id = 1
+"""
+
+ALT_CATALOG_NAME = os.getenv("DBT_DATABRICKS_ALT_CATALOG")
+
+
+@pytest.mark.skip_profile("databricks_cluster")
+class TestUnityCatalogIntegration(BaseCatalogIntegrationValidation):
+    @pytest.fixture(scope="class")
+    def catalogs(self):
+        return {
+            "catalogs": [
+                {
+                    "name": "a-different-catalog",
+                    "active_write_integration": ALT_CATALOG_NAME,
+                    "write_integrations": [
+                        {
+                            "name": ALT_CATALOG_NAME,
+                            "catalog_type": "unity",
+                            "table_format": "iceberg",
+                            "file_format": "delta",
+                        }
+                    ],
+                },
+            ]
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "basic_iceberg_table.sql": MODEL__BASIC_ICEBERG_TABLE,
+            "iceberg_table_with_location.sql": MODEL__ICEBERG_TABLE_WITH_LOCATION,
+            "ref_iceberg_table.sql": MODEL__REF_ICEBERG_TABLE,
+        }
+
+    def test_unity_catalog_iceberg_integration(self, project):
+        """Test that Unity Catalog can create and reference Iceberg tables"""
+        # Run all models
+        run_results = run_dbt(["run", "--log-level", "debug"])
+
+        # Verify all models ran successfully
+        assert len(run_results) == 3  # type: ignore
+
+        # Get the expected catalog name from environment variable
+        expected_catalog = os.getenv("DBT_DATABRICKS_ALT_CATALOG")
+        assert expected_catalog, "DBT_DATABRICKS_ALT_CATALOG environment variable must be set"
+
+        # Verify tables were created in the expected catalog
+        expected_tables = [
+            "basic_iceberg_table",
+            "iceberg_table_with_location",
+        ]
+
+        for table_name in expected_tables:
+            # Query system information schema to verify table exists in expected catalog
+            result = project.run_sql(
+                "select table_catalog, table_type from `system`.`information_schema`.`tables` "
+                f"where table_catalog = '{expected_catalog}' "
+                f"and table_schema = '{project.test_schema}' "
+                f"and table_name = '{table_name}'",
+                fetch="one",
+            )
+
+            # Assert table exists in expected catalog
+            assert result is not None, f"Table {table_name} not found in catalog {expected_catalog}"
+            actual_catalog, table_type = result
+            assert (
+                actual_catalog == expected_catalog
+            ), f"Table {table_name} found in catalog {actual_catalog}, expected {expected_catalog}"
+
+            # For iceberg tables, verify they have the correct table type
+            if "iceberg" in table_name.lower() or table_name == "basic_iceberg_table":
+                assert table_type in [
+                    "MANAGED",
+                    "EXTERNAL",
+                ], f"Expected table type MANAGED or EXTERNAL for {table_name}, got {table_type}"
+
+        # Verify we can run again (idempotency)
+        run_results = run_dbt(["run"])
+        assert len(run_results) == 3  # type: ignore
diff --git a/tests/unit/macros/adapters/test_persist_docs_macros.py b/tests/unit/macros/adapters/test_persist_docs_macros.py
diff --git a/tests/unit/macros/relations/test_table_macros.py b/tests/unit/macros/relations/test_table_macros.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-version = "1.10.9"`
	`1`	`+version = "1.10.10"`