Catalog Integration Config Standardization (#1134)

colin-rogers-dbt · web-flow · commit 2cfb72caa9cb · 2025-06-03T10:56:28.000-07:00
diff --git a/dbt-adapters/.changes/unreleased/Features-20250530-153938.yaml b/dbt-adapters/.changes/unreleased/Features-20250530-153938.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Add constant defining name of logical catalog in model config
+time: 2025-05-30T15:39:38.251541-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "1134"
diff --git a/dbt-adapters/.changes/unreleased/Under the Hood-20250529-114448.yaml b/dbt-adapters/.changes/unreleased/Under the Hood-20250529-114448.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: populate file_format from top level integration config field
+time: 2025-05-29T11:44:48.898354-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "1129"
diff --git a/dbt-adapters/src/dbt/adapters/base/impl.py b/dbt-adapters/src/dbt/adapters/base/impl.py
@@ -107,7 +107,6 @@
 if TYPE_CHECKING:
     import agate
 
-
 GET_CATALOG_MACRO_NAME = "get_catalog"
 GET_CATALOG_RELATIONS_MACRO_NAME = "get_catalog_relations"
 FRESHNESS_MACRO_NAME = "collect_freshness"
diff --git a/dbt-adapters/src/dbt/adapters/catalogs/__init__.py b/dbt-adapters/src/dbt/adapters/catalogs/__init__.py
@@ -9,3 +9,5 @@
     CatalogIntegrationConfig,
     CatalogRelation,
 )
+
+from dbt.adapters.catalogs._constants import CATALOG_INTEGRATION_MODEL_CONFIG_NAME
diff --git a/dbt-adapters/src/dbt/adapters/catalogs/_constants.py b/dbt-adapters/src/dbt/adapters/catalogs/_constants.py
@@ -0,0 +1 @@
+CATALOG_INTEGRATION_MODEL_CONFIG_NAME = "catalog_name"
diff --git a/dbt-adapters/src/dbt/adapters/catalogs/_integration.py b/dbt-adapters/src/dbt/adapters/catalogs/_integration.py
@@ -40,13 +40,15 @@ class CatalogIntegrationConfig(Protocol):
     catalog_name: Optional[str]
     table_format: Optional[str]
     external_volume: Optional[str]
+    file_format: Optional[str]
     adapter_properties: Dict[str, Any]
 
 
 class CatalogRelation(Protocol):
     catalog_name: Optional[str]
     table_format: Optional[str]
     external_volume: Optional[str]
+    file_format: Optional[str]
 
 
 class CatalogIntegration(abc.ABC):
@@ -78,16 +80,17 @@ class CatalogIntegration(abc.ABC):
 
     catalog_type: str
     table_format: Optional[str] = None
+    file_format: Optional[str] = None
     allows_writes: bool = False
 
     def __init__(self, config: CatalogIntegrationConfig) -> None:
         # table_format is often fixed for a catalog type, allow it to be defined at the class level
         if config.table_format is not None:
             self.table_format = config.table_format
-
         self.name: str = config.name
         self.catalog_name: Optional[str] = config.catalog_name
         self.external_volume: Optional[str] = config.external_volume
+        self.file_format: Optional[str] = config.file_format
 
     def build_relation(self, config: RelationConfig) -> CatalogRelation:
         """
diff --git a/dbt-adapters/tests/unit/test_catalogs.py b/dbt-adapters/tests/unit/test_catalogs.py
@@ -17,12 +17,13 @@
 
 
 @dataclass
-class FakeCatalogIntegrationConfig:
+class FakeCatalogIntegrationConfig(CatalogIntegrationConfig):
     name: str
     catalog_type: str
     catalog_name: Optional[str] = None
     table_format: Optional[str] = None
     external_volume: Optional[str] = None
+    file_format: Optional[str] = None
     adapter_properties: Optional[Dict[str, Any]] = None
 
 
diff --git a/dbt-bigquery/.changes/unreleased/Breaking Changes-20250530-154438.yaml b/dbt-bigquery/.changes/unreleased/Breaking Changes-20250530-154438.yaml
@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: rename 'catalog' model config value to 'catalog_name'
+time: 2025-05-30T15:44:38.10309-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "1134"
diff --git a/dbt-bigquery/src/dbt/adapters/bigquery/catalogs/_generic.py b/dbt-bigquery/src/dbt/adapters/bigquery/catalogs/_generic.py
@@ -1,10 +1,7 @@
 from dataclasses import dataclass
 from typing import Optional
 
-from dbt.adapters.catalogs import (
-    CatalogIntegration,
-    CatalogIntegrationConfig,
-)
+from dbt.adapters.catalogs import CatalogIntegration
 from dbt.adapters.contracts.relation import RelationConfig
 
 from dbt.adapters.bigquery import constants, parse_model
@@ -31,12 +28,6 @@ class BigQueryCatalogIntegration(CatalogIntegration):
     catalog_type = constants.GENERIC_CATALOG_TYPE
     allows_writes = True
 
-    def __init__(self, config: CatalogIntegrationConfig) -> None:
-        super().__init__(config)
-        self.file_format: Optional[str] = config.adapter_properties.get(
-            "file_format", constants.INFO_SCHEMA_FILE_FORMAT
-        )
-
     @property
     def storage_uri(self) -> Optional[str]:
         return self.external_volume
diff --git a/dbt-bigquery/src/dbt/adapters/bigquery/constants.py b/dbt-bigquery/src/dbt/adapters/bigquery/constants.py
@@ -21,17 +21,13 @@
     catalog_type=GENERIC_CATALOG_TYPE,
     table_format=INFO_SCHEMA_TABLE_FORMAT,
     external_volume=None,
-    adapter_properties={
-        "file_format": INFO_SCHEMA_FILE_FORMAT,
-    },
+    file_format=INFO_SCHEMA_FILE_FORMAT,
 )
 DEFAULT_ICEBERG_CATALOG = SimpleNamespace(
     name="managed_iceberg",
     catalog_name="managed_iceberg",
     catalog_type=GENERIC_CATALOG_TYPE,
     table_format=ICEBERG_TABLE_FORMAT,
     external_volume=None,
-    adapter_properties={
-        "file_format": PARQUET_FILE_FORMAT,
-    },
+    file_format=PARQUET_FILE_FORMAT,
 )
diff --git a/dbt-bigquery/src/dbt/adapters/bigquery/parse_model.py b/dbt-bigquery/src/dbt/adapters/bigquery/parse_model.py
@@ -2,6 +2,7 @@
 
 from dbt.adapters.contracts.relation import RelationConfig
 
+from dbt.adapters.catalogs import CATALOG_INTEGRATION_MODEL_CONFIG_NAME
 from dbt.adapters.bigquery import constants
 
 
@@ -10,6 +11,10 @@ def catalog_name(model: RelationConfig) -> Optional[str]:
     if not hasattr(model, "config") or not model.config:
         return None
 
+    if _catalog := model.config.get(CATALOG_INTEGRATION_MODEL_CONFIG_NAME):
+        return _catalog
+
+    # TODO: deprecate this as it's been replaced with catalog_name
     if _catalog := model.config.get("catalog"):
         return _catalog
 
diff --git a/dbt-bigquery/tests/functional/adapter/catalog_integrations/__init__.py b/dbt-bigquery/tests/functional/adapter/catalog_integrations/__init__.py
diff --git a/dbt-bigquery/tests/functional/adapter/catalog_integrations/test_generic_catalog.py b/dbt-bigquery/tests/functional/adapter/catalog_integrations/test_generic_catalog.py
@@ -31,7 +31,7 @@ def prefix():
     """
 {{ config(
     materialized='table',
-    catalog='managed_iceberg',
+    catalog_name='managed_iceberg',
     storage_uri='gs://"""
     + os.getenv("BIGQUERY_TEST_ICEBERG_BUCKET")
     + """/"""
@@ -46,7 +46,7 @@ def prefix():
 {{ config(
     materialized="incremental",
     unique_key="id",
-    catalog='managed_iceberg',
+    catalog_name='managed_iceberg',
     storage_uri='gs://"""
     + os.getenv("BIGQUERY_TEST_ICEBERG_BUCKET")
     + """/"""
diff --git a/dbt-snowflake/.changes/unreleased/Features-20250530-153848.yaml b/dbt-snowflake/.changes/unreleased/Features-20250530-153848.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Use catalog_name as signifier for logical catalog
+time: 2025-05-30T15:38:48.283619-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "1134"
diff --git a/dbt-snowflake/.changes/unreleased/Under the Hood-20250529-092759.yaml b/dbt-snowflake/.changes/unreleased/Under the Hood-20250529-092759.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: populate file_format from top level integration config field
+time: 2025-05-29T09:27:59.155182-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "1134"
diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/catalogs/_built_in.py b/dbt-snowflake/src/dbt/adapters/snowflake/catalogs/_built_in.py
@@ -17,6 +17,7 @@ class BuiltInCatalogRelation:
     catalog_name: Optional[str] = constants.DEFAULT_BUILT_IN_CATALOG.name
     table_format: Optional[str] = constants.ICEBERG_TABLE_FORMAT
     external_volume: Optional[str] = None
+    file_format: Optional[str] = None
     cluster_by: Optional[str] = None
     automatic_clustering: Optional[bool] = False
     is_transient: Optional[bool] = False
@@ -26,6 +27,7 @@ class BuiltInCatalogIntegration(CatalogIntegration):
     catalog_name = constants.DEFAULT_BUILT_IN_CATALOG.name
     catalog_type = constants.DEFAULT_BUILT_IN_CATALOG.catalog_type
     table_format = constants.ICEBERG_TABLE_FORMAT
+    file_format = None  # no file format for built-in catalogs
     allows_writes = True
 
     def __init__(self, config: CatalogIntegrationConfig) -> None:
diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/catalogs/_info_schema.py b/dbt-snowflake/src/dbt/adapters/snowflake/catalogs/_info_schema.py
@@ -16,6 +16,7 @@ class InfoSchemaCatalogRelation:
     catalog_name: Optional[str] = constants.DEFAULT_INFO_SCHEMA_CATALOG.name
     table_format: Optional[str] = constants.INFO_SCHEMA_TABLE_FORMAT
     external_volume: Optional[str] = None
+    file_format: Optional[str] = None
     cluster_by: Optional[str] = None
     automatic_clustering: Optional[bool] = False
     is_transient: Optional[bool] = False
diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/constants.py b/dbt-snowflake/src/dbt/adapters/snowflake/constants.py
@@ -15,6 +15,7 @@
     name="INFO_SCHEMA",  # these don't show up in Snowflake; this is a dbt convention
     catalog_type="INFO_SCHEMA",  # these don't show up in Snowflake; this is a dbt convention
     external_volume=None,
+    file_format=None,
     adapter_properties={},
 )
 # catalog names must be uppercase since Snowflake will uppercase them in their metadata tables
@@ -23,5 +24,6 @@
     catalog_type="BUILT_IN",
     # assume you are using the default volume or specify in the model
     external_volume=None,
+    file_format=None,
     adapter_properties={},
 )
diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/parse_model.py b/dbt-snowflake/src/dbt/adapters/snowflake/parse_model.py
@@ -1,6 +1,8 @@
 from typing import Iterable, Optional
 
 from dbt_common.exceptions import DbtConfigError
+
+from dbt.adapters.catalogs import CATALOG_INTEGRATION_MODEL_CONFIG_NAME
 from dbt.adapters.contracts.relation import RelationConfig
 
 from dbt.adapters.snowflake import constants
@@ -28,7 +30,7 @@ def catalog_name(model: RelationConfig) -> Optional[str]:
     if not model.config:
         return None
 
-    if _catalog := model.config.get("catalog"):
+    if _catalog := model.config.get(CATALOG_INTEGRATION_MODEL_CONFIG_NAME):
         # make catalog_name case-insensitive
         return _catalog.upper()
 
diff --git a/dbt-snowflake/tests/functional/adapter/catalog_integrations/test_catalog_integrations.py b/dbt-snowflake/tests/functional/adapter/catalog_integrations/test_catalog_integrations.py
@@ -0,0 +1,45 @@
+import os
+from datetime import datetime as dt
+import pytest
+from dbt.tests.adapter.catalog_integrations.test_catalog_integration import (
+    BaseCatalogIntegrationValidation,
+)
+from dbt.tests.util import run_dbt
+
+MODEL__BASIC_ICEBERG_TABLE = """
+                            {{ config(materialized='table', catalog_name='basic_iceberg_catalog') }}
+                            select 1 as id
+                            """
+
+
+class TestSnowflakeBuiltInCatalogIntegration(BaseCatalogIntegrationValidation):
+
+    @pytest.fixture(scope="class")
+    def catalogs(self):
+        return {
+            "catalogs": [
+                {
+                    "name": "basic_iceberg_catalog",
+                    "active_write_integration": "basic_iceberg_catalog_integration",
+                    "write_integrations": [
+                        {
+                            "name": "basic_iceberg_catalog_integration",
+                            "catalog_type": "BUILT_IN",
+                            "table_format": "iceberg",
+                            "external_volume": f"s3_iceberg_snow",
+                        }
+                    ],
+                },
+            ]
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "models": {
+                "basic_iceberg_table.sql": MODEL__BASIC_ICEBERG_TABLE,
+            }
+        }
+
+    def test_basic_iceberg_catalog_integration(self, project):
+        run_dbt(["run"])
diff --git a/dbt-tests-adapter/.changes/unreleased/Features-20250530-155031.yaml b/dbt-tests-adapter/.changes/unreleased/Features-20250530-155031.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Add BaseCatalogIntegrationValidation to support catalog integration testing
+time: 2025-05-30T15:50:31.521198-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "1134"
diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/catalog_integrations/test_catalog_integration.py b/dbt-tests-adapter/src/dbt/tests/adapter/catalog_integrations/test_catalog_integration.py
@@ -0,0 +1,12 @@
+import pytest
+from dbt.tests.util import write_config_file
+
+
+class BaseCatalogIntegrationValidation:
+    @pytest.fixture
+    def catalogs(self):
+        return {}
+
+    @pytest.fixture(scope="class", autouse=True)
+    def write_catalogs_config_file(self, project, catalogs):
+        write_config_file(catalogs, project.project_root, "catalogs.yml")

Original file line number	Diff line number	Diff line change
`@@ -9,3 +9,5 @@`
`9`	`9`	`CatalogIntegrationConfig,`
`10`	`10`	`CatalogRelation,`
`11`	`11`	`)`
	`12`	`+`
	`13`	`+from dbt.adapters.catalogs._constants import CATALOG_INTEGRATION_MODEL_CONFIG_NAME`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+CATALOG_INTEGRATION_MODEL_CONFIG_NAME = "catalog_name"`