refactor: Improve python model tests success rate (#1233)

### Description

So I think the repeated failure of python model tests in our CI/CD is
because we're trying to launch workflows on compute that already has an
overwhelmed driver. For SQL, I think extra statements get queued by sql
gateway, but I think for python models we eventually just time out
saying the cluster couldn't be reached. Also taking this example to
backport AGENTS.md because it's really annoying when working on 1.10.x
stuff to not have the latest changes.

### Checklist

- [ ] I have run this code in development and it appears to resolve the
stated issue
- [ ] This PR includes tests, or tests are not required/relevant for
this PR
- [ ] I have updated the `CHANGELOG.md` and added information about my
change to the "dbt-databricks next" section.

Author: benc-db

.pre-commit-config.yaml

@@ -1,15 +1,15 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.8.0
+    rev: v0.14.2
     hooks:
       # Run the linter.
       - id: ruff
       # Run the formatter.
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.13.0
+    rev: v1.18.2
     hooks:
       - id: mypy
         additional_dependencies: [types-requests]

AGENTS.md

@@ -20,6 +20,7 @@ dbt/adapters/databricks/
 ├── connections.py             # Connection management and SQL execution
 ├── credentials.py             # Authentication (token, OAuth, Azure AD)
 ├── relation.py               # Databricks-specific relation handling
+├── dbr_capabilities.py       # DBR version capability system
 ├── python_models/            # Python model execution on clusters
 ├── relation_configs/         # Table/view configuration management
 └── catalogs/                 # Unity Catalog vs Hive Metastore logic
@@ -37,20 +38,33 @@ dbt/include/databricks/macros/ # Jinja2 SQL templates
 
 **Install Hatch** (recommended):
 
+For Linux:
+
 ```bash
-# Install Hatch globally - see https://hatch.pypa.io/dev/install/
-pip install hatch
+# Download and install standalone binary
+curl -Lo hatch.tar.gz https://github.com/pypa/hatch/releases/latest/download/hatch-x86_64-unknown-linux-gnu.tar.gz
+tar -xzf hatch.tar.gz
+mkdir -p $HOME/bin
+mv hatch $HOME/bin/hatch
+chmod +x $HOME/bin/hatch
+echo 'export PATH="$HOME/bin:$PATH"' >> ~/.zshrc
+export PATH="$HOME/bin:$PATH"
 
 # Create default environment (Hatch installs needed Python versions)
 hatch env create
 ```
 
+For other platforms: see https://hatch.pypa.io/latest/install/
+
 **Essential commands**:
 
 ```bash
 hatch run code-quality           # Format, lint, type-check
 hatch run unit                   # Run unit tests
 hatch run cluster-e2e            # Run functional tests
+
+# For specific tests, use pytest directly:
+hatch run pytest path/to/test_file.py::TestClass::test_method -v
 ```
 
 > 📖 **See [Development Guide](docs/dbt-databricks-dev.md)** for comprehensive setup documentation
@@ -113,17 +127,38 @@ class TestCreateTable(MacroTestBase):
 
 #### Functional Test Example
 
+**Important**: SQL models and YAML schemas should be defined in a `fixtures.py` file in the same directory as the test, not inline in the test class. This keeps tests clean and fixtures reusable.
+
+**fixtures.py:**
+
+```python
+my_model_sql = """
+{{ config(materialized='incremental', unique_key='id') }}
+select 1 as id, 'test' as name
+"""
+
+my_schema_yml = """
+version: 2
+models:
+  - name: my_model
+    columns:
+      - name: id
+        description: 'ID column'
+"""
+```
+
+**test_my_feature.py:**
+
 ```python
 from dbt.tests import util
+from tests.functional.adapter.my_feature import fixtures
 
 class TestIncrementalModel:
     @pytest.fixture(scope="class")
     def models(self):
         return {
-            "my_model.sql": """
-                {{ config(materialized='incremental', unique_key='id') }}
-                select 1 as id, 'test' as name
-            """
+            "my_model.sql": fixtures.my_model_sql,
+            "schema.yml": fixtures.my_schema_yml,
         }
 
     def test_incremental_run(self, project):
@@ -147,6 +182,46 @@ DatabricksAdapter (impl.py)
 
 ### Key Components
 
+#### DBR Capability System (`dbr_capabilities.py`)
+
+- **Purpose**: Centralized management of DBR version-dependent features
+- **Key Features**:
+  - Per-compute caching (different clusters can have different capabilities)
+  - Named capabilities instead of magic version numbers
+  - Automatic detection of DBR version and SQL warehouse environments
+- **Supported Capabilities**:
+  - `TIMESTAMPDIFF` (DBR 10.4+): Advanced date/time functions
+  - `INSERT_BY_NAME` (DBR 12.2+): Name-based column matching in INSERT
+  - `ICEBERG` (DBR 14.3+): Apache Iceberg table format
+  - `COMMENT_ON_COLUMN` (DBR 16.1+): Modern column comment syntax
+  - `JSON_COLUMN_METADATA` (DBR 16.2+): Efficient metadata retrieval
+- **Usage in Code**:
+
+  ```python
+  # In Python code
+  if adapter.has_capability(DBRCapability.ICEBERG):
+      # Use Iceberg features
+
+  # In Jinja macros
+  {% if adapter.has_dbr_capability('comment_on_column') %}
+      COMMENT ON COLUMN ...
+  {% else %}
+      ALTER TABLE ... ALTER COLUMN ...
+  {% endif %}
+
+  {% if adapter.has_dbr_capability('insert_by_name') %}
+      INSERT INTO table BY NAME SELECT ...
+  {% else %}
+      INSERT INTO table SELECT ... -- positional
+  {% endif %}
+  ```
+
+- **Adding New Capabilities**:
+  1. Add to `DBRCapability` enum
+  2. Add `CapabilitySpec` with version requirements
+  3. Use `has_capability()` or `require_capability()` in code
+- **Important**: Each compute resource (identified by `http_path`) maintains its own capability cache
+
 #### Connection Management (`connections.py`)
 
 - Extends Spark connection manager for Databricks
@@ -184,6 +259,7 @@ DatabricksAdapter (impl.py)
 - Override Spark macros with Databricks-specific logic
 - Handle materializations (table, view, incremental, snapshot)
 - Implement Databricks features (liquid clustering, column masks, tags)
+- **Important**: To override a `spark__macro_name` macro, create `databricks__macro_name` (NOT `spark__macro_name`)
 
 ### Configuration System
 
@@ -256,6 +332,7 @@ Models can be configured with Databricks-specific options:
 
 - **Development**: `docs/dbt-databricks-dev.md` - Setup and workflow
 - **Testing**: `docs/testing.md` - Comprehensive testing guide
+- **DBR Capabilities**: `docs/dbr-capability-system.md` - Version-dependent features
 - **Contributing**: `CONTRIBUTING.MD` - Code standards and PR process
 - **User Docs**: [docs.getdbt.com](https://docs.getdbt.com/reference/resource-configs/databricks-configs)
 
@@ -273,6 +350,11 @@ Models can be configured with Databricks-specific options:
 3. **SQL Generation**: Prefer macros over Python string manipulation
 4. **Testing**: Write both unit and functional tests for new features
 5. **Configuration**: Use dataclasses with validation for new config options
+6. **Imports**: Always import at the top of the file, never use local imports within functions or methods
+7. **Version Checks**: Use capability system instead of direct version comparisons:
+   - ❌ `if adapter.compare_dbr_version(16, 1) >= 0:`
+   - ✅ `if adapter.has_capability(DBRCapability.COMMENT_ON_COLUMN):`
+   - ✅ `{% if adapter.has_dbr_capability('comment_on_column') %}`
 
 ## 🚨 Common Pitfalls for Agents
 
@@ -284,12 +366,20 @@ Models can be configured with Databricks-specific options:
 6. **Follow SQL normalization** in test assertions with `assert_sql_equal()`
 7. **Handle Unity Catalog vs HMS differences** in feature implementations
 8. **Consider backward compatibility** when modifying existing behavior
+9. **Use capability system for version checks** - Never add new `compare_dbr_version()` calls
+10. **Remember per-compute caching** - Different clusters may have different capabilities in the same run
 
 ## 🎯 Success Metrics
 
 When working on this codebase, ensure:
 
 - [ ] All tests pass (`hatch run code-quality && hatch run unit`)
+- [ ] **CRITICAL: Run affected functional tests before declaring success**
+  - If you modified connection/capability logic: Run tests that use multiple computes or check capabilities
+  - If you modified incremental materializations: Run `tests/functional/adapter/incremental/`
+  - If you modified Python models: Run `tests/functional/adapter/python_model/`
+  - If you modified macros: Run tests that use those macros
+  - **NEVER declare "mission accomplished" without running functional tests for affected features**
 - [ ] New features have both unit and functional tests
 - [ ] SQL generation follows Databricks best practices
 - [ ] Changes maintain backward compatibility

dbt/adapters/databricks/__init__.py

@@ -1,4 +1,5 @@
 from dbt.adapters.base import AdapterPlugin
+
 from dbt.adapters.databricks.credentials import DatabricksCredentials
 from dbt.adapters.databricks.impl import DatabricksAdapter
 from dbt.include import databricks

dbt/adapters/databricks/behaviors/columns.py

@@ -1,12 +1,12 @@
 from abc import ABC, abstractmethod
 
+from dbt.adapters.sql import SQLAdapter
 from dbt_common.exceptions import DbtDatabaseError
 from dbt_common.utils.dict import AttrDict
 
 from dbt.adapters.databricks.column import DatabricksColumn
 from dbt.adapters.databricks.relation import DatabricksRelation
 from dbt.adapters.databricks.utils import handle_missing_objects
-from dbt.adapters.sql import SQLAdapter
 
 
 class GetColumnsBehavior(ABC):

dbt/adapters/databricks/catalogs/_hive_metastore.py

@@ -2,6 +2,7 @@
 
 from dbt.adapters.catalogs import CatalogIntegration, CatalogIntegrationConfig
 from dbt.adapters.contracts.relation import RelationConfig
+
 from dbt.adapters.databricks import constants, parse_model
 from dbt.adapters.databricks.catalogs._relation import DatabricksCatalogRelation
 

dbt/adapters/databricks/catalogs/_unity.py

@@ -2,6 +2,7 @@
 
 from dbt.adapters.catalogs import CatalogIntegration, CatalogIntegrationConfig
 from dbt.adapters.contracts.relation import RelationConfig
+
 from dbt.adapters.databricks import constants, parse_model
 from dbt.adapters.databricks.catalogs._relation import DatabricksCatalogRelation
 

dbt/adapters/databricks/column.py

@@ -1,9 +1,10 @@
 from dataclasses import dataclass
 from typing import Any, ClassVar, Optional
 
-from dbt.adapters.databricks.utils import quote
 from dbt.adapters.spark.column import SparkColumn
 
+from dbt.adapters.databricks.utils import quote
+
 
 @dataclass
 class DatabricksColumn(SparkColumn):

dbt/adapters/databricks/connections.py

@@ -5,13 +5,6 @@
 from multiprocessing.context import SpawnContext
 from typing import TYPE_CHECKING, Any, Optional, cast
 
-from dbt_common.events.contextvars import get_node_info
-from dbt_common.events.functions import fire_event
-from dbt_common.exceptions import DbtDatabaseError, DbtRuntimeError
-from dbt_common.utils import cast_to_str
-
-from databricks.sql import __version__ as dbsql_version
-from databricks.sql.exc import Error
 from dbt.adapters.base.query_headers import MacroQueryStringSetter
 from dbt.adapters.contracts.connection import (
     DEFAULT_QUERY_COMMENT,
@@ -22,6 +15,22 @@
     Identifier,
     LazyHandle,
 )
+from dbt.adapters.events.types import (
+    ConnectionClosedInCleanup,
+    ConnectionReused,
+    ConnectionUsed,
+    NewConnection,
+    SQLQuery,
+    SQLQueryStatus,
+)
+from dbt.adapters.spark.connections import SparkConnectionManager
+from dbt_common.events.contextvars import get_node_info
+from dbt_common.events.functions import fire_event
+from dbt_common.exceptions import DbtDatabaseError, DbtRuntimeError
+from dbt_common.utils import cast_to_str
+
+from databricks.sql import __version__ as dbsql_version
+from databricks.sql.exc import Error
 from dbt.adapters.databricks.__version__ import version as __version__
 from dbt.adapters.databricks.api_client import DatabricksApiClient
 from dbt.adapters.databricks.credentials import (
@@ -37,15 +46,6 @@
 from dbt.adapters.databricks.logging import logger
 from dbt.adapters.databricks.python_models.run_tracking import PythonRunTracker
 from dbt.adapters.databricks.utils import is_cluster_http_path, redact_credentials
-from dbt.adapters.events.types import (
-    ConnectionClosedInCleanup,
-    ConnectionReused,
-    ConnectionUsed,
-    NewConnection,
-    SQLQuery,
-    SQLQueryStatus,
-)
-from dbt.adapters.spark.connections import SparkConnectionManager
 
 if TYPE_CHECKING:
     from agate import Table

dbt/adapters/databricks/constraints.py

@@ -3,6 +3,8 @@
 from typing import Any, ClassVar, Optional, TypeVar
 from uuid import uuid4
 
+from dbt.adapters.base import ConstraintSupport
+from dbt.adapters.events.types import ConstraintNotEnforced, ConstraintNotSupported
 from dbt_common.contracts.constraints import (
     ColumnLevelConstraint,
     ConstraintType,
@@ -11,9 +13,6 @@
 from dbt_common.events.functions import warn_or_error
 from dbt_common.exceptions import DbtValidationError
 
-from dbt.adapters.base import ConstraintSupport
-from dbt.adapters.events.types import ConstraintNotEnforced, ConstraintNotSupported
-
 # Support constants
 CONSTRAINT_SUPPORT = {
     ConstraintType.check: ConstraintSupport.ENFORCED,

dbt/adapters/databricks/credentials.py

@@ -5,14 +5,14 @@
 from dataclasses import dataclass, field
 from typing import Any, Callable, Optional, cast
 
+from dbt.adapters.contracts.connection import Credentials
 from dbt_common.exceptions import DbtConfigError, DbtValidationError
 from mashumaro import DataClassDictMixin
 from requests import PreparedRequest
 from requests.auth import AuthBase
 
 from databricks.sdk import WorkspaceClient
 from databricks.sdk.core import Config, CredentialsProvider
-from dbt.adapters.contracts.connection import Credentials
 from dbt.adapters.databricks.global_state import GlobalState
 from dbt.adapters.databricks.logging import logger