Add Enterprise Catalog tests (#307)

### Summary

Add Enterprise Catalog related tests and adjustments.

### Description

- Added test profiles with Enterprise Catalog as storage option and
respective tests → `TestProfileTemplate` and `TestProfileValidation`
- Added test to confirm that twin strategy is **not** applied when
Enterprise Catalog is used → `TestTwinStrategyNotAppliedDremio`
- Updated expected failures

### Test Results

All tests are passing/failing as expected

### Changelog

-   [x] Added a summary of what this PR accomplishes to CHANGELOG.md

### Related Issue

#298

Author: 99Lys

.github/expected_failures.txt

@@ -1,4 +1,3 @@
-tests/component/test_profile_template.py::TestProfileTemplate::test_cloud_options
 tests/functional/adapter/dremio_specific/test_drop_temp_table.py::TestDropTempTableDremio::test_drop_temp_table
 tests/functional/adapter/dremio_specific/test_schema_parsing.py::TestSchemaParsingDremio::test_schema_with_dots
 tests/functional/adapter/dremio_specific/test_verify_ssl.py::TestVerifyCertificateDremio::test_insecure_request_warning_not_exist

.github/expected_failures_cloud.txt

@@ -1,24 +1,14 @@
 tests/functional/adapter/unit_testing/test_unit_testing.py::TestDremioUnitTestingTypes::test_unit_test_data_type
-tests/functional/adapter/dremio_specific/test_persist_docs.py::TestPersistDocs::test_view_model_delete_wikis_and_tags
-tests/functional/adapter/dremio_specific/test_persist_docs.py::TestPersistDocs::test_view_model_update_wikis_and_tags
-tests/functional/adapter/dremio_specific/test_persist_docs.py::TestPersistDocs::test_view_model_wikis_and_tags_remain_when_no_changes
 tests/functional/adapter/grants/test_incremental_grants.py::TestIncrementalGrantsDremio::test_incremental_grants
 tests/functional/adapter/grants/test_model_grants.py::TestTableGrantsDremio::test_view_table_grants
 tests/functional/adapter/grants/test_model_grants.py::TestViewGrantsDremio::test_view_table_grants
 tests/functional/adapter/grants/test_seed_grants.py::TestSeedGrantsDremio::test_seed_grants
 tests/functional/adapter/materialization/test_base_mat.py::TestSimpleMaterializationsDremio::test_base
-tests/functional/adapter/utils/test_array_append.py::TestArrayAppend::test_expected_actual
-tests/functional/adapter/utils/test_array_concat.py::TestArrayConcat::test_expected_actual
 tests/functional/adapter/utils/test_null_compare.py::TestMixedNullCompare::test_build_assert_equal
 tests/functional/adapter/utils/test_null_compare.py::TestNullCompare::test_build_assert_equal
 tests/simple_copy/test_simple_copy.py::TestSimpleCopyBaseDremio::test_simple_copy_with_materialized_views
 tests/functional/adapter/grants/test_snapshot_grants.py::TestSnapshotGrantsDremio::test_snapshot_grants
 tests/component/test_profile_template.py::TestProfileTemplate::test_software_pat_options
 tests/component/test_profile_template.py::TestProfileTemplate::test_software_username_password_options
-tests/functional/adapter/basic/test_docs_generate.py::TestBaseDocsGenerateDremio::test_run_and_generate
 tests/functional/adapter/dremio_specific/test_exact_search.py::TestExactSearchEnabled::test_exact_match_succeeds
 tests/functional/adapter/dremio_specific/test_exact_search.py::TestExactSearchEnabled::test_ilike_match_succeeds
-tests/hooks/test_model_hooks.py::TestPrePostModelHooksInConfigDremio::test_pre_and_post_model_hooks_model
-tests/hooks/test_model_hooks.py::TestPrePostModelHooksInConfigWithCountDremio::test_pre_and_post_model_hooks_model_and_project
-tests/hooks/test_model_hooks.py::TestPrePostModelHooksInConfigKwargsDremio::test_pre_and_post_model_hooks_model
-tests/hooks/test_model_hooks.py::TestPrePostModelHooksUnderscoresDremio::test_pre_and_post_run_hooks

.github/scripts/compare_test_failures.sh

@@ -1,7 +1,21 @@
 #!/bin/bash
 set -e
 
-echo "Comparing actual test failures with expected failures..."
+# Check if expected failures file is provided as argument
+if [ $# -eq 0 ]; then
+    echo "Error: Expected failures file not provided."
+    exit 1
+fi
+
+expected_failures_file="$1"
+
+# Check if the expected failures file exists
+if [ ! -f "$expected_failures_file" ]; then
+    echo "Error: Expected failures file '$expected_failures_file' not found."
+    exit 1
+fi
+
+echo "Comparing actual test failures with expected failures from: $expected_failures_file"
 
 # Enable globstar for recursive globbing
 shopt -s globstar
@@ -10,7 +24,7 @@ shopt -s globstar
 actual_failures=$(grep -E "(FAILED tests|ERROR tests)" reports/**/*.txt | awk '{print $2}' | sort)
 
 # Read expected failures
-expected_failures=$(sort .github/expected_failures.txt)
+expected_failures=$(sort "$expected_failures_file")
 
 echo "Expected Failures:"
 echo "$expected_failures"

.github/scripts/profiles.yml

@@ -28,3 +28,29 @@ test_sw_up_options:
       use_ssl: false
       user: dremio
   target: dev
+test_sw_enterprise_catalog_pat_options:
+  outputs:
+    dev:
+      enterprise_catalog_namespace: test_catalog
+      enterprise_catalog_folder: no_schema
+      pat: dremio123
+      port: 9047
+      software_host: localhost
+      threads: 1
+      type: dremio
+      use_ssl: false
+      user: dremio
+  target: dev
+test_sw_enterprise_catalog_up_options:
+  outputs:
+    dev:
+      enterprise_catalog_namespace: test_catalog
+      enterprise_catalog_folder: no_schema
+      password: dremio123
+      port: 9047
+      software_host: localhost
+      threads: 1
+      type: dremio
+      use_ssl: false
+      user: dremio
+  target: dev

.github/workflows/ci.yml

@@ -136,4 +136,4 @@ jobs:
           path: reports/
 
       - name: Compare Actual Failures with Expected Failures
-        run: bash .github/scripts/compare_test_failures.sh
+        run: bash .github/scripts/compare_test_failures.sh .github/expected_failures.txt

CHANGELOG.md

@@ -5,6 +5,7 @@
 - [#299](https://github.com/dremio/dbt-dremio/pull/299) Enhance persist_docs macro to wrap model and column metadata (including descriptions, tags and tests) into a Markdown wiki for Dremio.
 - Refactored CI
 - Fixed tests for hooks and grants
+- Added Dremio Enterprise Catalog tests
 
 # dbt-dremio v1.9.0
 

tests/component/test_profile_template.py

@@ -19,9 +19,11 @@
 from dbt.adapters.dremio.credentials import DremioCredentials
 from tests.utils.util import SOURCE
 
+DREMIO_EDITION = os.getenv("DREMIO_EDITION")
+
 # Tests require manual setup before executing.
 #
-#     Prior to running these tests, create three dbt projects:
+#     Prior to running these tests, create six dbt projects:
 #
 #     1. `dbt init test_cloud_options`
 #         - accept all default options
@@ -35,8 +37,27 @@
 #         - accept all default options
 #         - provide any value for mandatory options
 #
+#     4. `dbt init test_sw_enterprise_catalog_pat_options`
+#         - select software_with_pat
+#         - select enterprise catalog storage option
+#         - select PAT authentication
+#         - accept all default options
+#         - provide any value for mandatory options
+#
+#     5. `dbt init test_sw_enterprise_catalog_up_options`
+#         - select software_with_username_password
+#         - select enterprise catalog storage option
+#         - select username/password authentication
+#         - accept all default options
+#         - provide any value for mandatory options
+#
+#     6. `dbt init test_cloud_enterprise_catalog_options`
+#         - select cloud
+#         - select enterprise catalog storage option
+#         - accept all default options
+#         - provide any value for mandatory options
 #     These tests assumes there exists a $HOME/.dbt/profiles.yml
-#     file containing these three dbt projects.
+#     file containing these six dbt projects.
 
 
 class TestProfileTemplate:
@@ -51,6 +72,9 @@ class TestProfileTemplate:
         "test_sw_up_options"  # nosec hardcoded_password_string
     )
     _TEST_SOFTWARE_PAT_PROFILE_PROJECT = "test_sw_pat_options"
+    _TEST_SW_ENTERPRISE_CATALOG_PAT_PROFILE_PROJECT = "test_sw_enterprise_catalog_pat_options"
+    _TEST_SW_ENTERPRISE_CATALOG_USER_PASSWORD_PROFILE_PROJECT = "test_sw_enterprise_catalog_up_options"
+    _TEST_CLOUD_ENTERPRISE_CATALOG_PROFILE_PROJECT = "test_cloud_enterprise_catalog_options"
 
     _PASSWORD_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS = {"password": None}
     _PAT_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS = {"pat": None}
@@ -71,6 +95,18 @@ class TestProfileTemplate:
         "port": 9047,
         "use_ssl": False,
     }
+    _DREMIO_SW_ENTERPRISE_CATALOG_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS = {
+        "enterprise_catalog_namespace": None,
+        "enterprise_catalog_folder": None,
+        "software_host": None,
+        "port": 9047,
+        "use_ssl": False,
+    }
+    _DREMIO_CLOUD_ENTERPRISE_CATALOG_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS = {
+        "enterprise_catalog_namespace": None,
+        "enterprise_catalog_folder": None,
+        "use_ssl": False,
+    }
 
     _DREMIO_CLOUD_PROFILE_OPTIONS_WITH_DEFAULTS = (
         _COMMON_PROFILE_OPTIONS_WITH_DEFAULTS
@@ -87,6 +123,21 @@ class TestProfileTemplate:
         | _DREMIO_SW_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS
         | _PAT_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS
     )
+    _DREMIO_SW_ENTERPRISE_CATALOG_USERNAME_PASSWORD_PROFILE_OPTIONS = (
+        _COMMON_PROFILE_OPTIONS_WITH_DEFAULTS
+        | _DREMIO_SW_ENTERPRISE_CATALOG_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS
+        | _PASSWORD_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS
+    )
+    _DREMIO_SW_ENTERPRISE_CATALOG_PAT_PROFILE_OPTIONS = (
+        _COMMON_PROFILE_OPTIONS_WITH_DEFAULTS
+        | _DREMIO_SW_ENTERPRISE_CATALOG_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS
+        | _PAT_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS
+    )
+    _DREMIO_CLOUD_ENTERPRISE_CATALOG_PROFILE_OPTIONS_WITH_DEFAULTS = (
+        _COMMON_PROFILE_OPTIONS_WITH_DEFAULTS
+        | _DREMIO_CLOUD_ENTERPRISE_CATALOG_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS
+        | _PAT_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS
+    )
 
     _PROFILE_OPTIONS_ALIASES = {
         "username": "UID",
@@ -98,6 +149,10 @@ class TestProfileTemplate:
         "dremio_space_folder": "schema",
     }
 
+    @pytest.mark.skipif(
+        DREMIO_EDITION != "cloud",
+        reason="Cloud options are not available in Dremio community/enterprise edition.",
+    )
     def test_cloud_options(self) -> None:
         self._test_project_profile_options(
             self._get_dbt_test_project_dict(self._TEST_CLOUD_PROFILE_PROJECT),
@@ -125,6 +180,37 @@ def test_aliases(self) -> None:
             assert credential_option_aliases[option] is not None
             assert alias == credential_option_aliases[option]
 
+
+    @pytest.mark.skipif(
+        DREMIO_EDITION != "enterprise",
+        reason="Dremio Software enterprise catalog options are specific to Dremio Enterprise edition.",
+    )
+    def test_sw_enterprise_catalog_username_password_options(self) -> None:
+        self._test_project_profile_options(
+            self._get_dbt_test_project_dict(self._TEST_SW_ENTERPRISE_CATALOG_USER_PASSWORD_PROFILE_PROJECT),
+            self._DREMIO_SW_ENTERPRISE_CATALOG_USERNAME_PASSWORD_PROFILE_OPTIONS,
+        )
+
+    @pytest.mark.skipif(
+        DREMIO_EDITION != "enterprise",
+        reason="Dremio Software enterprise catalog options are specific to Dremio Enterprise edition.",
+    )
+    def test_sw_enterprise_catalog_pat_options(self) -> None:
+        self._test_project_profile_options(
+            self._get_dbt_test_project_dict(self._TEST_SW_ENTERPRISE_CATALOG_PAT_PROFILE_PROJECT),
+            self._DREMIO_SW_ENTERPRISE_CATALOG_PAT_PROFILE_OPTIONS,
+        )
+
+    @pytest.mark.skipif(
+        DREMIO_EDITION != "cloud",
+        reason="Dremio Software enterprise catalog options are specific to Dremio Enterprise edition.",
+    )
+    def test_cloud_enterprise_catalog_options(self) -> None:
+        self._test_project_profile_options(
+            self._get_dbt_test_project_dict(self._TEST_CLOUD_ENTERPRISE_CATALOG_PROFILE_PROJECT),
+            self._DREMIO_CLOUD_ENTERPRISE_CATALOG_PROFILE_OPTIONS_WITH_DEFAULTS,
+        )
+
     @pytest.mark.skip
     def _get_dbt_test_project_dict(self, dbt_test_project_name: str) -> Dict[str, any]:
         # read_profile returns dictionary with the following layout:
@@ -142,3 +228,30 @@ def _test_project_profile_options(
             assert test_project[option] is not None
             if test_options[option] is not None:
                 assert test_project[option] == test_options[option]
+
+class TestProfileValidation:
+    @pytest.mark.skipif(
+        DREMIO_EDITION == "community",
+        reason="Enterprise catalog options are not available in Dremio community edition.",
+    )
+    def test_invalid_enterprise_catalog_profile(self) -> None:
+        """Test that using both enterprise catalog and space & source configs raises validation error."""
+        from dbt_common.exceptions import DbtValidationError
+
+        # Simulate profile data with both enterprise catalog and individual storage configurations
+        conflicting_data = {
+            "enterprise_catalog_namespace": "my_enterprise_catalog",
+            "enterprise_catalog_folder": "analytics_folder",
+            "datalake": "my_source",
+            "root_path": "my_path",
+            "database": "my_space",
+            "schema": "my_schema",
+            "user": "test_user",
+            "pat": "test_token",
+            "threads": 1,
+            "type": "dremio"
+        }
+
+        # Verify that validation raises an error for conflicting configurations
+        with pytest.raises(DbtValidationError, match="Cannot use both enterprise catalog and individual storage configurations"):
+            DremioCredentials._validate_and_restructure_data(conflicting_data)

tests/functional/adapter/dremio_specific/test_twin_strategy.py

@@ -1,13 +1,18 @@
+import os
 import pytest
 from tests.fixtures.profiles import unique_schema, dbt_profile_data
 
-from dbt.tests.util import run_dbt, write_file
+from dbt.tests.util import run_dbt, write_file, run_dbt_and_capture
 from tests.utils.util import (
     check_relation_types,
     relation_from_name,
-    get_connection
+    get_connection,
+    BUCKET
 )
 
+DREMIO_EDITION = os.getenv("DREMIO_EDITION")
+DREMIO_ENTERPRISE_CATALOG = os.getenv("DREMIO_ENTERPRISE_CATALOG")
+
 schema_prevent_yml = """
 version: 2
 models:
@@ -231,3 +236,60 @@ def test_overwrite_table(self, project):
         assert columns_view[0].name == "table_column" 
         assert len(columns_table) == 1
         assert columns_table[0].name == "table_column"
+
+class TestTwinStrategyNotAppliedDremio:
+    # Override unique_schema to be the schema defined in Jenkins tests, i.e., tests_functional_adapter_dremio_specific
+    @pytest.fixture(scope="class")
+    def unique_schema(self, request, prefix) -> str:
+        test_file_path = request.module.__file__
+        relative_path = os.path.relpath(test_file_path, os.getcwd())
+        # Get directory path and remove filename
+        dir_path = os.path.dirname(relative_path)
+        # Replace '/' with '_' to create schema name
+        test_file = dir_path.replace('/', '_')
+        unique_schema = test_file
+        return unique_schema
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "schema.yml": schema_allow_yml,
+            "view_model.sql": view_model_sql,
+            "table_model.sql": table_model_sql,
+        }
+
+    @pytest.fixture(scope="class")
+    def dbt_profile_data(
+        self, unique_schema, dbt_profile_target, profiles_config_update
+    ):
+        profile = {
+            "test": {
+                "outputs": {
+                    "default": {},
+                },
+                "target": "default",
+            },
+        }
+        target = dbt_profile_target
+        # For enterprise catalog: object_storage_source == dremio_space AND object_storage_path == dremio_space_folder
+        # This maps to: target.datalake == target.database AND target.root_path == target.schema
+        enterprise_catalog_name = DREMIO_ENTERPRISE_CATALOG
+        target["schema"] = unique_schema
+        target["root_path"] = unique_schema # Make object_storage_path == dremio_space_folder
+        target["datalake"] = enterprise_catalog_name  # Set object_storage_source to enterprise catalog
+        target["database"] = enterprise_catalog_name  # Set dremio_space to same enterprise catalog
+
+        profile["test"]["outputs"]["default"] = target
+        if profiles_config_update:
+            profile.update(profiles_config_update)
+        return profile
+
+    @pytest.mark.skipif(DREMIO_EDITION == "community" or not DREMIO_ENTERPRISE_CATALOG, reason="Enterprise catalog is only supported in Dremio EE/DC editions.")
+    def test_twin_strategy_not_applied_with_enterprise_catalog(self, project):
+        # Run with twin_strategy configured but enterprise catalog enabled
+        # Should show warning and not apply twin strategy
+        (results, log_output) = run_dbt_and_capture(["--debug","run"])
+        assert len(results) == 2 # Both models should build successfully
+
+        # Check that the warning message appears in the logs
+        assert "WARNING: Twin strategy not applied - using enterprise catalog" in log_output