diff --git a/CHANGELOG.md b/CHANGELOG.md index d3583cc9..8ca1d4b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# dbt-dremio v1.10.1 + +## Changes + +- Reenabled `object_storage_path` and `root_path` as config parameters. + # dbt-dremio v1.10.0 ## Changes diff --git a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql index 67d8ba94..0479afe9 100644 --- a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql +++ b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql @@ -15,6 +15,8 @@ limitations under the License.*/ {% macro dremio__generate_schema_name(custom_schema_name, node) -%} {%- set default_schema = target.schema if not is_datalake_node(node) else target.root_path -%} + {%- set custom_schema_name = custom_schema_name if not is_datalake_node(node) + else node.config.root_path -%} {{ generate_schema_name_impl(default_schema, custom_schema_name, node) }} {%- endmacro %} diff --git a/dbt/include/dremio/profile_template.yml b/dbt/include/dremio/profile_template.yml index e13eedb2..5f297e27 100644 --- a/dbt/include/dremio/profile_template.yml +++ b/dbt/include/dremio/profile_template.yml @@ -66,13 +66,13 @@ prompts: hint: 'object storage source for seeds, tables, etc. [dbt alias: datalake]' object_storage_path: default: 'no_schema' - hint: 'object storage path [dbt alias: schema]' + hint: 'object storage path [dbt alias: root_path]' dremio_space: default: '@user' hint: 'space for creating views [dbt alias: database]' dremio_space_folder: default: 'no_schema' - hint: 'dremio space folder [dbt alias: root_path]' + hint: 'dremio space folder [dbt alias: schema]' threads: hint: '1 or more' type: 'int' diff --git a/tests/functional/adapter/dremio_specific/test_exact_search.py b/tests/functional/adapter/dremio_specific/test_exact_search.py index 8c2ca6d2..82200e93 100644 --- a/tests/functional/adapter/dremio_specific/test_exact_search.py +++ b/tests/functional/adapter/dremio_specific/test_exact_search.py @@ -2,6 +2,7 @@ from dbt.tests.util import run_dbt, get_connection from tests.utils.util import relation_from_name +from tests.fixtures.profiles import unique_schema, dbt_profile_data class TestExactSearchEnabled: @@ -12,7 +13,7 @@ def models(self): "create_table.sql": """ {{ config( materialized='table', - schema='schema') }} + root_path='schema') }} select 1 as ilike """ } @@ -117,4 +118,4 @@ def test_ilike_case_sensitive_list_relations_without_caching(self, project): with get_connection(adapter): columns = adapter.list_relations_without_caching(table_relation) - assert len(columns) == 1 \ No newline at end of file + assert len(columns) == 1 diff --git a/tests/functional/adapter/dremio_specific/test_nested_schema.py b/tests/functional/adapter/dremio_specific/test_nested_schema.py index c32b5edf..778eed95 100644 --- a/tests/functional/adapter/dremio_specific/test_nested_schema.py +++ b/tests/functional/adapter/dremio_specific/test_nested_schema.py @@ -1,8 +1,9 @@ import pytest from dbt.tests.util import run_dbt, get_connection -from tests.utils.util import relation_from_name +from tests.fixtures.profiles import unique_schema, dbt_profile_data +# Table tests - should use root_path config custom_schema_table_no_schema_model = """ {{ config( materialized='table' @@ -10,22 +11,31 @@ select 1 as id """ -custom_schema_table_model = """ +custom_schema_table_with_root_path_model = """ {{ config( materialized='table', - schema='schema' + root_path='schema' ) }} select 1 as id """ -custom_schema_table_nested_model = """ +custom_schema_table_with_schema_config_model = """ {{ config( materialized='table', - schema='nested.schema' + schema='useless_config' +) }} +select 1 as id +""" + +custom_schema_table_nested_with_root_path_model = """ +{{ config( + materialized='table', + root_path='nested.schema' ) }} select 1 as id """ +# View tests - should use schema config custom_schema_view_no_schema_model = """ {{ config( materialized='view' @@ -33,7 +43,7 @@ select 1 as id """ -custom_schema_view_model = """ +custom_schema_view_with_schema_config_model = """ {{ config( materialized='view', schema='schema' @@ -41,7 +51,15 @@ select 1 as id """ -custom_schema_view_nested_model = """ +custom_schema_view_with_root_path_config_model = """ +{{ config( + materialized='view', + root_path='useless_config' +) }} +select 1 as id +""" + +custom_schema_view_nested_with_schema_config_model = """ {{ config( materialized='view', schema='nested.schema' @@ -54,64 +72,233 @@ class TestGetCustomSchema: @pytest.fixture(scope="class") def models(self): return { + # Table models "custom_schema_table_no_schema.sql": custom_schema_table_no_schema_model, - "custom_schema_table.sql": custom_schema_table_model, - "custom_schema_table_nested.sql": custom_schema_table_nested_model, + "custom_schema_table_with_root_path.sql": custom_schema_table_with_root_path_model, + "custom_schema_table_with_schema_config.sql": custom_schema_table_with_schema_config_model, + "custom_schema_table_nested_with_root_path.sql": custom_schema_table_nested_with_root_path_model, + # View models "custom_schema_view_no_schema.sql": custom_schema_view_no_schema_model, - "custom_schema_view.sql": custom_schema_view_model, - "custom_schema_view_nested.sql": custom_schema_view_nested_model, + "custom_schema_view_with_schema_config.sql": custom_schema_view_with_schema_config_model, + "custom_schema_view_with_root_path_config.sql": custom_schema_view_with_root_path_config_model, + "custom_schema_view_nested_with_schema_config.sql": custom_schema_view_nested_with_schema_config_model, } + # ===== TABLE TESTS ===== + def test_custom_schema_table_no_schema(self, project): + """Test table without custom schema - should use default root_path from profile""" run_dbt(["run", "--select", "custom_schema_table_no_schema"]) - table_relation = relation_from_name( - project.adapter, "custom_schema_table_no_schema") + + # Expected path components + credentials = project.adapter.config.credentials + expected_database = credentials.datalake + expected_schema = credentials.root_path # e.g., "dbtdremios3.test17636798006768308215_test_nested_schema" + expected_identifier = "custom_schema_table_no_schema" + + # Get the actual relation from Dremio + with get_connection(project.adapter): + actual_relation = project.adapter.get_relation( + database=expected_database, + schema=expected_schema, + identifier=expected_identifier + ) + + # Verify the relation was created with the expected path + assert actual_relation is not None, f"Table should have been created at {expected_database}.{expected_schema}.{expected_identifier}" + assert actual_relation.type == "table" + + columns = project.adapter.get_columns_in_relation(actual_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + def test_custom_schema_table_with_root_path(self, project): + """Test table with root_path config - should be appended to default root_path""" + run_dbt(["run", "--select", "custom_schema_table_with_root_path"]) + + # Expected path components + credentials = project.adapter.config.credentials + expected_database = credentials.datalake + expected_schema = f"{credentials.root_path}.schema" + expected_identifier = "custom_schema_table_with_root_path" + + # Get the actual relation from Dremio with get_connection(project.adapter): - columns = project.adapter.get_columns_in_relation(table_relation) - assert len(columns) == 1 - assert columns[0].name == "id" - - def test_custom_schema_table(self, project): - run_dbt(["run", "--select", "custom_schema_table"]) - table_relation = relation_from_name( - project.adapter, "schema.custom_schema_table") + actual_relation = project.adapter.get_relation( + database=expected_database, + schema=expected_schema, + identifier=expected_identifier + ) + + # Verify the relation was created with the expected path + assert actual_relation is not None, f"Table should have been created at {expected_database}.{expected_schema}.{expected_identifier}" + assert actual_relation.type == "table" + + # Verify we can query it + columns = project.adapter.get_columns_in_relation(actual_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + def test_custom_schema_table_with_schema_config(self, project): + """Test table with schema config - should be IGNORED for tables (root_path is used instead)""" + run_dbt(["run", "--select", "custom_schema_table_with_schema_config"]) + + # Expected path components + credentials = project.adapter.config.credentials + expected_database = credentials.datalake + expected_schema = credentials.root_path # schema config is ignored for tables + expected_identifier = "custom_schema_table_with_schema_config" + + # Get the actual relation from Dremio with get_connection(project.adapter): - columns = project.adapter.get_columns_in_relation(table_relation) - assert len(columns) == 1 - assert columns[0].name == "id" - - def test_custom_schema_table_nested(self, project): - run_dbt(["run", "--select", "custom_schema_table_nested"]) - table_relation = relation_from_name( - project.adapter, "nested.schema.custom_schema_table_nested") + actual_relation = project.adapter.get_relation( + database=expected_database, + schema=expected_schema, + identifier=expected_identifier + ) + + # Verify the relation was created with the expected path + assert actual_relation is not None, f"Table should have been created at {expected_database}.{expected_schema}.{expected_identifier}" + assert actual_relation.type == "table" + + # Verify we can query it + columns = project.adapter.get_columns_in_relation(actual_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + def test_custom_schema_table_nested_with_root_path(self, project): + """Test table with nested root_path config - should be appended to default root_path""" + run_dbt(["run", "--select", "custom_schema_table_nested_with_root_path"]) + + # Expected path components + credentials = project.adapter.config.credentials + expected_database = credentials.datalake + expected_schema = f"{credentials.root_path}.nested.schema" + expected_identifier = "custom_schema_table_nested_with_root_path" + + # Get the actual relation from Dremio with get_connection(project.adapter): - columns = project.adapter.get_columns_in_relation(table_relation) - assert len(columns) == 1 - assert columns[0].name == "id" + actual_relation = project.adapter.get_relation( + database=expected_database, + schema=expected_schema, + identifier=expected_identifier + ) + + # Verify the relation was created with the expected path + assert actual_relation is not None, f"Table should have been created at {expected_database}.{expected_schema}.{expected_identifier}" + assert actual_relation.type == "table" + + # Verify we can query it + columns = project.adapter.get_columns_in_relation(actual_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + # ===== VIEW TESTS ===== def test_custom_schema_view_no_schema(self, project): + """Test view without custom schema - should use default schema from profile""" run_dbt(["run", "--select", "custom_schema_view_no_schema"]) - view_relation = relation_from_name( - project.adapter, "custom_schema_view_no_schema") + + # Expected path components + credentials = project.adapter.config.credentials + expected_database = credentials.database + expected_schema = credentials.schema # e.g., "test17636798006768308215_test_nested_schema" + expected_identifier = "custom_schema_view_no_schema" + + # Get the actual relation from Dremio + with get_connection(project.adapter): + actual_relation = project.adapter.get_relation( + database=expected_database, + schema=expected_schema, + identifier=expected_identifier + ) + + # Verify the relation was created with the expected path + assert actual_relation is not None, f"View should have been created at {expected_database}.{expected_schema}.{expected_identifier}" + assert actual_relation.type == "view" + + # Verify we can query it + columns = project.adapter.get_columns_in_relation(actual_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + def test_custom_schema_view_with_schema_config(self, project): + """Test view with schema config - should be appended to default schema""" + run_dbt(["run", "--select", "custom_schema_view_with_schema_config"]) + + # Expected path components + credentials = project.adapter.config.credentials + expected_database = credentials.database + expected_schema = f"{credentials.schema}.schema" # Appended! + expected_identifier = "custom_schema_view_with_schema_config" + + # Get the actual relation from Dremio with get_connection(project.adapter): - columns = project.adapter.get_columns_in_relation(view_relation) - assert len(columns) == 1 - assert columns[0].name == "id" - - def test_custom_schema_view(self, project): - run_dbt(["run", "--select", "custom_schema_view"]) - view_relation = relation_from_name( - project.adapter, "schema.custom_schema_view") + actual_relation = project.adapter.get_relation( + database=expected_database, + schema=expected_schema, + identifier=expected_identifier + ) + + # Verify the relation was created with the expected path + assert actual_relation is not None, f"View should have been created at {expected_database}.{expected_schema}.{expected_identifier}" + assert actual_relation.type == "view" + + # Verify we can query it + columns = project.adapter.get_columns_in_relation(actual_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + def test_custom_schema_view_with_root_path_config(self, project): + """Test view with root_path config - should be IGNORED for views (schema is used instead)""" + run_dbt(["run", "--select", "custom_schema_view_with_root_path_config"]) + + # Expected path components - root_path config should be ignored for views + credentials = project.adapter.config.credentials + expected_database = credentials.database + expected_schema = credentials.schema # root_path config is ignored for views + expected_identifier = "custom_schema_view_with_root_path_config" + + # Get the actual relation from Dremio with get_connection(project.adapter): - columns = project.adapter.get_columns_in_relation(view_relation) - assert len(columns) == 1 - assert columns[0].name == "id" - - def test_custom_schema_view_nested(self, project): - run_dbt(["run", "--select", "custom_schema_view_nested"]) - view_relation = relation_from_name( - project.adapter, "nested.schema.custom_schema_view_nested") + actual_relation = project.adapter.get_relation( + database=expected_database, + schema=expected_schema, + identifier=expected_identifier + ) + + # Verify the relation was created with the expected path (root_path config ignored) + assert actual_relation is not None, f"View should have been created at {expected_database}.{expected_schema}.{expected_identifier} (root_path config should be ignored for views)" + assert actual_relation.type == "view" + + # Verify we can query it + columns = project.adapter.get_columns_in_relation(actual_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + def test_custom_schema_view_nested_with_schema_config(self, project): + """Test view with nested schema config - should be appended to default schema""" + run_dbt(["run", "--select", "custom_schema_view_nested_with_schema_config"]) + + # Expected path components + credentials = project.adapter.config.credentials + expected_database = credentials.database + expected_schema = f"{credentials.schema}.nested.schema" # Appended! + expected_identifier = "custom_schema_view_nested_with_schema_config" + + # Get the actual relation from Dremio with get_connection(project.adapter): - columns = project.adapter.get_columns_in_relation(view_relation) - assert len(columns) == 1 - assert columns[0].name == "id" + actual_relation = project.adapter.get_relation( + database=expected_database, + schema=expected_schema, + identifier=expected_identifier + ) + + # Verify the relation was created with the expected path + assert actual_relation is not None, f"View should have been created at {expected_database}.{expected_schema}.{expected_identifier}" + assert actual_relation.type == "view" + + # Verify we can query it + columns = project.adapter.get_columns_in_relation(actual_relation) + assert len(columns) == 1 + assert columns[0].name == "id"