diff --git a/dbt-bigquery/src/dbt/include/bigquery/macros/adapters.sql b/dbt-bigquery/src/dbt/include/bigquery/macros/adapters.sql index 6fd441228..451b9efc4 100644 --- a/dbt-bigquery/src/dbt/include/bigquery/macros/adapters.sql +++ b/dbt-bigquery/src/dbt/include/bigquery/macros/adapters.sql @@ -209,3 +209,34 @@ having count(*) > 1 {% do adapter.upload_file(local_file_path, database, table_schema, table_name, kwargs=kwargs) %} {% endmacro %} + +{% macro bigquery__make_relation_with_suffix(base_relation, suffix, dstring) %} + {% if dstring %} + {% set dt = modules.datetime.datetime.now() %} + {% set dtstring = dt.strftime("%H%M%S%f") %} + {% set suffix = suffix ~ dtstring %} + {% endif %} + {% set suffix_length = suffix|length %} + {% set relation_max_name_length = 1024 %} {# BigQuery limit #} + {% if suffix_length > relation_max_name_length %} + {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %} + {% endif %} + {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %} + + {{ return(base_relation.incorporate(path={"identifier": identifier })) }} + +{% endmacro %} + +{% macro bigquery__make_temp_relation(base_relation, suffix) %} + {% set temp_relation = bigquery__make_relation_with_suffix(base_relation, suffix, dstring=True) %} + {{ return(temp_relation) }} +{% endmacro %} + +{% macro bigquery__make_intermediate_relation(base_relation, suffix) %} + {{ return(bigquery__make_relation_with_suffix(base_relation, suffix, dstring=False)) }} +{% endmacro %} + +{% macro bigquery__make_backup_relation(base_relation, backup_relation_type, suffix) %} + {% set backup_relation = bigquery__make_relation_with_suffix(base_relation, suffix, dstring=False) %} + {{ return(backup_relation.incorporate(type=backup_relation_type)) }} +{% endmacro %} diff --git a/dbt-bigquery/tests/functional/adapter/test_temp_relation_naming.py b/dbt-bigquery/tests/functional/adapter/test_temp_relation_naming.py new file mode 100644 index 000000000..637563475 --- /dev/null +++ b/dbt-bigquery/tests/functional/adapter/test_temp_relation_naming.py @@ -0,0 +1,54 @@ +import pytest +import re +from pathlib import Path +from dbt.tests.util import run_dbt + + +_MODEL_INCREMENTAL = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='append_new_columns' + ) +}} + +select + 1 as id, + 'value' as name +""" + + +class TestTempRelationNaming: + @pytest.fixture(scope="class") + def models(self): + return {"my_incremental_model.sql": _MODEL_INCREMENTAL} + + def test_incremental_with_schema_change_creates_temp_relation(self, project): + results = run_dbt(["run"]) + assert len(results) == 1 + + results = run_dbt(["run", "--log-level", "debug", "--log-format", "text"]) + assert len(results) == 1 + + log_dir = Path(project.project_root) / "logs" + if log_dir.exists(): + log_files = sorted( + log_dir.glob("dbt.log*"), key=lambda p: p.stat().st_mtime, reverse=True + ) + if log_files: + log_content = log_files[0].read_text() + temp_table_pattern = r"__dbt_tmp\d{12}\b" + matches = re.findall(temp_table_pattern, log_content) + + assert len(matches) > 0, ( + f"Expected temp table with pattern '{temp_table_pattern}' " + f"(12 digits from %H%M%S%f format, not old 9-digit %j%H%M%S format)" + ) + + for match in matches: + suffix = match.replace("__dbt_tmp", "") + assert len(suffix) == 12 and suffix.isdigit() + + results = run_dbt(["run"]) + assert len(results) == 1 diff --git a/dbt-postgres/tests/functional/adapter/test_temp_relation_naming.py b/dbt-postgres/tests/functional/adapter/test_temp_relation_naming.py new file mode 100644 index 000000000..d2330ef79 --- /dev/null +++ b/dbt-postgres/tests/functional/adapter/test_temp_relation_naming.py @@ -0,0 +1,54 @@ +import pytest +import re +from pathlib import Path +from dbt.tests.util import run_dbt + + +_MODEL_INCREMENTAL = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='append_new_columns' + ) +}} + +select + 1 as id, + 'value' as name +""" + + +class TestTempRelationNaming: + @pytest.fixture(scope="class") + def models(self): + return {"my_incremental_model.sql": _MODEL_INCREMENTAL} + + def test_incremental_with_schema_change_creates_temp_relation(self, project): + results = run_dbt(["run"]) + assert len(results) == 1 + + results = run_dbt(["run", "--log-level", "debug", "--log-format", "text"]) + assert len(results) == 1 + + log_dir = Path(project.project_root) / "logs" + if log_dir.exists(): + log_files = sorted( + log_dir.glob("dbt.log*"), key=lambda p: p.stat().st_mtime, reverse=True + ) + if log_files: + log_content = log_files[0].read_text() + temp_table_pattern = r"__dbt_tmp\d{12}\b" + matches = re.findall(temp_table_pattern, log_content) + + assert len(matches) > 0, ( + f"Expected temp table with pattern '{temp_table_pattern}' " + f"(12 digits from %H%M%S%f format)" + ) + + for match in matches: + suffix = match.replace("__dbt_tmp", "") + assert len(suffix) == 12 and suffix.isdigit() + + results = run_dbt(["run"]) + assert len(results) == 1