Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions dbt-bigquery/src/dbt/include/bigquery/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -209,3 +209,34 @@ having count(*) > 1
{% do adapter.upload_file(local_file_path, database, table_schema, table_name, kwargs=kwargs) %}

{% endmacro %}

{% macro bigquery__make_relation_with_suffix(base_relation, suffix, dstring) %}
{% if dstring %}
{% set dt = modules.datetime.datetime.now() %}
{% set dtstring = dt.strftime("%H%M%S%f") %}
{% set suffix = suffix ~ dtstring %}
{% endif %}
{% set suffix_length = suffix|length %}
{% set relation_max_name_length = 1024 %} {# BigQuery limit #}
{% if suffix_length > relation_max_name_length %}
{% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}
{% endif %}
{% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}

{{ return(base_relation.incorporate(path={"identifier": identifier })) }}

{% endmacro %}

{% macro bigquery__make_temp_relation(base_relation, suffix) %}
{% set temp_relation = bigquery__make_relation_with_suffix(base_relation, suffix, dstring=True) %}
{{ return(temp_relation) }}
{% endmacro %}

{% macro bigquery__make_intermediate_relation(base_relation, suffix) %}
{{ return(bigquery__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}
{% endmacro %}

{% macro bigquery__make_backup_relation(base_relation, backup_relation_type, suffix) %}
{% set backup_relation = bigquery__make_relation_with_suffix(base_relation, suffix, dstring=False) %}
{{ return(backup_relation.incorporate(type=backup_relation_type)) }}
{% endmacro %}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import pytest
import re
from pathlib import Path
from dbt.tests.util import run_dbt


_MODEL_INCREMENTAL = """
{{
config(
materialized='incremental',
unique_key='id',
on_schema_change='append_new_columns'
)
}}

select
1 as id,
'value' as name
"""


class TestTempRelationNaming:
@pytest.fixture(scope="class")
def models(self):
return {"my_incremental_model.sql": _MODEL_INCREMENTAL}

def test_incremental_with_schema_change_creates_temp_relation(self, project):
results = run_dbt(["run"])
assert len(results) == 1

results = run_dbt(["run", "--log-level", "debug", "--log-format", "text"])
assert len(results) == 1

log_dir = Path(project.project_root) / "logs"
if log_dir.exists():
log_files = sorted(
log_dir.glob("dbt.log*"), key=lambda p: p.stat().st_mtime, reverse=True
)
if log_files:
log_content = log_files[0].read_text()
temp_table_pattern = r"__dbt_tmp\d{12}\b"
matches = re.findall(temp_table_pattern, log_content)

assert len(matches) > 0, (
f"Expected temp table with pattern '{temp_table_pattern}' "
f"(12 digits from %H%M%S%f format, not old 9-digit %j%H%M%S format)"
)

for match in matches:
suffix = match.replace("__dbt_tmp", "")
assert len(suffix) == 12 and suffix.isdigit()

results = run_dbt(["run"])
assert len(results) == 1
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import pytest
import re
from pathlib import Path
from dbt.tests.util import run_dbt


_MODEL_INCREMENTAL = """
{{
config(
materialized='incremental',
unique_key='id',
on_schema_change='append_new_columns'
)
}}

select
1 as id,
'value' as name
"""


class TestTempRelationNaming:
@pytest.fixture(scope="class")
def models(self):
return {"my_incremental_model.sql": _MODEL_INCREMENTAL}

def test_incremental_with_schema_change_creates_temp_relation(self, project):
results = run_dbt(["run"])
assert len(results) == 1

results = run_dbt(["run", "--log-level", "debug", "--log-format", "text"])
assert len(results) == 1

log_dir = Path(project.project_root) / "logs"
if log_dir.exists():
log_files = sorted(
log_dir.glob("dbt.log*"), key=lambda p: p.stat().st_mtime, reverse=True
)
if log_files:
log_content = log_files[0].read_text()
temp_table_pattern = r"__dbt_tmp\d{12}\b"
matches = re.findall(temp_table_pattern, log_content)

assert len(matches) > 0, (
f"Expected temp table with pattern '{temp_table_pattern}' "
f"(12 digits from %H%M%S%f format)"
)

for match in matches:
suffix = match.replace("__dbt_tmp", "")
assert len(suffix) == 12 and suffix.isdigit()

results = run_dbt(["run"])
assert len(results) == 1
Loading