Skip to content

Commit 16f18df

Browse files
authored
Fix databricks temp relations (#810)
* creating a view relation when creating a temp relation for databricks because dbt-databricks 1.10 now creates views only if the context of the model has language=sql but this macro also runs outside model context and then it fails. * separating the create_intermediate_relation from the make_temp_table_relation, because we want different databricks behaviors in these cases
1 parent 390ceb7 commit 16f18df

File tree

2 files changed

+22
-12
lines changed

2 files changed

+22
-12
lines changed

macros/utils/table_operations/create_intermediate_relation.sql

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{% macro create_intermediate_relation(base_relation, rows, temporary, like_columns=none) %}
2-
{% set int_relation = elementary.make_temp_table_relation(base_relation).incorporate(type='table') %}
2+
{% set int_relation = elementary.edr_make_intermediate_relation(base_relation).incorporate(type='table') %}
33

44
{% if not elementary.has_temp_table_support() %}
55
{% set temporary = false %}
@@ -9,3 +9,21 @@
99
{% do elementary.insert_rows(int_relation, rows, should_commit=false, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %}
1010
{% do return(int_relation) %}
1111
{% endmacro %}
12+
13+
{% macro edr_make_intermediate_relation(base_relation) %}
14+
{% do return(adapter.dispatch("edr_make_intermediate_relation", "elementary")(base_relation)) %}
15+
{% endmacro %}
16+
17+
{% macro default__edr_make_intermediate_relation(base_relation) %}
18+
{% do return(elementary.make_temp_table_relation(base_relation)) %}
19+
{% endmacro %}
20+
21+
{% macro databricks__edr_make_intermediate_relation(base_relation) %}
22+
{% set tmp_identifier = elementary.table_name_with_suffix(base_relation.identifier, elementary.get_timestamped_table_suffix()) %}
23+
{% set tmp_relation = api.Relation.create(
24+
identifier=tmp_identifier,
25+
schema=base_relation.schema,
26+
database=base_relation.database,
27+
type='table') %}
28+
{% do return(tmp_relation) %}
29+
{% endmacro %}

macros/utils/table_operations/make_temp_relation.sql

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
{% endmacro %}
1818

1919
{% macro databricks__edr_make_temp_relation(base_relation, suffix) %}
20-
{% do return(dbt.make_temp_relation(base_relation, suffix)) %}
20+
{% set tmp_identifier = elementary.table_name_with_suffix(base_relation.identifier, suffix) %}
21+
{% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}
22+
{% do return(tmp_relation) %}
2123
{% endmacro %}
2224

2325
--- VIEWS
@@ -43,16 +45,6 @@
4345
{% do return(elementary.edr_make_temp_relation(base_relation, suffix)) %}
4446
{% endmacro %}
4547

46-
{% macro databricks__make_temp_table_relation(base_relation, suffix) %}
47-
{% set tmp_identifier = elementary.table_name_with_suffix(base_relation.identifier, suffix) %}
48-
{% set tmp_relation = api.Relation.create(
49-
identifier=tmp_identifier,
50-
schema=base_relation.schema,
51-
database=base_relation.database,
52-
type='table') %}
53-
{% do return(tmp_relation) %}
54-
{% endmacro %}
55-
5648
{% macro clickhouse__make_temp_table_relation(base_relation, suffix) %}
5749
{% set tmp_identifier = elementary.table_name_with_suffix(base_relation.identifier, suffix) %}
5850
{% set tmp_relation = api.Relation.create(

0 commit comments

Comments
 (0)