From d2c678973004644503e22ee09ed2c3b79d0df4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Wed, 8 Jan 2025 20:41:08 +0000 Subject: [PATCH 1/6] changing destination Update incremental.sql --- .changes/unreleased/Features-20250108-204100.yaml | 6 ++++++ .../bigquery/macros/materializations/incremental.sql | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20250108-204100.yaml diff --git a/.changes/unreleased/Features-20250108-204100.yaml b/.changes/unreleased/Features-20250108-204100.yaml new file mode 100644 index 000000000..0e17005fa --- /dev/null +++ b/.changes/unreleased/Features-20250108-204100.yaml @@ -0,0 +1,6 @@ +kind: Features +body: allow changing the schema destination of tmp tables +time: 2025-01-08T20:41:00.190868Z +custom: + Author: borjavb + Issue: "1443" diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index 25a83b0c6..58014fade 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -72,12 +72,20 @@ {% materialization incremental, adapter='bigquery', supported_languages=['sql', 'python'] -%} {%- set unique_key = config.get('unique_key') -%} + {%- set temp_schema = config.get('temp_schema') -%} {%- set full_refresh_mode = (should_full_refresh()) -%} {%- set language = model['language'] %} {%- set target_relation = this %} {%- set existing_relation = load_relation(this) %} - {%- set tmp_relation = make_temp_relation(this) %} + {%- set temp_schema = config.get('temp_schema') -%} + {%- if temp_schema is not none-%} + {%- set temp_relation = this.incorporate(path={ + "schema": temp_schema + }) -%} + {%- do create_schema(temp_relation) -%} + {% endif %} + {%- set tmp_relation = make_temp_relation(temp_relation) %} {#-- Validate early so we don't run SQL if the strategy is invalid --#} {% set strategy = dbt_bigquery_validate_get_incremental_strategy(config) -%} From e75ccd881723dc625dfb2b373c29771b5b496d70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Fri, 10 Jan 2025 18:55:30 +0100 Subject: [PATCH 2/6] update schema --- .../macros/materializations/incremental.sql | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index 58014fade..8843fa8d2 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -69,23 +69,27 @@ {% endmacro %} + +{% macro generate_temp_schema(base_relation, temp_schema = none) %} + {%- if temp_schema is not none-%} + {%- set temp_relation = base_relation.incorporate(path={ + "schema": temp_schema + }) -%} + {%- do create_schema(temp_relation) -%} + {% endif %} +{% endmacro %} + {% materialization incremental, adapter='bigquery', supported_languages=['sql', 'python'] -%} {%- set unique_key = config.get('unique_key') -%} - {%- set temp_schema = config.get('temp_schema') -%} + {%- set tmp_schema = config.get('tmp_schema') -%} {%- set full_refresh_mode = (should_full_refresh()) -%} {%- set language = model['language'] %} {%- set target_relation = this %} {%- set existing_relation = load_relation(this) %} - {%- set temp_schema = config.get('temp_schema') -%} - {%- if temp_schema is not none-%} - {%- set temp_relation = this.incorporate(path={ - "schema": temp_schema - }) -%} - {%- do create_schema(temp_relation) -%} - {% endif %} - {%- set tmp_relation = make_temp_relation(temp_relation) %} + {%- set tmp_updated_relation = generate_tmp_schema(this, tmp_schema) -%} + {%- set tmp_relation = make_temp_relation(tmp_updated_relation) %} {#-- Validate early so we don't run SQL if the strategy is invalid --#} {% set strategy = dbt_bigquery_validate_get_incremental_strategy(config) -%} From 5606a84e0cc64bc7071184c5f64cbb8fc3f082b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Fri, 10 Jan 2025 18:56:01 +0100 Subject: [PATCH 3/6] Update incremental.sql --- .../macros/materializations/incremental.sql | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index 8843fa8d2..29a209fa3 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -70,13 +70,15 @@ {% endmacro %} -{% macro generate_temp_schema(base_relation, temp_schema = none) %} - {%- if temp_schema is not none-%} - {%- set temp_relation = base_relation.incorporate(path={ - "schema": temp_schema +{% macro generate_temp_schema(base_relation, tmp_schema = none) %} + {%- if tmp_schema is not none-%} + {%- set tmp_relation = base_relation.incorporate(path={ + "schema": tmp_schema }) -%} - {%- do create_schema(temp_relation) -%} + {%- do create_schema(tmp_relation) -%} + {{ return(tmp_relation) }} {% endif %} + {{ return(base_relation) }} {% endmacro %} {% materialization incremental, adapter='bigquery', supported_languages=['sql', 'python'] -%} @@ -88,8 +90,8 @@ {%- set target_relation = this %} {%- set existing_relation = load_relation(this) %} - {%- set tmp_updated_relation = generate_tmp_schema(this, tmp_schema) -%} - {%- set tmp_relation = make_temp_relation(tmp_updated_relation) %} + {%- set tmp_set_schema = generate_tmp_schema(this, tmp_schema) -%} + {%- set tmp_relation = make_temp_relation(tmp_set_schema) %} {#-- Validate early so we don't run SQL if the strategy is invalid --#} {% set strategy = dbt_bigquery_validate_get_incremental_strategy(config) -%} From 65c0fd2dcef517678d375befd6ef7c88f4aa416b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Fri, 10 Jan 2025 18:57:59 +0100 Subject: [PATCH 4/6] Update incremental.sql --- dbt/include/bigquery/macros/materializations/incremental.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index 29a209fa3..99170876d 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -70,7 +70,7 @@ {% endmacro %} -{% macro generate_temp_schema(base_relation, tmp_schema = none) %} +{% macro generate_tmp_schema(base_relation, tmp_schema = none) %} {%- if tmp_schema is not none-%} {%- set tmp_relation = base_relation.incorporate(path={ "schema": tmp_schema From 0879d857e8cd4792dec860c6d0d2bd073aa13b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Fri, 10 Jan 2025 18:58:23 +0100 Subject: [PATCH 5/6] Update incremental.sql --- dbt/include/bigquery/macros/materializations/incremental.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index 99170876d..c9a520ea0 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -70,7 +70,7 @@ {% endmacro %} -{% macro generate_tmp_schema(base_relation, tmp_schema = none) %} +{% macro generate_tmp_schema(base_relation, tmp_schema=none) %} {%- if tmp_schema is not none-%} {%- set tmp_relation = base_relation.incorporate(path={ "schema": tmp_schema From 599c1266fed59dc96dc8e915c95b03c82a5e0e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20V=C3=A1zquez-Barreiros?= Date: Fri, 10 Jan 2025 18:59:43 +0100 Subject: [PATCH 6/6] Update incremental.sql --- .../bigquery/macros/materializations/incremental.sql | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index c9a520ea0..3c211d4e1 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -72,11 +72,9 @@ {% macro generate_tmp_schema(base_relation, tmp_schema=none) %} {%- if tmp_schema is not none-%} - {%- set tmp_relation = base_relation.incorporate(path={ - "schema": tmp_schema - }) -%} - {%- do create_schema(tmp_relation) -%} - {{ return(tmp_relation) }} + {%- set tmp_relation = base_relation.incorporate(path={"schema": tmp_schema }) -%} + {%- do create_schema(tmp_relation) -%} + {{ return(tmp_relation) }} {% endif %} {{ return(base_relation) }} {% endmacro %}