Add new macros for Fusion compatibility

Author: b-per

macros/materialisations/period_mat_helpers/get_start_stop_dates.sql

@@ -5,9 +5,9 @@
 
 {% macro get_start_stop_dates(timestamp_field, date_source_models) %}
 
-    {% if config.get('start_date', default=none) is not none %}
-        {%- set start_date = config.get('start_date') -%}
-        {%- set stop_date = config.get('stop_date', default=none) -%}
+    {% if automate_dv.config_meta_get('start_date', default=none) is not none %}
+        {%- set start_date = automate_dv.config_meta_get('start_date') -%}
+        {%- set stop_date = automate_dv.config_meta_get('stop_date', default=none) -%}
 
         {% do return({'start_date': start_date,'stop_date': stop_date}) %}
 

macros/materialisations/vault_insert_by_period_materialization.sql

@@ -7,7 +7,7 @@
 
     {%- set full_refresh_mode = (should_full_refresh()) -%}
 
-    {%- set period = config.get('period', default='day') -%}
+    {%- set period = automate_dv.config_meta_get('period', default='day') -%}
 
     {#- Raise the errors/warnings in this order so that we do not get both -#}
     {%- if period == 'microsecond' -%}
@@ -22,8 +22,8 @@
     {%- set existing_relation = load_relation(this) -%}
     {%- set tmp_relation = make_temp_relation(target_relation) -%}
 
-    {%- set timestamp_field = config.require('timestamp_field') -%}
-    {%- set date_source_models = config.get('date_source_models', default=none) -%}
+    {%- set timestamp_field = automate_dv.config_meta_require('timestamp_field') -%}
+    {%- set date_source_models = automate_dv.config_meta_get('date_source_models', default=none) -%}
 
     {%- set start_stop_dates = automate_dv.get_start_stop_dates(timestamp_field, date_source_models) | as_native -%}
 

macros/materialisations/vault_insert_by_rank_materialization.sql

@@ -18,8 +18,8 @@
     {%- set existing_relation = load_relation(this) -%}
     {%- set tmp_relation = make_temp_relation(target_relation) -%}
 
-    {%- set rank_column = config.require('rank_column') -%}
-    {%- set rank_source_models = config.require('rank_source_models') -%}
+    {%- set rank_column = automate_dv.config_meta_require('rank_column') -%}
+    {%- set rank_source_models = automate_dv.config_meta_require('rank_source_models') -%}
 
     {%- set min_max_ranks = automate_dv.get_min_max_ranks(rank_column, rank_source_models) | as_native -%}
 

macros/supporting/fusion_compat.sql

@@ -0,0 +1,23 @@
+{% macro config_meta_get(key, default=none) %}
+    {%- if config.get(key) != none -%}
+        {{ return(config.get(key)) }}
+    {%- elif config.get("meta") != none and (key in config.get("meta", {}).keys()) -%}
+        {{ return(config.get("meta").get(key)) }}
+    {%- else -%}
+        {{ return(default) }}
+    {%- endif -%}
+{% endmacro %}
+
+
+{% macro config_meta_require(key) %}
+    {# the first case is required to avoid errors #}
+    {%- if config == {} -%}
+        {{ return(none) }}
+    {%- elif config.get(key) != none -%}
+        {{ return(config.get(key)) }}
+    {%- elif config.get("meta") != none and (key in config.get("meta", {}).keys()) -%}
+        {{ return(config.get("meta").get(key)) }}
+    {%- else -%}
+        {% do exceptions.raise_compiler_error("Configuration '" ~ key ~ "' is required but was not found under config or meta (Fusion requires custom configuration under meta)") %}
+    {%- endif -%}
+{% endmacro %}

macros/tables/bigquery/hub.sql

@@ -8,7 +8,7 @@
 {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%}
 
 {%- if model.config.materialized == 'vault_insert_by_rank' %}
-    {%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%}
+    {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%}
 {%- endif %}
 
 {{ 'WITH ' -}}

macros/tables/postgres/hub.sql

@@ -8,7 +8,7 @@
 {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%}
 
 {%- if model.config.materialized == 'vault_insert_by_rank' %}
-    {%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%}
+    {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%}
 {%- endif %}
 
 {{ 'WITH ' -}}

macros/tables/postgres/link.sql

@@ -9,7 +9,7 @@
 {%- set fk_cols = automate_dv.expand_column_list([src_fk]) -%}
 
 {%- if model.config.materialized == 'vault_insert_by_rank' %}
-    {%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%}
+    {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%}
 {%- endif %}
 
 {{ 'WITH ' -}}

macros/tables/postgres/sat.sql

@@ -5,15 +5,15 @@
 
 {%- macro postgres__sat(src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%}
 
-{%- set apply_source_filter = config.get('apply_source_filter', false) -%}
+{%- set apply_source_filter = automate_dv.config_meta_get('apply_source_filter', false) -%}
 {%- set enable_ghost_record = var('enable_ghost_records', false) %}
 
 {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source]) -%}
 {%- set window_cols = automate_dv.expand_column_list(columns=[src_pk, src_hashdiff, src_ldts]) -%}
 {%- set pk_cols = automate_dv.expand_column_list(columns=[src_pk]) -%}
 
 {%- if model.config.materialized == 'vault_insert_by_rank' %}
-    {%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%}
+    {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%}
 {%- endif %}
 
 WITH source_data AS (

macros/tables/snowflake/eff_sat.sql

@@ -24,7 +24,7 @@
 {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_dfk, src_sfk, src_extra_columns, src_start_date, src_end_date, src_eff, src_ldts, src_source]) -%}
 {%- set fk_cols = automate_dv.expand_column_list(columns=[src_dfk, src_sfk]) -%}
 {%- set dfk_cols = automate_dv.expand_column_list(columns=[src_dfk]) -%}
-{%- set is_auto_end_dating = config.get('is_auto_end_dating', default=false) %}
+{%- set is_auto_end_dating = automate_dv.config_meta_get('is_auto_end_dating', default=false) %}
 
 {%- set max_datetime = automate_dv.max_datetime() %}
 

macros/tables/snowflake/hub.sql

@@ -29,7 +29,7 @@
 {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%}
 
 {%- if model.config.materialized == 'vault_insert_by_rank' %}
-    {%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%}
+    {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%}
 {%- endif %}
 
 {{ 'WITH ' -}}