Add support for clustering and automatic clustering in materialized views

Author: jonhopper-dataengineers

CHANGELOG.md

@@ -1,5 +1,10 @@
 # dbt_dataengineers_materializations Changelog
 
+
+## 0.2.10.3
+
+* Bug fix for `materialized_view` macro to ensure that the correct SQL statements are generated for applying clustering and enabling automatic clustering
+
 ## 0.2.10.2
 
 * Bug fix for `stored_procedure` materialization to ensure that the parameters are correctly set up when no parameters are passed in

macros/materialized-views/snowflake/apply_clusters.sql

@@ -0,0 +1,17 @@
+{% macro apply_clusters(relation, config) -%}
+    {%- set cluster_by_keys = config.get('cluster_by', default=none) -%}
+
+    {%- if cluster_by_keys is not none and cluster_by_keys is string -%}
+        {%- set cluster_by_keys = [cluster_by_keys] -%}
+    {%- endif -%}
+    {%- if cluster_by_keys is not none -%}
+        {%- set cluster_by_string = cluster_by_keys|join(", ")-%}
+    {% else %}
+        {%- set cluster_by_string = none -%}
+    {%- endif -%}
+
+    {% if cluster_by_string is not none and not temporary -%}
+        ALTER MATERIALIZED VIEW {{relation}} CLUSTER BY ({{cluster_by_string}});
+    {%- endif -%}
+
+{% endmacro %}

macros/materialized-views/snowflake/create_materialized_view_as.sql

@@ -1,29 +1,11 @@
 {% macro create_materialized_view_as(relation, sql, config) -%}
     {%- set secure = config.get('secure', default=false) -%}
-    {%- set cluster_by_keys = config.get('cluster_by', default=none) -%}
-    {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%}
-
-    {%- if cluster_by_keys is not none and cluster_by_keys is string -%}
-        {%- set cluster_by_keys = [cluster_by_keys] -%}
-    {%- endif -%}
-    {%- if cluster_by_keys is not none -%}
-        {%- set cluster_by_string = cluster_by_keys|join(", ")-%}
-    {% else %}
-        {%- set cluster_by_string = none -%}
-    {%- endif -%}
 
     CREATE OR REPLACE
-        {% if secure -%} SECURE {%- endif %} 
+        {% if secure -%} SECURE {%- endif %}
         MATERIALIZED VIEW {{relation}}
     AS (
         {{sql}}
     );
-    
-    {% if cluster_by_string is not none and not temporary -%}
-        ALTER MATERIALIZED VIEW {{relation}} CLUSTER BY ({{cluster_by_string}});
-    {%- endif -%}
-    {% if enable_automatic_clustering and cluster_by_string is not none and not temporary  -%}
-        ALTER MATERIALIZED VIEW {{relation}} RESUME RECLUSTER;
-    {%- endif -%}
 
 {% endmacro %}

macros/materialized-views/snowflake/enable_automatic_clustering.sql

@@ -0,0 +1,17 @@
+{% macro enable_automatic_clustering(relation, config) -%}
+    {%- set cluster_by_keys = config.get('cluster_by', default=none) -%}
+    {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%}
+
+    {%- if cluster_by_keys is not none and cluster_by_keys is string -%}
+        {%- set cluster_by_keys = [cluster_by_keys] -%}
+    {%- endif -%}
+    {%- if cluster_by_keys is not none -%}
+        {%- set cluster_by_string = cluster_by_keys|join(", ")-%}
+    {% else %}
+        {%- set cluster_by_string = none -%}
+    {%- endif -%}
+
+    {% if enable_automatic_clustering and cluster_by_string is not none and not temporary  -%}
+        ALTER MATERIALIZED VIEW {{relation}} RESUME RECLUSTER;
+    {%- endif -%}
+{% endmacro %}

macros/materialized-views/snowflake/materialized_view.sql

@@ -5,26 +5,33 @@
   {% set full_refresh_mode = (should_full_refresh()) %}
 
   {% set target_relation = this %}
-  {% set existing_relation = load_relation(this) %}
-  {% set tmp_relation = make_temp_relation(this) %}
+  {%- set existing_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+  {% set statements = []%}
 
   {{ run_hooks(pre_hooks) }}
+  {% if existing_relation.is_table %}
+       {{ log("Dropping relation " ~ target_relation ~ " because it is a " ~ existing_relation.type ~ " and this model is a materialized view.") }}
+       {% do adapter.drop_relation(existing_relation) %}
+  {% endif %}
 
-  {% if (existing_relation is none or full_refresh_mode) %}
-      {% set build_sql = dbt_dataengineers_materializations.create_materialized_view_as(target_relation, sql, config) %}
-  {% elif existing_relation.is_table %}
-      {#-- Can't overwrite a view with a table - we must drop --#}
-      {{ log("Dropping relation " ~ target_relation ~ " because it is a " ~ existing_relation.type ~ " and this model is a materialized view.") }}
-      {% do adapter.drop_relation(existing_relation) %}
-      {% set build_sql = dbt_dataengineers_materializations.create_materialized_view_as(target_relation, sql, config) %}
+  {% if (existing_relation is none or full_refresh_mode or existing_relation.is_table) %}
+      {% do statements.append(dbt_dataengineers_materializations.create_materialized_view_as(target_relation, sql, config)) %}
+      {% do statements.append(dbt_dataengineers_materializations.apply_clusters(target_relation, config)) %}
+      {% do statements.append(dbt_dataengineers_materializations.enable_automatic_clustering(target_relation, config)) %}
   {% else %}
       {# noop #}
   {% endif %}
-  
-  {% if build_sql %}
-      {% call statement("main") %}
-          {{ build_sql }}
-      {% endcall %}
+
+  {% if statements | length > 0 %}
+    {% for statement in statements %}
+        {% set statement = statement | trim %}
+        {% if statement == '' %}
+            {% continue %}
+        {% endif %}
+        {% call statement("main") %}
+            {{ statement }}
+        {% endcall %}
+    {% endfor %}
   {% else %}
     {{ store_result('main', 'SKIP') }}
   {% endif %}