Merge pull request #865 from elementary-data/ele-5095-dremio-duplication-in-synced-schema

Dremio: Formatting timestamp column

Author: elazarlachkar

macros/commands/dump_table.sql

@@ -21,9 +21,10 @@
     {% set order_by_dedup_column = "generated_at" %}
     {% set query %}
         {% if dedup and (dedup_by_column in column_names) and (order_by_dedup_column in column_names) %}
-            {{ elementary.dedup_by_column_query(dedup_by_column, order_by_dedup_column, column_names, relation) }}
+            {{ elementary.dedup_by_column_query(dedup_by_column, order_by_dedup_column, column_names, relation, timestamp_column=timestamp_column) }}
         {% else %}
-            select {{ elementary.escape_select(column_names) }} from {{ relation }}
+            select {{ elementary.select_columns(column_names, timestamp_column) }}
+            from {{ relation }}
         {% endif %}
         {% if timestamp_column %}
             {% if since %}
@@ -49,7 +50,7 @@
 {% endmacro %}
 
 
-{% macro dedup_by_column_query(dedup_by_column, order_by_dedup_column, column_names, relation) %}
+{% macro dedup_by_column_query(dedup_by_column, order_by_dedup_column, column_names, relation, timestamp_column=none) %}
     with indexed_relation as (
         select 
             {{ elementary.escape_select(column_names) }}, 
@@ -63,6 +64,6 @@
         where row_index = 1
     )
 
-    select {{ elementary.escape_select(column_names) }}
+    select {{ elementary.select_columns(column_names, timestamp_column) }}
     from deduped_relation
 {% endmacro %}

macros/utils/sql_utils/select_columns.sql

@@ -0,0 +1,14 @@
+{% macro select_columns(column_names, timestamp_column=none) %}
+    {%- set processed_columns = [] -%}
+    {%- set timestamp_column_lower = timestamp_column | lower if timestamp_column else none -%}
+    
+    {%- for column_name in column_names -%}
+        {%- if timestamp_column and column_name | lower == timestamp_column_lower -%}
+            {%- do processed_columns.append(elementary.select_timestamp_column(column_name)) -%}
+        {%- else -%}
+            {%- do processed_columns.append(elementary.escape_select([column_name])) -%}
+        {%- endif -%}
+    {%- endfor -%}
+
+    {{ processed_columns | join(", ") }}
+{% endmacro %}

macros/utils/sql_utils/select_timestamp_column.sql

@@ -0,0 +1,16 @@
+{% macro select_timestamp_column(column_name) %}
+    {{ return(adapter.dispatch('select_timestamp_column', 'elementary')(column_name)) }}
+{% endmacro %}
+
+{% macro default__select_timestamp_column(column_name) %}
+    {{ return(elementary.escape_select([column_name])) }}
+{% endmacro %}
+
+{% macro dremio__select_timestamp_column(column_name) %}
+    {# 
+        Dremio truncates milliseconds when selecting timestamps. 
+        Using TO_CHAR with FFF format preserves them. 
+    #}
+    {% set escaped_name = elementary.escape_select([column_name]) %}
+    {{ return('TO_CHAR(' ~ escaped_name ~ ', \'YYYY-MM-DD"T"HH24:MI:SS.FFF\') as ' ~ escaped_name) }}
+{% endmacro %}