Merge pull request #10 from TechArtists/multiple-project-source

rename + workaround incremental

Author: SorinSV

analyses/measure_dimension_rollup_explosion.sql

@@ -13,7 +13,7 @@
 {%- set miniColumnsToIgnoreInGroupBy = miniColumnsToIgnoreInGroupBy + ta_firebase.list_map_and_add_prefix(timezones, 'event_dates.') + ta_firebase.list_map_and_add_prefix(timezones, 'install_dates.') -%}
 
 
-{%- set tmp_res = ta_firebase.get_filtered_columns_for_table("fb_analytics_events_raw", columnNamesEventDimensions, miniColumnsToIgnoreInGroupBy) -%}
+{%- set tmp_res = ta_firebase.get_filtered_columns_for_table("google_analytics_events_raw", columnNamesEventDimensions, miniColumnsToIgnoreInGroupBy) -%}
 {%- set columnsForEventDimensions = tmp_res[0] -%}
 
 
@@ -22,7 +22,7 @@
 
 WITH
 {%- for column in minicolumns -%}
-    {{ ", " if not loop.first else "" }} dim_{{loop.index}} AS ( SELECT COUNT(DISTINCT({{ column[0] }})) AS dist_cnt FROM  {{ ref("fb_analytics_events_raw") }} WHERE DATE(event_date) = '2023-10-10')
+    {{ ", " if not loop.first else "" }} dim_{{loop.index}} AS ( SELECT COUNT(DISTINCT({{ column[0] }})) AS dist_cnt FROM  {{ ref("google_analytics_events_raw") }} WHERE DATE(event_date) = '2023-10-10')
     {% set _ = unionAllSelects.append("SELECT '" ~ column[1]  ~ "' AS dim_name, dist_cnt FROM dim_" ~ loop.index) -%}
 {% endfor -%}
 

dbt_project.yml

@@ -46,7 +46,7 @@ vars:
     #     events_table: events_*,
     #     crashlytics_dataset_id: firebase_crashlytics,
     #     crashlytics_table: com_dave_logomaker_*}
-    
+    # "TA:SQLMESH_LOOKBACK"
     # don't define variables here, because if they aren't overwritten in the client yml, then they will resolve to "None". 
     # Instead, use them in the package where you need them with them with the appropriate default value
     # Furthermore, whatever we use in sources.yml, we can't define in this package's dbt_project.yml, or else they won't be able to be overwritten by the client

macros/dateFilters.sql

@@ -1,6 +1,7 @@
 {%- macro analyticsDateFilterFor(dateField, extend = 0) -%}
   {%- if sqlmesh_incremental is defined -%}
-    {{ dateField }} BETWEEN '{{ start_ds }}' AND '{{ end_ds }}'
+    {%- set INCREMENTAL_DAYS =  var('TA:SQLMESH_LOOKBACK', 5 ) +extend  -%}
+    {{ dateField }} BETWEEN '{{ start_ds }}' - {{ INCREMENTAL_DAYS }} AND '{{ end_ds }}'
   {%- else -%}
     {%- set startEndTSTuple = ta_firebase.analyticsStartEndTimestampsTuple(extend = extend) -%}
     {{ dateField }} BETWEEN DATE({{ startEndTSTuple[0] }}) AND DATE({{ startEndTSTuple[1] }})
@@ -9,7 +10,8 @@
 
 {%- macro crashlyticsDateFilterFor(dateField, extend = 0) -%}
   {%- if sqlmesh_incremental is defined -%}
-    {{ dateField }} BETWEEN '{{ start_ds }}' AND '{{ end_ds }}'
+    {%- set INCREMENTAL_DAYS =  var('TA:SQLMESH_LOOKBACK', 5 ) +extend  -%}
+    {{ dateField }} BETWEEN '{{ start_ds }}' - {{ INCREMENTAL_DAYS }} AND '{{ end_ds }}'
   {%- else -%}
     {%- set startEndTSTuple = ta_firebase.crashlyticsStartEndTimestampsTuple(extend=extend) -%}
     {{ dateField }} BETWEEN DATE({{ startEndTSTuple[0] }}) AND DATE({{ startEndTSTuple[1] }})
@@ -18,7 +20,8 @@
 
 {%- macro analyticsTSFilterFor(tsField, extend = 0) -%}
   {%- if sqlmesh_incremental is defined -%}
-    {{ tsField }} BETWEEN '{{ start_ds }}' AND '{{ end_ds }}'
+    {%- set INCREMENTAL_DAYS =  var('TA:SQLMESH_LOOKBACK', 5 ) +extend  -%}
+    {{ tsField }} BETWEEN TIMESTAMP('{{ start_ds }}' - {{ INCREMENTAL_DAYS }}) AND TIMESTAMP('{{ end_ds }}')
   {%- else -%}
     {%- set startEndTSTuple = ta_firebase.analyticsStartEndTimestampsTuple(extend=extend) -%}
     {{ tsField }} BETWEEN {{ startEndTSTuple[0] }} AND {{ startEndTSTuple[1] }}
@@ -27,7 +30,8 @@
 
 {%- macro crashlyticsTSFilterFor(tsField, extend = 0) -%}
   {%- if sqlmesh_incremental is defined -%}
-    {{ tsField }}  BETWEEN '{{ start_ds }}' AND '{{ end_ds }}'
+    {%- set INCREMENTAL_DAYS =  var('TA:SQLMESH_LOOKBACK', 5 ) +extend  -%}
+    {{ tsField }}  BETWEEN TIMESTAMP('{{ start_ds }}' - {{ INCREMENTAL_DAYS }}) AND TIMESTAMP('{{ end_ds }}')
   {%- else -%}
     {%- set startEndTSTuple = ta_firebase.crashlyticsStartEndTimestampsTuple(extend=extend) -%}
     {{ tsField }} BETWEEN {{ startEndTSTuple[0] }} AND {{ startEndTSTuple[1] }}
@@ -36,8 +40,9 @@
 
 {%- macro analyticsTableSuffixFilter(extend = 0) -%}
   {%- if sqlmesh_incremental is defined -%}
+    {%- set INCREMENTAL_DAYS =  var('TA:SQLMESH_LOOKBACK', 5 ) +extend  -%}
     REPLACE(_TABLE_SUFFIX, 'intraday_', '')
-      BETWEEN FORMAT_DATE('%Y%m%d', '{{ start_ds }}' -1)
+      BETWEEN FORMAT_DATE('%Y%m%d', '{{ start_ds }}' -1 - {{ INCREMENTAL_DAYS }} )
           AND FORMAT_DATE('%Y%m%d', '{{ end_ds }}')
   {%- else -%}
     {%- set startEndTSTuple = ta_firebase.analyticsStartEndTimestampsTuple(extend=extend + 1) -%}

macros/ta_mandatory_vars.sql

@@ -9,7 +9,7 @@
 
 {% macro verify_all_ta_mandatory_variables() -%}
 {{- ta_firebase.compile_time_mandatory_var("TA:SOURCES", "ta") -}}
-{{- ta_firebase.compile_time_mandatory_var("TA:SOURCES_READY", "firebase_analytics_raw_test") -}}
+{{- ta_firebase.compile_time_mandatory_var("TA:SOURCES_MULTIPLE_PROJECTS_GENERATED", "firebase_analytics_raw_test") -}}
 {{- ta_firebase.compile_time_mandatory_var("TA:FIREBASE_ANALYTICS_FULL_REFRESH_START_DATE", "2018-01-01") -}}
 {{- ta_firebase.compile_time_mandatory_var("TA:FIREBASE_CRASHLYTICS_FULL_REFRESH_START_DATE", "2018-01-01") -}}
 

models/crashlytics/crashlytics_events.sql

@@ -16,7 +16,7 @@
 ] -%}
 
 {%- set miniColumnsToIgnoreInGroupBy = [] -%}
-{%- set tmp_res = ta_firebase.get_filtered_columns_for_table("fb_crashlytics_events_raw", columnNamesEventDimensions, miniColumnsToIgnoreInGroupBy) -%}
+{%- set tmp_res = ta_firebase.get_filtered_columns_for_table("crashlytics_events_raw", columnNamesEventDimensions, miniColumnsToIgnoreInGroupBy) -%}
 {%- set columnsForEventDimensions = tmp_res[0] -%}
 {%- set eventDimensionsUnnestedCount = tmp_res[1]  -%}
 
@@ -25,7 +25,7 @@ WITH data as (
             , {{ ta_firebase.unpack_columns_into_minicolumns(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, [], "", "") }}
             , COUNT(1) as cnt -- same as COUNT(DISTINCT(event_id))
             , COUNT(DISTINCT(crashlytics_user_pseudo_id)) as users
-    FROM {{ ref("fb_crashlytics_events_raw") }}
+    FROM {{ ref("crashlytics_events_raw") }}
     WHERE {{ ta_firebase.analyticsDateFilterFor('event_date') }}
     GROUP BY 1 {% for n in range(2, 2 + eventDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
 )

models/crashlytics/crashlytics_events_raw.sql

@@ -109,8 +109,9 @@
 
             SELECT
                 '{{ pid }}' AS project_id,
-                '{{ ds }}'  AS dataset_id,
-                 DATE(event_timestamp) as event_date
+                '{{ ds }}'  AS dataset_id
+    , event_timestamp as event_ts
+    , DATE(event_timestamp) as event_date
     , received_timestamp as received_ts
     , installation_uuid as crashlytics_user_pseudo_id
     , (SELECT value FROM UNNEST(custom_keys) WHERE key = 'fb_user_pseudo_id') as firebase_analytics_user_pseudo_id

tests/analytics_installs_unique.sql

@@ -1,6 +1,6 @@
 WITH counts AS (
 SELECT COUNT(DISTINCT user_pseudo_id) AS dist_cnt,count(1) AS cnt
-FROM {{ref('fb_analytics_installs_raw')}}
+FROM {{ref('google_analytics_installs_raw')}}
 WHERE {{ ta_firebase.analyticsTestDateFilter('event_date',extend = 2) }}
 AND event_date >='2024-08-22' --date when incrementel model was fixed
 AND user_pseudo_id is not null

tests/events_raw_incremental.sql

@@ -5,7 +5,7 @@ WITH stg AS (
     event_date,
     project_id,
     SUM(duplicates_cnt) AS cnt
-  FROM {{ ref('fb_analytics_events_raw') }}
+  FROM {{ ref('google_analytics_events_raw') }}
   WHERE {{ ta_firebase.analyticsTestDateFilter('event_date', extend=2) }}
     AND event_date <= CURRENT_DATE() - 5
   GROUP BY 1, 2
@@ -47,7 +47,7 @@ src AS (
 
       {%- if ads_list | length == 0 -%}
         {# Naming style: firebase_analytics__<pid> #}
-        {%- if not ns.first %}UNION ALL{% endif -%}
+        {% if not ns.first %} UNION ALL {% endif %}
         {%- set ns.first = false -%}
         SELECT
           DATE(TIMESTAMP_MICROS(event_timestamp)) AS event_date,
@@ -62,7 +62,7 @@ src AS (
       {%- else -%}
         {# Naming style: firebase_analytics__<pid>__<dataset> for each dataset #}
         {%- for ds in ads_list -%}
-          {%- if not ns.first %}UNION ALL{% endif -%}
+          {% if not ns.first %} UNION ALL {% endif %}
           {%- set ns.first = false -%}
           SELECT
             DATE(TIMESTAMP_MICROS(event_timestamp)) AS event_date,