Merge pull request #6 from TechArtists/multiple-project-source

SorinSV · web-flow · commit d906e6b65dcf · 2025-11-11T01:44:23.000+02:00
Multiple project source
diff --git a/analyses/measure_dimension_rollup_explosion.sql b/analyses/measure_dimension_rollup_explosion.sql
@@ -5,19 +5,19 @@
 ] -%}
 
 
-{%- set miniColumnsToIgnoreInGroupBy = ta_firebase..get_mini_columns_to_ignore_when_rolling_up() -%}
+{%- set miniColumnsToIgnoreInGroupBy = ta_firebase.get_mini_columns_to_ignore_when_rolling_up() -%}
 
 {# Ignore all time zones except the first & last (they're all the same, just save the computational effort) #}
-{%- set timezones = ta_firebase..generate_date_timezone_age_struct('dont care') | map(attribute=0) | list -%}
+{%- set timezones = ta_firebase.generate_date_timezone_age_struct('dont care') | map(attribute=0) | list -%}
 {%- set timezones = timezones[1:-1] -%}
-{%- set miniColumnsToIgnoreInGroupBy = miniColumnsToIgnoreInGroupBy + ta_firebase..list_map_and_add_prefix(timezones, 'event_dates.') + ta_firebase..list_map_and_add_prefix(timezones, 'install_dates.') -%}
+{%- set miniColumnsToIgnoreInGroupBy = miniColumnsToIgnoreInGroupBy + ta_firebase.list_map_and_add_prefix(timezones, 'event_dates.') + ta_firebase.list_map_and_add_prefix(timezones, 'install_dates.') -%}
 
 
-{%- set tmp_res = ta_firebase..get_filtered_columns_for_table("fb_analytics_events_raw", columnNamesEventDimensions, miniColumnsToIgnoreInGroupBy) -%}
+{%- set tmp_res = ta_firebase.get_filtered_columns_for_table("fb_analytics_events_raw", columnNamesEventDimensions, miniColumnsToIgnoreInGroupBy) -%}
 {%- set columnsForEventDimensions = tmp_res[0] -%}
 
 
-{%- set minicolumns = ta_firebase..unpack_columns_into_minicolumns_array(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, [], "", "") -%}
+{%- set minicolumns = ta_firebase.unpack_columns_into_minicolumns_array(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, [], "", "") -%}
 {%- set unionAllSelects = [] -%}
 
 WITH
diff --git a/macros/generate_struct_parameters_for_raw.sql b/macros/generate_struct_parameters_for_raw.sql
@@ -1,18 +1,18 @@
 {% macro generate_struct_for_raw_user_properties() -%}
-    {{ overbase_firebase.generate_struct(overbase_firebase.get_user_property_tuples_all(), 'user_properties') }}
+    {{ ta_firebase.generate_struct(ta_firebase.get_user_property_tuples_all(), 'user_properties') }}
 {%- endmacro%}
 
 {% macro generate_struct_for_raw_event_parameters() -%}
-    {{ overbase_firebase.generate_struct(overbase_firebase.get_event_parameter_tuples_for_raw(), 'event_params') }}
+    {{ ta_firebase.generate_struct(ta_firebase.get_event_parameter_tuples_for_raw(), 'event_params') }}
 {%- endmacro%}
 
 {% macro generate_struct_for_raw_crashlytics_custom_keys() -%}
-    {{ overbase_firebase.generate_struct(overbase_firebase.get_crashlytics_custom_key_tuples_all(), 'custom_keys') }}
+    {{ ta_firebase.generate_struct(ta_firebase.get_crashlytics_custom_key_tuples_all(), 'custom_keys') }}
 {%- endmacro%}
 
 
 {% macro generate_struct(all_parameters, firebase_record_name) -%}
-    {{ return(adapter.dispatch('generate_struct', 'overbase_firebase')(all_parameters, firebase_record_name) ) }}
+    {{ return(adapter.dispatch('generate_struct', 'ta_firebase')(all_parameters, firebase_record_name) ) }}
 {%- endmacro %}
 
 
@@ -33,7 +33,7 @@
     {% for parameter in all_parameters -%}
         {%- set key_name = parameter['key_name'] -%}
         {%- set extract_transformation = parameter['extract_transformation'] -%}
-        {%- set event_name_condition = overbase_firebase.makeListIntoSQLInFilter("event_name", parameter['event_name_filter']) -%}
+        {%- set event_name_condition = ta_firebase.makeListIntoSQLInFilter("event_name", parameter['event_name_filter']) -%}
         {%- set _ = structValues.append("(SELECT IF(" ~ event_name_condition ~ "," ~ extract_transformation ~ ",NULL) FROM  UNNEST(" ~ firebase_record_name ~ ") WHERE key = '" ~ key_name + "')") -%}
     {%- endfor -%}
     {%- if structFieldNames|length > 0 -%}
diff --git a/macros/ta_mandatory_vars.sql b/macros/ta_mandatory_vars.sql
@@ -2,7 +2,7 @@
 -- the default value is meant only to be used within this project, as test values 
 {% macro compile_time_mandatory_var(variable_name, default_value_in_ta) -%}
 {% if var(variable_name, default_value_in_ta) == default_value_in_ta and env_var('DBT_INSIDE_TA','')|length == 0 -%}
-   {{ exceptions.raise_compiler_error("Variable '%s' must be overwritten inside your dbt_project.yml. Check the Overbase documentation to see all mandatory variables" % variable_name) }}
+   {{ exceptions.raise_compiler_error("Variable '%s' must be overwritten inside your dbt_project.yml. Check the TA documentation to see all mandatory variables" % variable_name) }}
 {% else %}
 {%- endif %}
 {%- endmacro %}
diff --git a/models/analytics/fb_analytics_ui_wid.sql b/models/analytics/fb_analytics_ui_wid.sql
@@ -13,12 +13,12 @@
 
 SELECT *
 FROM {{ ref("fb_analytics_events_wid") }}
-WHERE event_name IN ('ta_ui_view_shown', 'ta_ui_button_tapped', 'ui_view_shown', 'ui_button_tapped')
+WHERE event_name IN ('ta_ui_view_show', 'ta_ui_button_tapped', 'ui_view_show', 'ui_button_tapped')
 AND {{ ta_firebase.analyticsDateFilterFor('event_date') }}
 
 UNION ALL 
 
 SELECT *
 FROM {{ ref("fb_analytics_events_wid_forced_nulls") }}
-WHERE event_name IN ('ta_ui_view_shown', 'ta_ui_button_tapped', 'ui_view_shown', 'ui_button_tapped')
+WHERE event_name IN ('ta_ui_view_show', 'ta_ui_button_tapped', 'ui_view_show', 'ui_button_tapped')
 AND {{ ta_firebase.analyticsDateFilterFor('event_date') }}
diff --git a/models/app_health/app_health.sql b/models/app_health/app_health.sql
@@ -13,19 +13,19 @@
 {# mini_measures": ["cnt", "users"] are implicit in all of them, as the default #}
 {%- set builtinMeasures = [
   {"model": "analytics", "name":"user_engagement"  , "agg": "SUM(##)", "event_name": "user_engagement"},
-  {"model": "analytics", "name":"ob_app_foreground", "agg": "SUM(##)", "event_name": "ob_app_foreground"},
-  {"model": "analytics", "name":"ob_app_background", "agg": "SUM(##)", "event_name": "ob_app_background"},
+  {"model": "analytics", "name":"ta_app_foreground", "agg": "SUM(##)", "event_name": "ta_app_foreground"},
+  {"model": "analytics", "name":"ta_app_background", "agg": "SUM(##)", "event_name": "ta_app_background"},
   {"model": "analytics", "name":"app_update"   , "agg": "SUM(##)", "event_name": "app_update"},
-  {"model": "analytics", "name":"ob_app_update", "agg": "SUM(##)", "event_name": "ob_app_update"},
+  {"model": "analytics", "name":"ta_app_update", "agg": "SUM(##)", "event_name": "ta_app_update"},
   {"model": "analytics", "name":"errors"  , "agg": "SUM(##)", "event_name": "LIKE 'error_%'"},
-  {"model": "analytics", "name":"ob_errors"  , "agg": "SUM(##)", "event_name": "LIKE 'ob_error_%'"},
+  {"model": "analytics", "name":"ta_errors"  , "agg": "SUM(##)", "event_name": "LIKE 'ta_error_%'"},
 
   {"model": "crashlytics", "name":"crashlytics_all_errors", "agg": "SUM(##)"},
   {"model": "crashlytics", "name":"fatal_crashes", "agg": "SUM(##)", "additional_filter": "error_type = 'FATAL'"},
   {"model": "crashlytics", "name":"fatal_foreground_crashes", "agg": "SUM(##)", "additional_filter": "error_type = 'FATAL' AND process_state = 'FOREGROUND'"},
   {"model": "crashlytics", "name":"fatal_background_crashes", "agg": "SUM(##)", "additional_filter": "error_type = 'FATAL' AND process_state = 'BACKGROUND' "}
 ] %}
-{%- set allUnprocessedHealthMeasures = builtinMeasures + var("OVERBASE:CUSTOM_APP_HEALTH_MEASURES", []) %}
+{%- set allUnprocessedHealthMeasures = builtinMeasures + var("TA:CUSTOM_APP_HEALTH_MEASURES", []) %}
 {%- set allAnalyticsEventNames = set([]) -%}
 {%- set allAnalyticsForcedNullEventNames = set([]) -%}
 {%- for customHealthMeasure in allUnprocessedHealthMeasures -%}
@@ -74,8 +74,8 @@ WITH analytics AS (
            {% endfor -%}
           , {{ custom_summed_measures | selectattr("model", "equalto", "analytics") | map(attribute='agg')|join("\n          , ") }}
     FROM {{ ref("fb_analytics_events") }}
-    WHERE {{ overbase_firebase.analyticsDateFilterFor('event_date') }}
-    AND {{ overbase_firebase.makeListIntoSQLInFilter("event_name", allAnalyticsEventNames| list) }}
+    WHERE {{ ta_firebase.analyticsDateFilterFor('event_date') }}
+    AND {{ ta_firebase.makeListIntoSQLInFilter("event_name", allAnalyticsEventNames| list) }}
     GROUP BY {{ range(1, 1 + commonDimensionsAndAliases | length) | list | join(",") }} 
 )
 , analyticsForcedNulls AS (
@@ -86,11 +86,11 @@ WITH analytics AS (
             , {{ measure }}
           {%- endfor %}
     FROM {{ ref("fb_analytics_events_forced_nulls") }}
-    WHERE {{ overbase_firebase.analyticsDateFilterFor('event_date') }}
+    WHERE {{ ta_firebase.analyticsDateFilterFor('event_date') }}
     {% if allAnalyticsForcedNullEventNames | length == 0 -%}
     AND False
     {%- else -%}
-    AND {{ overbase_firebase.makeListIntoSQLInFilter("event_name", allAnalyticsForcedNullEventNames| list) }}
+    AND {{ ta_firebase.makeListIntoSQLInFilter("event_name", allAnalyticsForcedNullEventNames| list) }}
     {%- endif %}
     GROUP BY {{ range(1, 1 + commonDimensionsAndAliases | length) | list | join(",") }} 
 )
@@ -100,7 +100,7 @@ WITH analytics AS (
            {% endfor -%}
           , SUM(users) as users
     FROM {{ ref("fb_analytics_installs") }}
-    WHERE {{ overbase_firebase.analyticsDateFilterFor('event_date') }}
+    WHERE {{ ta_firebase.analyticsDateFilterFor('event_date') }}
     GROUP BY {{ range(1, 1 + commonDimensionsAndAliases | length) | list | join(",") }} 
 )
 , crashlytics AS (
@@ -109,7 +109,7 @@ WITH analytics AS (
            {% endfor -%}
             , {{ custom_summed_measures | selectattr("model", "equalto", "crashlytics") | map(attribute='agg')|join("\n          , ") }}
     FROM {{ ref("fb_crashlytics_events") }}
-    WHERE {{ overbase_firebase.crashlyticsDateFilterFor('event_date') }}
+    WHERE {{ ta_firebase.crashlyticsDateFilterFor('event_date') }}
     GROUP BY {{ range(1, 1 + commonDimensionsAndAliases | length) | list | join(",") }} 
 )
 , joined_unpacked AS (
@@ -136,13 +136,13 @@ SELECT  event_date
       , platform 
       , app_id
       , reverse_app_id
-      , {{ overbase_firebase.get_version_record_from_normalized('app_version_join_value') }} AS app_version
-      , {{ overbase_firebase.get_version_record_from_normalized('platform_version_join_value') }} AS platform_version
+      , {{ ta_firebase.get_version_record_from_normalized('app_version_join_value') }} AS app_version
+      , {{ ta_firebase.get_version_record_from_normalized('platform_version_join_value') }} AS platform_version
       , STRUCT<type STRING, manufacturer STRING, os_model STRING>(
          device_hardware_type, device_hardware_manufacturer, device_hardware_os_model
       ) as device_hardware
       , installs
-      , {{ overbase_firebase.list_map_and_add_prefix(custom_summed_measures | map(attribute='alias')) | join("\n          ,") }}
+      , {{ ta_firebase.list_map_and_add_prefix(custom_summed_measures | map(attribute='alias')) | join("\n          ,") }}
      
 FROM joined_unpacked
 
diff --git a/models/crashlytics/fb_crashlytics_events_raw.sql b/models/crashlytics/fb_crashlytics_events_raw.sql
@@ -1,4 +1,4 @@
-{{ overbase_firebase.verify_all_overbase_mandatory_variables() }}
+{{ ta_firebase.verify_all_ta_mandatory_variables() }}
 
 {{ config(
     materialized='incremental',
@@ -13,8 +13,8 @@
 
 -- https://firebase.google.com/docs/crashlytics/bigquery-export#without_stack_traces
 
-{% set projects = var('OVERBASE:SOURCES', []) %}
-{% set ready = var('OVERBASE:SOURCES_READY', false) %}
+{% set projects = var('TA:SOURCES', []) %}
+{% set ready = var('TA:SOURCES_READY', false) %}
 
 {% set first = projects[0] if projects and (projects[0] is mapping) else {} %}
 {% set pid0 = first.get('project_id', 'fallback_project') %}
@@ -51,17 +51,17 @@
           ) as orientation
         , STRUCT<firebase_value STRING, build_no STRING, major INT64, minor INT64, bugfix INT64, major_minor FLOAT64, major_minor_bugfix STRING, normalized INT64, join_value STRING>(
             {%- set v = "application.display_version" -%}
-            {{ v }}, application.build_version, {{ overbase_firebase.get_version(v, "major") }}, {{ overbase_firebase.get_version(v, "minor") }}, {{ overbase_firebase.get_version(v, "bugfix") }}, {{ overbase_firebase.get_version(v, "major.minor") }}, {{ overbase_firebase.get_version(v, "major.minor.bugfix") }}, {{ overbase_firebase.get_version(v, "normalized") }}, COALESCE(CAST({{ overbase_firebase.get_version(v, "normalized") }} AS STRING), {{ v }} )
+            {{ v }}, application.build_version, {{ ta_firebase.get_version(v, "major") }}, {{ ta_firebase.get_version(v, "minor") }}, {{ ta_firebase.get_version(v, "bugfix") }}, {{ ta_firebase.get_version(v, "major.minor") }}, {{ ta_firebase.get_version(v, "major.minor.bugfix") }}, {{ ta_firebase.get_version(v, "normalized") }}, COALESCE(CAST({{ ta_firebase.get_version(v, "normalized") }} AS STRING), {{ v }} )
         ) AS app_version
         , STRUCT<firebase_value STRING, name STRING, major INT64, minor INT64, bugfix INT64, major_minor FLOAT64, major_minor_bugfix STRING, normalized INT64, join_value STRING>(
             {%- set v = "operating_system.display_version" -%}
-            {{ v }}, operating_system.name, {{ overbase_firebase.get_version(v, "major") }}, {{ overbase_firebase.get_version(v, "minor") }}, {{ overbase_firebase.get_version(v, "bugfix") }}, {{ overbase_firebase.get_version(v, "major.minor") }}, {{ overbase_firebase.get_version(v, "major.minor.bugfix") }}, {{ overbase_firebase.get_version(v, "normalized") }}, COALESCE(CAST( {{ overbase_firebase.get_version(v, "normalized") }} AS STRING), {{ v }} )
+            {{ v }}, operating_system.name, {{ ta_firebase.get_version(v, "major") }}, {{ ta_firebase.get_version(v, "minor") }}, {{ ta_firebase.get_version(v, "bugfix") }}, {{ ta_firebase.get_version(v, "major.minor") }}, {{ ta_firebase.get_version(v, "major.minor.bugfix") }}, {{ ta_firebase.get_version(v, "normalized") }}, COALESCE(CAST( {{ ta_firebase.get_version(v, "normalized") }} AS STRING), {{ v }} )
         ) AS platform_version
         , operating_system.modification_state as jailbroken_state
         , STRUCT<type STRING, manufacturer STRING, os_model STRING, architecture STRING>(
             LOWER(operating_system.device_type), LOWER(device.manufacturer), LOWER(device.model), device.architecture 
         ) AS device_hardware
-        , {{ overbase_firebase.generate_struct_for_raw_crashlytics_custom_keys() }} as custom_keys
+        , {{ ta_firebase.generate_struct_for_raw_crashlytics_custom_keys() }} as custom_keys
         , custom_keys as custom_keys_raw
         , STRUCT<used_bytes INT64, free_bytes INT64>(memory.used, memory.free) as memory
         , STRUCT<used_bytes INT64, free_bytes INT64>(storage.used, storage.free) as storage
@@ -87,14 +87,14 @@
              + screen_refresh_rate
             Those values are NULLed for the time being
           #}
-          {{ overbase_firebase.list_map_and_add_prefix([
+          {{ ta_firebase.list_map_and_add_prefix([
             "unity_version","debug_build","processor_type","processor_count",none,none,none,"graphics_device_id","graphics_device_vendor_id","graphics_device_name","graphics_device_vendor","graphics_device_version","graphics_device_type","graphics_shader_level","graphics_render_target_count","graphics_copy_texture_support","graphics_max_texture_size",none,none,none,"processor_frequency_mhz","system_memory_size_mb","graphics_memory_size_mb","screen_size_px","screen_refresh_rate_hz","screen_resolution_dpi"
             ], "unity_metadata." )| join(", ") }}
           ) AS unity_metadata
         , COUNT(1) OVER (PARTITION BY installation_uuid, event_id, variant_id) as duplicates_cnt
 
     FROM {{ source('firebase_crashlytics__fallback', 'events') }}
-    WHERE {{ overbase_firebase.crashlyticsTSFilterFor("event_timestamp") }}
+    WHERE {{ ta_firebase.crashlyticsTSFilterFor("event_timestamp") }}
     QUALIFY ROW_NUMBER() OVER (PARTITION BY crashlytics_user_pseudo_id, event_id, variant_id ORDER BY received_ts) = 1
 
 {% else %}
@@ -133,17 +133,17 @@
           ) as orientation
         , STRUCT<firebase_value STRING, build_no STRING, major INT64, minor INT64, bugfix INT64, major_minor FLOAT64, major_minor_bugfix STRING, normalized INT64, join_value STRING>(
             {%- set v = "application.display_version" -%}
-            {{ v }}, application.build_version, {{ overbase_firebase.get_version(v, "major") }}, {{ overbase_firebase.get_version(v, "minor") }}, {{ overbase_firebase.get_version(v, "bugfix") }}, {{ overbase_firebase.get_version(v, "major.minor") }}, {{ overbase_firebase.get_version(v, "major.minor.bugfix") }}, {{ overbase_firebase.get_version(v, "normalized") }}, COALESCE(CAST({{ overbase_firebase.get_version(v, "normalized") }} AS STRING), {{ v }} )
+            {{ v }}, application.build_version, {{ ta_firebase.get_version(v, "major") }}, {{ ta_firebase.get_version(v, "minor") }}, {{ ta_firebase.get_version(v, "bugfix") }}, {{ ta_firebase.get_version(v, "major.minor") }}, {{ ta_firebase.get_version(v, "major.minor.bugfix") }}, {{ ta_firebase.get_version(v, "normalized") }}, COALESCE(CAST({{ ta_firebase.get_version(v, "normalized") }} AS STRING), {{ v }} )
         ) AS app_version
         , STRUCT<firebase_value STRING, name STRING, major INT64, minor INT64, bugfix INT64, major_minor FLOAT64, major_minor_bugfix STRING, normalized INT64, join_value STRING>(
             {%- set v = "operating_system.display_version" -%}
-            {{ v }}, operating_system.name, {{ overbase_firebase.get_version(v, "major") }}, {{ overbase_firebase.get_version(v, "minor") }}, {{ overbase_firebase.get_version(v, "bugfix") }}, {{ overbase_firebase.get_version(v, "major.minor") }}, {{ overbase_firebase.get_version(v, "major.minor.bugfix") }}, {{ overbase_firebase.get_version(v, "normalized") }}, COALESCE(CAST( {{ overbase_firebase.get_version(v, "normalized") }} AS STRING), {{ v }} )
+            {{ v }}, operating_system.name, {{ ta_firebase.get_version(v, "major") }}, {{ ta_firebase.get_version(v, "minor") }}, {{ ta_firebase.get_version(v, "bugfix") }}, {{ ta_firebase.get_version(v, "major.minor") }}, {{ ta_firebase.get_version(v, "major.minor.bugfix") }}, {{ ta_firebase.get_version(v, "normalized") }}, COALESCE(CAST( {{ ta_firebase.get_version(v, "normalized") }} AS STRING), {{ v }} )
         ) AS platform_version
         , operating_system.modification_state as jailbroken_state
         , STRUCT<type STRING, manufacturer STRING, os_model STRING, architecture STRING>(
             LOWER(operating_system.device_type), LOWER(device.manufacturer), LOWER(device.model), device.architecture 
         ) AS device_hardware
-        , {{ overbase_firebase.generate_struct_for_raw_crashlytics_custom_keys() }} as custom_keys
+        , {{ ta_firebase.generate_struct_for_raw_crashlytics_custom_keys() }} as custom_keys
         , custom_keys as custom_keys_raw
         , STRUCT<used_bytes INT64, free_bytes INT64>(memory.used, memory.free) as memory
         , STRUCT<used_bytes INT64, free_bytes INT64>(storage.used, storage.free) as storage
@@ -169,14 +169,14 @@
              + screen_refresh_rate
             Those values are NULLed for the time being
           #}
-          {{ overbase_firebase.list_map_and_add_prefix([
+          {{ ta_firebase.list_map_and_add_prefix([
             "unity_version","debug_build","processor_type","processor_count",none,none,none,"graphics_device_id","graphics_device_vendor_id","graphics_device_name","graphics_device_vendor","graphics_device_version","graphics_device_type","graphics_shader_level","graphics_render_target_count","graphics_copy_texture_support","graphics_max_texture_size",none,none,none,"processor_frequency_mhz","system_memory_size_mb","graphics_memory_size_mb","screen_size_px","screen_refresh_rate_hz","screen_resolution_dpi"
             ], "unity_metadata." )| join(", ") }}
           ) AS unity_metadata
         , COUNT(1) OVER (PARTITION BY installation_uuid, event_id, variant_id) as duplicates_cnt
 
             FROM {{ source('firebase_crashlytics__' ~ pid, 'events') }}
-            WHERE {{ overbase_firebase.crashlyticsTSFilterFor("event_timestamp") }}
+            WHERE {{ ta_firebase.crashlyticsTSFilterFor("event_timestamp") }}
 
         {% endif %}
     QUALIFY ROW_NUMBER() OVER (PARTITION BY crashlytics_user_pseudo_id, event_id, variant_id ORDER BY received_ts) = 1
