Merge pull request #12 from TechArtists/multiple-project-source

SorinSV · web-flow · commit 15f34f395a08 · 2025-12-19T17:06:53.000+02:00
Multiple project source
diff --git a/models/analytics/google_analytics_events.sql b/models/analytics/google_analytics_events.sql
@@ -7,11 +7,11 @@
      },
     incremental_strategy = 'insert_overwrite',
     require_partition_filter = true,
-    cluster_by = ["event_name", "platform", "app_id"]
+    cluster_by = ["event_name", "platform", "bundle_id"]
 ) }}
 
 
-{%- set columnNamesEventDimensions = ["app_id", "reverse_app_id", "event_name", "platform", "appstore", "app_version", "platform_version",
+{%- set columnNamesEventDimensions = ["bundle_id", "reverse_bundle_id", "event_name", "platform", "appstore", "app_version", "platform_version",
                                 "user_properties", "event_parameters",
                                 "geo", "device_hardware", "device_language", "device_time_zone_offset",
                                 "traffic_source"
@@ -35,6 +35,8 @@
 
 WITH data as (
     SELECT    DATE(event_ts) as event_date
+            , project_id
+            , dataset_id
             , {{ ta_firebase.install_age_group("install_age") }} AS install_age_group
             , {{ ta_firebase.unpack_columns_into_minicolumns(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, miniColumnsToAlsoNil, "", "") }}
             , COUNT(1) as cnt
@@ -43,9 +45,11 @@ WITH data as (
 
     FROM {{ ref("google_analytics_events_raw") }}
     WHERE {{ ta_firebase.analyticsDateFilterFor('event_date') }}
-    GROUP BY 1,2 {% for n in range(3, 3 + eventDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
+    GROUP BY 1,2,3,4 {% for n in range(5, 5 + eventDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
 )
 SELECT event_date
+        , project_id
+        , dataset_id
         , install_age_group
         , {{ ta_firebase.pack_minicolumns_into_structs_for_select(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, "", "") }}
         , cnt
diff --git a/models/analytics/google_analytics_events_count_raw.sql b/models/analytics/google_analytics_events_count_raw.sql
@@ -13,11 +13,13 @@
 
 SELECT
     event_date
+    ,project_id
+    ,dataset_id
     ,platform
     ,user_id
     ,sum(if(event_name = 'user_engagement',1,0)) as user_engagement
 FROM  {{ ref("google_analytics_events_raw") }}
 WHERE {{ ta_firebase.analyticsDateFilterFor('event_date') }}
 AND event_name in ('user_engagement')
-GROUP by 1,2,3
+GROUP by 1,2,3,4,5
 
diff --git a/models/analytics/google_analytics_events_forced_nulls.sql b/models/analytics/google_analytics_events_forced_nulls.sql
@@ -7,11 +7,11 @@
      },
     incremental_strategy = 'insert_overwrite',
     require_partition_filter = true,
-    cluster_by = ["event_name", "platform", "app_id"]
+    cluster_by = ["event_name", "platform", "bundle_id"]
 ) }}
 
 
-{%- set columnNamesEventDimensions = ["app_id", "reverse_app_id", "event_name", "platform", "appstore", "app_version", "platform_version",
+{%- set columnNamesEventDimensions = ["bundle_id", "reverse_bundle_id", "event_name", "platform", "appstore", "app_version", "platform_version",
                                 "user_properties", "event_parameters",
                                 "geo", "device_hardware", "device_language", "device_time_zone_offset",
                                 "traffic_source"
@@ -35,6 +35,8 @@
 
 WITH data as (
     SELECT    DATE(event_ts) as event_date
+            , project_id
+            , dataset_id
             , {{ ta_firebase.install_age_group("install_age") }} AS install_age_group
             , {{ ta_firebase.unpack_columns_into_minicolumns(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, miniColumnsToAlsoNil, "", "") }}
             , COUNT(1) as cnt
@@ -45,10 +47,12 @@ WITH data as (
     WHERE {{ ta_firebase.analyticsDateFilterFor('event_date') }}
     {%- set eventNamesToLookFor = set(ta_firebase.flatten_list_of_lists(ta_firebase.get_event_parameter_tuples_for_rollup_alsoNullDimensions() | map(attribute="force_null_dimension_event_name_filter") | list)) %}
     AND {{ ta_firebase.makeListIntoSQLInFilter("event_name", eventNamesToLookFor| list) }}
-    GROUP BY 1,2 {% for n in range(3, 3 + eventDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
+    GROUP BY 1,2,3,4 {% for n in range(5, 5 + eventDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
 )
 SELECT event_date
         , install_age_group
+        , project_id
+        , dataset_id
         , {{ ta_firebase.pack_minicolumns_into_structs_for_select(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, "", "") }}
         , cnt
         , users
diff --git a/models/analytics/google_analytics_events_raw.sql b/models/analytics/google_analytics_events_raw.sql
@@ -9,7 +9,7 @@
      incremental_strategy = 'insert_overwrite',
      lookback = 4,
      require_partition_filter = true,
-     cluster_by = ["event_name", "platform", "app_id"]
+     cluster_by = ["event_name", "platform", "bundle_id"]
 ) }}
 
 -- https://support.google.com/firebase/answer/7029846
@@ -22,8 +22,8 @@ SELECT
         , {{ ta_firebase.calculate_age_between_timestamps("TIMESTAMP_MICROS(event_timestamp)", "TIMESTAMP_MICROS(user_first_touch_timestamp)") }} as install_age
         , LOWER(user_pseudo_id) as user_pseudo_id
         , LOWER(user_id) as user_id
-        , app_info.id as app_id
-        , ARRAY_TO_STRING(ARRAY_REVERSE(SPLIT(app_info.id, '.')), '.') as reverse_app_id
+        , app_info.id as bundle_id
+        , ARRAY_TO_STRING(ARRAY_REVERSE(SPLIT(app_info.id, '.')), '.') as reverse_bundle_id
         , event_name
         , {{ var( 'TA:CUSTOM_PLATFORM_PREPROCESSOR', 'platform') }} as platform
         , app_info.install_source as appstore
@@ -53,8 +53,8 @@ SELECT
         ) AS device_language
         , IF(device.time_zone_offset_seconds >= 0,'+', '-') || LEFT(CAST(TIME(TIMESTAMP_SECONDS(ABS(device.time_zone_offset_seconds))) AS STRING),5) 
             AS device_time_zone_offset
-        , STRUCT<name STRING, medium STRING, source STRING>(
-            traffic_source.name, traffic_source.medium, traffic_source.source
+        , STRUCT<name STRING, medium STRING, source STRING, manual_campaign_id STRING, manual_campaign_name STRING, manual_source STRING, manual_medium STRING, manual_term STRING, manual_content STRING, manual_source_platform STRING, manual_creative_format STRING, manual_marketing_tactic STRING >(
+            traffic_source.name, traffic_source.medium, traffic_source.source, collected_traffic_source.manual_campaign_id, collected_traffic_source.manual_campaign_name, collected_traffic_source.manual_source, collected_traffic_source.manual_medium, collected_traffic_source.manual_term, collected_traffic_source.manual_content, collected_traffic_source.manual_source_platform, collected_traffic_source.manual_creative_format, collected_traffic_source.manual_marketing_tactic
         ) AS traffic_source
         , STRUCT<revenue FLOAT64, currency STRING>(
             user_ltv.revenue, user_ltv.currency
@@ -121,7 +121,7 @@ FROM
 {%- endif -%}
 ) as events
 LEFT JOIN {{ref('ta_iso_country')}} as country_codes
-    ON LOWER(events.geo.country) = LOWER(country_codes.firebase_name)
+    ON LOWER(events.geo.country) = LOWER(COALESCE(country_codes.firebase_name,country_codes.name))
 LEFT JOIN {{ref("ta_iso_language")}} as language_codes
     ON LOWER(SPLIT(events.device.language,'-')[SAFE_OFFSET(0)]) = language_codes.alpha_2
 LEFT JOIN {{ref('ta_iso_country')}} as language_region_codes -- some language have 3 parts (e.g. zh-hans-us), so just get the last one
diff --git a/models/analytics/google_analytics_events_wid.sql b/models/analytics/google_analytics_events_wid.sql
@@ -7,17 +7,17 @@
      },
     incremental_strategy = 'insert_overwrite',
     require_partition_filter = true,
-    cluster_by = ["event_name", "platform", "app_id"]
+    cluster_by = ["event_name", "platform", "bundle_id"]
 
 ) }}
 
 
-{%- set columnNamesEventDimensions = ["app_id", "reverse_app_id","event_name", "platform", "appstore", "app_version", "platform_version",
+{%- set columnNamesEventDimensions = ["bundle_id", "reverse_bundle_id","event_name", "platform", "appstore", "app_version", "platform_version",
                                 "user_properties", "event_parameters",
                                 "geo", "device_hardware", "device_language", "device_time_zone_offset",
                                 "traffic_source"
 ] -%}
-{%- set columnNamesInstallDimensions = ["app_id", "reverse_app_id", "event_name", "platform", "appstore", "app_version", "platform_version",
+{%- set columnNamesInstallDimensions = ["bundle_id", "reverse_bundle_id", "event_name", "platform", "appstore", "app_version", "platform_version",
                                 "user_properties", "event_parameters",
                                 "geo", "device_hardware", "device_language", "device_time_zone_offset",
                                 "traffic_source"
@@ -49,6 +49,8 @@
 {%- set miniColumnsToAlsoNil = [] -%}
 WITH data as (
     SELECT    DATE(events.event_ts) as event_date
+            , events.project_id
+            , events.dataset_id
             , {{ ta_firebase.unpack_columns_into_minicolumns(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, miniColumnsToAlsoNil ,"events.", "event_") }}
             , DATE(installs.install_ts) as install_date
             , events.install_age as install_age
@@ -64,11 +66,13 @@ WITH data as (
     AND installs.event_date >= '2020-01-01'
     -- TODO: max join on installs ? Have an LTV_MAX variable (e.g. 2 years)
     -- and then join on installs via installs > EVENT_DATE - 2 years?
-    GROUP BY 1,2,3,4 {% for n in range(5, 5 + eventDimensionsUnnestedCount + installedDatesDimensionsUnnestedCount + installDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
+    GROUP BY 1,2,3,4,5,6 {% for n in range(7, 7 + eventDimensionsUnnestedCount + installedDatesDimensionsUnnestedCount + installDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
 )
 
 
 SELECT event_date
+        , project_id
+        , dataset_id
         , {{ ta_firebase.pack_minicolumns_into_structs_for_select(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, "event_", "") }}
         , install_age
         , install_date
diff --git a/models/analytics/google_analytics_events_wid_forced_nulls.sql b/models/analytics/google_analytics_events_wid_forced_nulls.sql
@@ -7,17 +7,17 @@
      },
     incremental_strategy = 'insert_overwrite',
     require_partition_filter = true,
-    cluster_by = ["event_name", "platform", "app_id"]
+    cluster_by = ["event_name", "platform", "bundle_id"]
 
 ) }}
 
 
-{%- set columnNamesEventDimensions = ["app_id", "reverse_app_id","event_name", "platform", "appstore", "app_version", "platform_version",
+{%- set columnNamesEventDimensions = ["bundle_id", "reverse_bundle_id","event_name", "platform", "appstore", "app_version", "platform_version",
                                 "user_properties", "event_parameters",
                                 "geo", "device_hardware", "device_language", "device_time_zone_offset",
                                 "traffic_source"
 ] -%}
-{%- set columnNamesInstallDimensions = ["app_id", "reverse_app_id", "event_name", "platform", "appstore", "app_version", "platform_version",
+{%- set columnNamesInstallDimensions = ["bundle_id", "reverse_bundle_id", "event_name", "platform", "appstore", "app_version", "platform_version",
                                 "user_properties", "event_parameters",
                                 "geo", "device_hardware", "device_language", "device_time_zone_offset",
                                 "traffic_source"
@@ -50,6 +50,8 @@
 {%- set miniColumnsToAlsoNil = ta_firebase.get_mini_columns_to_also_force_null_when_rolling_up() -%}
 WITH data as (
     SELECT    DATE(events.event_ts) as event_date
+            , events.project_id
+            , events.dataset_id
             , {{ ta_firebase.unpack_columns_into_minicolumns(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, miniColumnsToAlsoNil ,"events.", "event_") }}
             , DATE(installs.install_ts) as install_date
             , events.install_age as install_age
@@ -66,11 +68,13 @@ WITH data as (
     AND {{ ta_firebase.makeListIntoSQLInFilter("events.event_name", eventNamesToLookFor| list) }}
     AND installs.event_date >= '2020-01-01'
     -- TODO: max join on installs ?
-    GROUP BY 1,2,3,4 {% for n in range(5, 5 + eventDimensionsUnnestedCount + installedDatesDimensionsUnnestedCount + installDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
+    GROUP BY 1,2,3,4,5,6 {% for n in range(7, 7 + eventDimensionsUnnestedCount + installedDatesDimensionsUnnestedCount + installDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
 )
 
 
 SELECT event_date
+        , project_id
+        , dataset_id
         , {{ ta_firebase.pack_minicolumns_into_structs_for_select(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, "event_", "") }}
         , install_age
         , install_date
diff --git a/models/analytics/google_analytics_installs.sql b/models/analytics/google_analytics_installs.sql
@@ -7,11 +7,11 @@
      },
     incremental_strategy = 'insert_overwrite',
     require_partition_filter = true,
-    cluster_by = ["event_name", "platform", "app_id"]
+    cluster_by = ["event_name", "platform", "bundle_id"]
 ) }}
 
 
-{%- set columnNamesEventDimensions = ["app_id", "reverse_app_id", "event_name", "platform", "appstore", "app_version", "platform_version",
+{%- set columnNamesEventDimensions = ["bundle_id", "reverse_bundle_id", "event_name", "platform", "appstore", "app_version", "platform_version",
                                 "user_properties", "event_parameters",
                                 "geo", "device_hardware", "device_language", "device_time_zone_offset",
                                 "traffic_source"
@@ -26,17 +26,21 @@
 WITH data as (
     SELECT    DATE(event_ts) as event_date
             , DATE(install_ts) as install_date
+            , project_id
+            , dataset_id
             , install_age
             , {{ ta_firebase.unpack_columns_into_minicolumns(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, [], "", "") }}
             , COUNT(1) as cnt
             , COUNT(DISTINCT(user_pseudo_id)) as users
 
     FROM {{ ref("google_analytics_installs_raw") }}
     WHERE {{ ta_firebase.analyticsDateFilterFor('event_date') }}
-    GROUP BY 1,2,3 {% for n in range(4, 4 + eventDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
+    GROUP BY 1,2,3,4,5 {% for n in range(6, 6 + eventDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
 )
 SELECT  event_date
       , install_date
+      , project_id
+      , dataset_id
       , install_age
       , {{ ta_firebase.pack_minicolumns_into_structs_for_select(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, "", "") }}
       , cnt
diff --git a/models/analytics/google_analytics_installs_raw.sql b/models/analytics/google_analytics_installs_raw.sql
@@ -7,7 +7,7 @@
      },
     incremental_strategy = 'insert_overwrite',
     require_partition_filter = true,
-    cluster_by = ["platform", "app_id"],
+    cluster_by = ["platform", "bundle_id"],
 ) }}
     -- incremental_strategy='insert_overwrite',
     -- require_partition_filter = false
diff --git a/models/analytics/google_analytics_ui.sql b/models/analytics/google_analytics_ui.sql
@@ -7,7 +7,7 @@
      },
     incremental_strategy = 'insert_overwrite',
     require_partition_filter = true,
-    cluster_by = ["event_name", "platform", "app_id"]
+    cluster_by = ["event_name", "platform", "bundle_id"]
 ) }}
 
 SELECT *
diff --git a/models/analytics/google_analytics_ui_wid.sql b/models/analytics/google_analytics_ui_wid.sql
@@ -7,7 +7,7 @@
      },
     incremental_strategy = 'insert_overwrite',
     require_partition_filter = true,
-    cluster_by = ["event_name", "platform", "app_id"]
+    cluster_by = ["event_name", "platform", "bundle_id"]
 
 ) }}
 
diff --git a/models/app_health/app_health.sql b/models/app_health/app_health.sql
@@ -61,7 +61,7 @@
     {%- endfor -%}
 {%- endfor -%}
 
-{%- set commonDimensions = ["event_date","platform","app_id","reverse_app_id",
+{%- set commonDimensions = ["event_date","platform","bundle_id","reverse_bundle_id",
                             "app_version.join_value","platform_version.join_value",
                             "device_hardware.type","device_hardware.manufacturer","device_hardware.os_model"] -%}
 
@@ -134,8 +134,8 @@ WITH analytics AS (
 )
 SELECT  event_date
       , platform 
-      , app_id
-      , reverse_app_id
+      , bundle_id
+      , reverse_bundle_id
       , {{ ta_firebase.get_version_record_from_normalized('app_version_join_value') }} AS app_version
       , {{ ta_firebase.get_version_record_from_normalized('platform_version_join_value') }} AS platform_version
       , STRUCT<type STRING, manufacturer STRING, os_model STRING>(
diff --git a/models/crashlytics/crashlytics_events.sql b/models/crashlytics/crashlytics_events.sql
@@ -10,7 +10,7 @@
 ) }}
 
 
-{%- set columnNamesEventDimensions = ["app_id", "reverse_app_id", "issue", "platform", "error_type", "process_state"
+{%- set columnNamesEventDimensions = ["bundle_id", "reverse_bundle_id", "issue", "platform", "error_type", "process_state"
                                      ,"orientation", "app_version", "platform_version", "jailbroken_state"
                                      ,"device_hardware", "custom_keys"
 ] -%}
@@ -22,12 +22,14 @@
  
 WITH data as (
     SELECT   DATE(event_ts) as event_date
+            , project_id
+            , dataset_id
             , {{ ta_firebase.unpack_columns_into_minicolumns(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, [], "", "") }}
             , COUNT(1) as cnt -- same as COUNT(DISTINCT(event_id))
             , COUNT(DISTINCT(crashlytics_user_pseudo_id)) as users
     FROM {{ ref("crashlytics_events_raw") }}
     WHERE {{ ta_firebase.analyticsDateFilterFor('event_date') }}
-    GROUP BY 1 {% for n in range(2, 2 + eventDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
+    GROUP BY 1,2,3 {% for n in range(4, 4 + eventDimensionsUnnestedCount) -%} ,{{ n }} {%- endfor %}
 )
 SELECT event_date
         , {{ ta_firebase.pack_minicolumns_into_structs_for_select(columnsForEventDimensions, miniColumnsToIgnoreInGroupBy, "", "") }}
diff --git a/models/crashlytics/crashlytics_events_raw.sql b/models/crashlytics/crashlytics_events_raw.sql
@@ -33,8 +33,8 @@
     , installation_uuid as crashlytics_user_pseudo_id
     , (SELECT value FROM UNNEST(custom_keys) WHERE key = 'fb_user_pseudo_id') as firebase_analytics_user_pseudo_id
     , COALESCE(user.id, (SELECT value FROM UNNEST(custom_keys) WHERE key = 'app_user_id')) as user_id
-    , bundle_identifier as app_id
-    , ARRAY_TO_STRING(ARRAY_REVERSE(SPLIT(bundle_identifier, '.')), '.') as reverse_app_id
+    , bundle_identifier as bundle_id
+    , ARRAY_TO_STRING(ARRAY_REVERSE(SPLIT(bundle_identifier, '.')), '.') as reverse_bundle_id
     , event_id
          -- the platform we get in operating_system.type is not populated for Android, only for iOS. So rely on _TABLE_SUFFIX instead
     , CASE WHEN _TABLE_SUFFIX LIKE '%ANDROID%' THEN'ANDROID'
@@ -116,8 +116,8 @@
     , installation_uuid as crashlytics_user_pseudo_id
     , (SELECT value FROM UNNEST(custom_keys) WHERE key = 'fb_user_pseudo_id') as firebase_analytics_user_pseudo_id
     , COALESCE(user.id, (SELECT value FROM UNNEST(custom_keys) WHERE key = 'app_user_id')) as user_id
-    , bundle_identifier as app_id
-    , ARRAY_TO_STRING(ARRAY_REVERSE(SPLIT(bundle_identifier, '.')), '.') as reverse_app_id
+    , bundle_identifier as bundle_id
+    , ARRAY_TO_STRING(ARRAY_REVERSE(SPLIT(bundle_identifier, '.')), '.') as reverse_bundle_id
     , event_id
          -- the platform we get in operating_system.type is not populated for Android, only for iOS. So rely on _TABLE_SUFFIX instead
     , CASE WHEN _TABLE_SUFFIX LIKE '%ANDROID%' THEN'ANDROID'
diff --git a/models/crashlytics/docs/crashlytics_events_raw.yml b/models/crashlytics/docs/crashlytics_events_raw.yml
@@ -12,9 +12,9 @@ models:
         description: "`fb_user_pseudo_id` custom key set in Firebase Crashlytics (by the TA SDK)"
       - name: user_id
         description: "`app_user_id` custom key set in Firebase Crashlytics (by the TA SDK)"
-      - name: app_id
+      - name: bundle_id
         description: "iOS: bundle_identifier, Android: package_name"
-      - name: reverse_app_id
+      - name: reverse_bundle_id
         description: "iOS: bundle_identifier, Android: package_name in reversed format. So 'com.ta.myapp' becomes 'myapp.ta.com'"
       - name: event_id
         description: ""
diff --git a/seeds/ta_iso_country.csv b/seeds/ta_iso_country.csv
