bqetl_project.yaml

default:
  project: moz-fx-data-shared-prod
  sql_dir: sql/
  sql_generators_dir: sql_generators/
  test_project: moz-fx-data-integration-tests
  non_user_facing_dataset_suffixes:  # suffixes of datasets with non-user-facing views
  - _derived
  - _external
  - _bi
  - _restricted
  - _syndicate
  - glam_etl
  user_facing_project: mozdata
  backfill_project: moz-fx-data-backfill
  additional_projects:  # additional valid projects managed outside of bigquery-etl
  - moz-fx-data-pioneer-nonprod
  - moz-fx-data-pioneer-prod
  - moz-fx-data-glam-prod-fca7
  - moz-fx-glam-prod
  public_project: mozilla-public-data

monitoring:
  bigeye_warehouse_id: 1939
  bigeye_conn_id: bigeye_connection
  bigeye_workspace_ids:
  - 463
  - 508
  - 509
  stable_tables_monitoring:
    # Firefox Desktop
    firefox_desktop_stable:
    - baseline_v1
    telemetry_stable:
    - main_v5
    # Firefox IOS
    org_mozilla_ios_fennec_stable:
    - baseline_v1
    org_mozilla_ios_firefoxbeta_stable:
    - baseline_v1
    org_mozilla_ios_firefox_stable:
    - baseline_v1
    # Focus Android
    org_mozilla_focus_nightly_stable:
    - baseline_v1
    org_mozilla_focus_beta_stable:
    - baseline_v1
    org_mozilla_focus_stable:
    - baseline_v1
    # Focus IOS
    org_mozilla_ios_focus_stable:
    - baseline_v1
    # Firefox (Fenix) Android
    org_mozilla_fenix_stable:
    - baseline_v1
    org_mozilla_fenix_nightly_stable:
    - baseline_v1
    org_mozilla_fennec_aurora_stable:
    - baseline_v1
    org_mozilla_firefox_beta_stable:
    - baseline_v1
    org_mozilla_firefox_stable:
    - baseline_v1

dry_run:
  function: https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/dryrun
  function_accounts:
  - bigquery-etl-dryrun@moz-fx-data-shared-prod.iam.gserviceaccount.com
  - bigquery-etl-dryrun@moz-fx-data-shar-nonprod-efed.iam.gserviceaccount.com
  cache_ttl_seconds: 900  # Cache dry run results for 15 minutes (900 seconds)
  skip:
  ## skip all data-observability-dev queries due to CI lacking permissions in that project.
  # TODO: once data observability platform assessment concludes this should be removed.
  - sql/data-observability-dev/**/**/*.sql
  # Access Denied
  - sql/moz-fx-data-shared-prod/monitoring_derived/metadata_standardization_v1/query.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/metadata_completeness_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_desktop_derived/desktop_installs_v1/query.sql
  - sql/moz-fx-data-experiments/monitoring/query_cost_v1/query.sql
  - sql/moz-fx-data-shared-prod/app_store/**/*.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql
  - sql/moz-fx-data-shared-prod/pocket/pocket_reach_mau/view.sql
  - sql/moz-fx-data-shared-prod/telemetry/buildhub2/view.sql
  - sql/moz-fx-data-shared-prod/app_store_v2_syndicate/apple_store_territory_report/view.sql
  - sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_accounts_v1/query.sql
  - sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_emails_v1/query.sql
  - sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
  - sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/table_storage_v1/query.sql
  - sql/moz-fx-data-shared-prod/activity_stream/impression_stats_by_experiment/view.sql
  - sql/moz-fx-data-shared-prod/activity_stream/impression_stats_flat/view.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql
  # uses time travel, will error on dates prior to the time travel window
  - sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql
  - sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_recovery_phones_counts_v1/query.sql
  - sql/moz-fx-data-shared-prod/accounts_db_external/**/*.sql
  - sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/**/*.sql
  - sql/moz-fx-data-shared-prod/ads/ppa_measurements/*.sql
  - sql/moz-fx-data-shared-prod/ads/ppa_measurements_limited/*.sql
  - sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v2/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_oauth_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_auth_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_content_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_first_seen_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_user_ids_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_stdout_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_gcp_stderr_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_gcp_stderr_events_v1_live/view.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_gcp_stdout_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_gcp_stdout_events_v1_live/view.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_auth_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_content_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_stdout_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_gcp_stdout_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_gcp_stderr_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/recent_fxa_gcp_stderr_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/recent_fxa_gcp_stdout_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v2/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/query.sql
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v2/query.sql
  - sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/query.sql
  - sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/query.sql
  - sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/query.sql
  - sql/moz-fx-data-shared-prod/fakespot/**/*.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/shredder_progress/view.sql
  - sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql
  - sql/moz-fx-data-shared-prod/stripe_derived/**/*.sql
  - sql/moz-fx-data-shared-prod/stripe_external/**/*.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/apple_subscriptions_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/apple_subscriptions_changelog_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/firestore_stripe_subscriptions_status_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/google_subscriptions_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/google_subscriptions_changelog_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_apple_subscriptions_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_google_subscriptions_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_stripe_subscriptions_history_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_customers_revised_changelog_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_logical_subscriptions_history_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_plans_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_products_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_revised_changelog_v1/query.sql
  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_history_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/devices_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/fxa_attribution_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/funnel_product_page_to_subscribed_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/guardian_apple_events_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/login_flows_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_empty_check_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_summary_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/subscriptions_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/users_attribution_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/users_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/query.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v3/query.sql
  - sql/moz-fx-data-shared-prod/fenix_derived/google_ads_campaign_cost_breakdowns_v1/query.sql
  - sql/moz-fx-data-shared-prod/search_terms*/**/*.sql
  - sql/moz-fx-data-bq-performance/release_criteria/dashboard_health_v1/query.sql
  - sql/moz-fx-data-bq-performance/release_criteria/rc_flattened_test_data_v1/query.sql
  - sql/moz-fx-data-bq-performance/release_criteria/release_criteria_summary_v1/query.sql
  - sql/moz-fx-data-bq-performance/release_criteria/stale_tests_v1/query.sql
  - sql/moz-fx-data-bq-performance/release_criteria/release_criteria_v1/query.sql
  - sql/moz-fx-data-shared-prod/contextual_services/**/*.sql
  - sql/moz-fx-data-shared-prod/contextual_services_derived/**/*.sql
  - sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql
  - sql/moz-fx-data-shared-prod/contextual_services_derived/adm_forecasting_v1/query.sql
  - sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql
  - sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/query.sql
  - sql/moz-fx-data-shared-prod/contextual_services_derived/suggest_revenue_levers_daily_v1/query.sql
  - sql/moz-fx-data-marketing-prod/adjust_derived/**/*.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/airflow_*/*.sql
  - sql/moz-fx-data-shared-prod/google_ads_derived/**/*.sql
  - sql/moz-fx-data-shared-prod/**/client_deduplication*/*.sql
  - sql/moz-fx-data-shared-prod/mozilla_org_derived/downloads_with_attribution_v1/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_org_derived/downloads_with_attribution_v2/query.sql
  - sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v1/query.sql
  - sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v2/query.sql
  - sql/moz-fx-data-shared-prod/google_play_store/reviews/view.sql
  - sql/moz-fx-data-shared-prod/google_search_console_derived/search_impressions_by_page_v1/query.sql
  - sql/moz-fx-data-shared-prod/google_search_console_derived/search_impressions_by_page_v2/query.sql
  - sql/moz-fx-data-shared-prod/google_search_console_derived/search_impressions_by_site_v1/query.sql
  - sql/moz-fx-data-shared-prod/google_search_console_derived/search_impressions_by_site_v2/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_org_derived/gclid_conversions_v1/query.sql
  - sql/moz-fx-data-shared-prod/fenix_derived/ltv_state_values_v1/query.sql
  - sql/moz-fx-data-shared-prod/fenix_derived/ltv_state_values_v2/query.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/glean_server_knob_experiments_v1/query.sql
  - sql/moz-fx-data-shared-prod/stub_attribution_service_derived/dl_token_ga_attribution_lookup_v1/query.sql
  - sql/moz-fx-data-shared-prod/stub_attribution_service_derived/dl_token_ga_attribution_lookup_v2/query.sql
  - sql/moz-fx-data-shared-prod/mozilla_org_derived/gclid_conversions_v2/query.sql
  - sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql
  - sql/moz-fx-data-shared-prod/search_derived/mobile_search_aggregates_v1/query.sql
  - sql/moz-fx-data-shared-prod/monitoring/structured_missing_columns/view.sql
  - sql/moz-fx-data-shared-prod/monitoring/telemetry_missing_columns/view.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v1/view.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/table_partition_expirations_v1/query.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/shredder_per_job_stats_v1/query.sql
  - sql/moz-fx-data-shared-prod/jira_service_desk/**/*.sql
  - sql/moz-fx-data-shared-prod/zoom/**/*.sql
  - sql/moz-fx-data-shared-prod/taskclusteretl/person_mozilla_com/view.sql
  # No matching signature for function IF
  - sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_last_seen/query.sql
  # Duplicate UDF
  - sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_daily/query.sql
  # HTTP Error 408: Request Time-out
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v2/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/events_daily_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/main_nightly_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/main_1pct_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/fog_decision_support_percentiles_v1/query.sql
  - sql/moz-fx-data-shared-prod/fenix_derived/nimbus_recorded_targeting_context_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/experiment_search_aggregates_v1/query.sql
  # Query parameter not found
  - sql/moz-fx-data-shared-prod/telemetry_derived/experiments_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_scalar_aggregates_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_scalar_aggregates_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_boolean_aggregates_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_content_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_gpu_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_parent_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_histogram_aggregates_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v2/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_probe_counts_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_bucket_counts_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/glam_client_probe_counts_extract_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/client_probe_counts/view.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/glam_sample_counts_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/glam_user_counts_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_scalar_probe_counts_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry/client_probe_counts/view.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_sql_run_check_v1/query.sql
  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_scalar_aggregates_v1/query.sql
  - sql/moz-fx-data-shared-prod/sumo_ga/analytics_314096102/view.sql
  - sql/moz-fx-data-shared-prod/sumo_ga/analytics_432581103/view.sql
  - sql/moz-fx-data-shared-prod/mofo/analytics_321347134_events/view.sql
  # Dataset sql/glam-fenix-dev:glam_etl was not found
  - sql/glam-fenix-dev/glam_etl/**/*.sql
  - sql/moz-fx-data-glam-prod-fca7/glam_etl/**/*.sql
  - sql/moz-fx-glam-prod/glam_etl/**/*.sql
  # Source table does not have a verifiable schema
  - sql/moz-fx-data-shared-prod/monitoring_derived/outerbounds_cost_per_flow_run_v1/view.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/outerbounds_cost_per_flow_v1/view.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/rayserve_cost_fakespot_tenant_v1/view.sql
  - sql/moz-fx-data-shared-prod/monitoring_derived/rayserve_cost_fakespot_tenant_prod_v1/view.sql
  # Query templates
  - sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/fenix_metrics.template.sql
  - sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql
  - sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v2/fenix_metrics.template.sql
  - sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v2/mobile_search_clients_daily.template.sql
  # Query too complex
  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/query.sql
  # Tests
  - sql/moz-fx-data-test-project/test/simple_view/view.sql
  default_projects:  # projects to use for query dry run jobs
  - moz-fx-data-backfill-10
  - moz-fx-data-backfill-11
  - moz-fx-data-backfill-12
  - moz-fx-data-backfill-13
  - moz-fx-data-backfill-14
  - moz-fx-data-backfill-15
  - moz-fx-data-backfill-16
  - moz-fx-data-backfill-17
  - moz-fx-data-backfill-18
  - moz-fx-data-backfill-19
  - moz-fx-data-backfill-20
  - moz-fx-data-backfill-21
  - moz-fx-data-backfill-22
  - moz-fx-data-backfill-23
  - moz-fx-data-backfill-24
  - moz-fx-data-backfill-25
  - moz-fx-data-backfill-26
  - moz-fx-data-backfill-27
  - moz-fx-data-backfill-28
  - moz-fx-data-backfill-29
  - moz-fx-data-backfill-31

deprecation:
  retain_dataset_roles:  # the following roles are retain permissions on datasets when a dataset table is deprecated
  - roles/bigquery.dataEditor  # operational access

format:
  skip:
  - bigquery_etl/glam/templates/**/*.sql
  - sql_generators/**/*.sql
  - tests/**/*.sql
  - sql/moz-fx-data-shared-prod/udf_js/jackknife_percentile_ci/udf.sql
  - sql/moz-fx-data-shared-prod/udf_js/bootstrap_percentile_ci/udf.sql
  - sql/moz-fx-data-shared-prod/udf_legacy/contains.sql
  - sql/moz-fx-data-shared-prod/udf_legacy/date_format.sql
  - sql/moz-fx-data-shared-prod/udf_legacy/date_trunc.sql
  - sql/moz-fx-data-shared-prod/udf_legacy/to_iso8601.sql
  - stored_procedures/safe_crc32_uuid.sql
  skip_qualifying_references:
  - sql/mozfun/**/examples/*.sql
  # Don't try to qualify references when Jinja expressions are being used for table/view IDs.
  - sql/**/bigeye_custom_rules.sql
  - sql/**/checks.sql

routine:
  dependency_dir: udf_js_lib/
  publish:
    gcs_bucket: moz-fx-data-prod-bigquery-etl
    gcs_path: ""
    skip:
    - sql/moz-fx-data-shared-prod/udf/main_summary_scalars/udf.sql
  example_dir: examples
  project: mozfun
  test_projects:
  - sql/mozfun
  - sql/moz-fx-data-shared-prod
  - sql/moz-fx-data-integration-tests

view:
  validation:
    skip:
    # tests
    - sql/moz-fx-data-test-project/test/simple_view/view.sql
  publish:
    skip:
    - sql/moz-fx-data-shared-prod/activity_stream/tile_id_types/view.sql
    - sql/moz-fx-data-shared-prod/pocket/pocket_reach_mau/view.sql
    - sql/moz-fx-data-shared-prod/telemetry/buildhub2/view.sql
    # Dataset glam-fenix-dev:glam_etl was not found
    # TODO: this should be removed if views are to be automatically deployed
    - sql/glam-fenix-dev/glam_etl/**/view.sql
    # tests
    - sql/moz-fx-data-test-project/test/simple_view/view.sql
    # Resources exceeded
    - sql/moz-fx-data-shared-prod/telemetry/experiment_cumulative_search_count/view.sql
    - sql/moz-fx-data-shared-prod/telemetry/experiment_cumulative_search_with_ads_count/view.sql
    - sql/moz-fx-data-shared-prod/telemetry/experiment_cumulative_ad_clicks/view.sql
    - sql/moz-fx-data-shared-prod/telemetry_derived/experiment_cumulative_ad_clicks_v1/view.sql
    - sql/moz-fx-data-shared-prod/telemetry_derived/experiment_cumulative_search_count_v1/view.sql
    - sql/moz-fx-data-shared-prod/telemetry_derived/experiment_cumulative_search_with_ads_count_v1/view.sql
    - sql/moz-fx-data-shared-prod/telemetry_derived/experiment_search_aggregates_live_v1/view.sql

schema:
  base_schemas_dir: bigquery_etl/schema
  mozilla_pipeline_schemas_uri: https://github.com/mozilla-services/mozilla-pipeline-schemas
  skip_prefixes:
  - pioneer
  - rally
  deploy:
    skip:
    # see comment in query file
    - sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v2_external/query.sql
    - sql/moz-fx-data-shared-prod/*_stable/**

docs:
  docs_dir: docs
  default_projects:
  - mozfun
  - moz-fx-data-shared-prod
  source_url: https://github.com/mozilla/bigquery-etl/blob/generated-sql
  edit_url: https://github.com/mozilla/bigquery-etl/edit/generated-sql

public_data:
  bucket: mozilla-public-data-http
  api_version: v1
  endpoint: https://public-data.telemetry.mozilla.org/
  review_link: https://bugzilla.mozilla.org/show_bug.cgi?id=

render:
  skip:
  # uses {%s} which results in unknown tag exception
  - sql/mozfun/hist/string_to_json/udf.sql

metadata:
  validation:
    skip:
    - sql/moz-fx-data-shared-prod/addons_derived/search_detection_v1/metadata.yaml  # backfill compatibility issue

generate:
  glean_usage:
    deprecated_apps:
    - firefox_fire_tv
    - firefox_reality
    - firefox_reality_pc
    skip_existing:  # Skip automatically updating the following artifacts
    - sql/moz-fx-data-shared-prod/fenix/client_deduplication/**
    - sql/moz-fx-data-shared-prod/org_mozilla_tv_firefox_derived/baseline_clients_last_seen_v1/checks.sql
    - sql/moz-fx-data-shared-prod/org_mozilla_reference_browser_derived/baseline_clients_last_seen_v1/checks.sql
    - sql/moz-fx-data-shared-prod/org_mozilla_ios_focus_derived/baseline_clients_last_seen_v1/metadata.yaml
    - sql/moz-fx-data-shared-prod/org_mozilla_ios_focus_derived/baseline_clients_last_seen_v1/bigconfig.yml
    - sql/moz-fx-data-shared-prod/org_mozilla_social_nightly_derived/baseline_clients_last_seen_v1/checks.sql
    - sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/checks.sql
    - sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/checks.sql
    skip_apps:
    - mlhackweek_search
    - moso_mastodon_android
    - moso_mastodon_web
    - moso_mastodon_backend
    - regrets_reporter
    - tiktokreporter_android
    - tiktokreporter_ios
    events_stream:
      skip_apps:
      - bergamot
      - firefox_echo_show
      - firefox_fire_tv
      - firefox_reality
      - firefox_reality_pc
      - lockwise_android
      - lockwise_ios
      - mozilla_vpn  # VPN events are in `main` pings rather than `events` pings (DENG-8596).
      - tiktokreporter_ios
      - tiktokreporter_android
      metrics_as_struct_apps:
      - accounts_frontend
      - accounts_backend
      - subscription_platform_backend
      slice_by_sample_id_apps:
      - firefox_desktop
      - firefox_desktop_background_update
    events_monitoring:
      skip_apps:
      - ads_backend  # restricted dataset, we don't want to include it aggregate view
      # event pings for this app were removed but still high volume due to nimbus bugs in versions 138 to 140
      # events are still sent in the background-update ping, but they are being skipped
      # See: https://mozilla-hub.atlassian.net/browse/DENG-9715
      - firefox_desktop_background_update
      skip_app_ids:
      - org.mozilla.fenix.nightly
      - org.mozilla.fennec.aurora
      skip_pings:
      - topsites-impression  # access denied
      - serp-categorization  # access denied
      - background-update  # table doesn't exist
      events_tables:  # overwrite event tables
        pine:  # Probe info returns pings that don't have tables, only use events_v1
        - events_v1
      manual_refresh_apps:  # apps to use manual materialized view refreshes for on-demand billing
      - firefox_desktop
    event_flow_monitoring:
      include_apps:
      - accounts_backend
      - accounts_frontend
      - focus_android
      - fenix
      - firefox_desktop
    events_first_seen:
      include_apps:
      - firefox_desktop
      - fenix
      - firefox_ios
    metrics_clients_last_seen:
      skip_apps:
      - accounts_cirrus
      - experimenter_cirrus
      - experimenter_backend
      - firefox_desktop_background_defaultagent
      - firefox_desktop_background_tasks
      - firefox_crashreporter
      - monitor_cirrus
      - mozilla_vpn
      - mozillavpn_backend_cirrus
      - pine
      - thunderbird_android
      - thunderbird_desktop
    clients_last_seen_joined:
      skip_apps:
      - accounts_cirrus
      - experimenter_cirrus
      - experimenter_backend
      - firefox_desktop_background_defaultagent
      - firefox_desktop_background_tasks
      - firefox_crashreporter
      - monitor_cirrus
      - mozilla_vpn
      - mozillavpn_backend_cirrus
      - pine
      - thunderbird_android
      - thunderbird_desktop
    bigconfig:
      skip_apps:
      - firefox_echo_show
      - firefox_fire_tv
      - thunderbird_android
      - moso_mastodon_android
      - mozilla_vpn
      - mozillavpn_backend_cirrus
      - accounts_backend
      - accounts_cirrus
      - burnham
      - firefox_crashreporter
      - firefox_reality
      - firefox_reality_pc
      - lockwise_android
      - mach
      - monitor_backend
      - monitor_cirrus
      - moso_mastodon_backend
      - mozphab
      - mozregression
      - focus_android
      - klar_android
      - focus_ios
      - klar_ios
      - pine
      - reference_browser
      - thunderbird_desktop
      - lockwise_ios
    bqetl_checks:
      skip_apps:
      - mozilla_vpn
      - mozillavpn_backend_cirrus
      - accounts_backend
      - accounts_cirrus
      - burnham
      - firefox_crashreporter
      - firefox_reality
      - firefox_reality_pc
      - lockwise_android
      - mach
      - monitor_backend
      - monitor_cirrus
      - moso_mastodon_backend
      - mozphab
      - mozregression
    app_ping_views:
      skip:  # don't add the following pings to the combined-channel ping view
      - org_mozilla_fennec_aurora.metrics
      - org_mozilla_fenix_nightly.metrics
  stable_views:
    skip_datasets:
    - mlhackweek_search
    - regrets_reporter
    - org_mozilla_ios_tiktokreporter
    - org_mozilla_tiktokreporter
    - regrets_reporter_ucs
    - org_mozilla_ios_tiktok_reporter_tiktok_reportershare
    skip_tables:
      telemetry:
      - modules
  usage_reporting:
    apps:
      fenix:
        channels:
        - nightly
        - beta
        - release
      firefox_ios:
        channels:
        - nightly
        - beta
        - release
      firefox_desktop:
        # firefox_desktop is a single app containing multiple channels hence why set to null.
        channels: null
      focus_android:
        channels:
        - release
        - beta
        - nightly
      focus_ios:
        channels:
        - release
  terms_of_use:
    dag_name: bqetl_terms_of_use
    bigeye_defaults:
      collection: Operational Checks
      notification_channel: '#de-bigeye-triage'
    apps:
      fenix:
        templates:
        - terms_of_use_status_v1
        - terms_of_use_events_v1
        bigeye:
          collection: Operational Checks
          notification_channel: '#de-bigeye-triage'
      firefox_ios:
        templates:
        - terms_of_use_status_v1
        - terms_of_use_events_v1
        bigeye:
          collection: Operational Checks
          notification_channel: '#de-bigeye-triage'
      firefox_desktop:
        templates:
        - terms_of_use_status_v1
        - terms_of_use_messages_v1
        bigeye:
          collection: Operational Checks
          notification_channel: '#de-bigeye-triage'
  stable_views_redacted:
    pings:
      firefox_desktop:
      - microsurvey
  baseline_clients_city_seen:
    apps:
    - firefox_desktop
    - fenix
    - firefox_ios
    - focus_ios
    - focus_android
    - klar_ios
    - klar_android
    - firefox_desktop_background_update
    - firefox_desktop_background_defaultagent
    - firefox_desktop_background_tasks

geo_deprecation:
  skip_apps:
  - ads_backend
  - mozilla_vpn
  - mozillavpn_backend_cirrus
  - accounts_backend
  - accounts_frontend
  skip_tables:
  - newtab
  - newtab_content

retention_exclusion_list:
- sql/moz-fx-data-shared-prod/search_derived/acer_cohort_v1
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v3
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v1
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_28_days_later_v3
- sql/moz-fx-data-shared-prod/firefox_desktop_derived/clients_first_seen_28_days_later_v1
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/fenix_derived/ltv_states_v1
- sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1
- sql/moz-fx-data-shared-prod/fenix_derived/client_ltv_v1
- sql/moz-fx-data-shared-prod/firefox_desktop_derived/ltv_states_v1
- sql/moz-fx-data-shared-prod/firefox_desktop_derived/client_ltv_v1
- sql/moz-fx-data-shared-prod/firefox_desktop_derived/events_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/events_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived.events_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora_derived.events_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived.events_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_fenix_nightly_derived.events_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/events_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived.events_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived.events_first_seen_v1
- sql/moz-fx-data-shared-prod/firefox_ios_derived/ltv_states_v1
- sql/moz-fx-data-shared-prod/firefox_ios_derived/firefox_ios_clients_v1
- sql/moz-fx-data-shared-prod/firefox_ios_derived/client_ltv_v1
- sql/moz-fx-data-shared-prod/ltv_derived/fenix_client_ltv_v1
- sql/moz-fx-data-shared-prod/monitor_cirrus_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/moso_mastodon_backend_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/mozilla_lockbox_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/mozilla_mach_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/mozillavpn_backend_cirrus_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/mozillavpn_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/mozphab_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_connect_firefox_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_fenix_nightly_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/fennec_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_firefoxreality_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_focus_beta_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_focus_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_focus_nightly_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_focus_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_klar_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_ios_lockbox_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_klar_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_mozregression_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_reference_browser_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_social_nightly_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_tv_firefox_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/org_mozilla_vrbrowser_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/pine_derived/baseline_clients_first_seen_v11
- sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_v1
- sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v1
- sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_first_seen_v1
- sql/moz-fx-data-shared-prod/fenix_derived/attribution_clients_v1
- sql/moz-fx-data-shared-prod/firefox_ios_derived/attribution_clients_v1
- sql/moz-fx-data-shared-prod/focus_android_derived/attribution_clients_v1
- sql/moz-fx-data-shared-prod/focus_ios_derived/attribution_clients_v1
- sql/moz-fx-data-shared-prod/klar_android_derived/attribution_clients_v1
- sql/moz-fx-data-shared-prod/klar_ios_derived/attribution_clients_v1
- sql/moz-fx-data-shared-prod/firefox_desktop_derived/adclick_history_v1
- sql/moz-fx-data-shared-prod/fenix_derived/client_adclicks_history_v1
- sql/moz-fx-data-shared-prod/fenix_derived/clients_yearly_v1
- sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_first_seen_v1
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_first_seen_v1
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_first_seen_v2
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_devices_first_seen_v1
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_first_seen_v1
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_first_seen_v2
- sql/moz-fx-data-shared-prod/fenix_derived/attributable_clients_v1
- sql/moz-fx-data-shared-prod/fenix_derived/attributable_clients_v2
- sql/moz-fx-data-shared-prod/firefox_ios_derived/client_adclicks_history_v1
- sql/moz-fx-data-shared-prod/acoustic_external/suppression_list_v1
- sql/moz-fx-data-shared-prod/telemetry_derived/fx_accounts_linked_clients_v1
- sql/moz-fx-data-shared-prod/firefox_desktop_derived/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/fenix_derived/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/firefox_ios_derived/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/focus_ios_derived/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/focus_android_derived/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/klar_ios_derived/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/klar_android_derived/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/firefox_desktop_background_update/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks/baseline_clients_city_seen_v1
- sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_first_seen_v1