Some infra adjustments (#55)

* masthead draft

* sync recent adjustment

Author: max-ostapenko

infra/tf/.terraform.lock.hcl

@@ -8,6 +8,8 @@ Update Frequency: Monthly
 
 ## Dataset Access
 
+[Get Started](https://har.fyi/guides/getting-started/#setting-up-bigquery-to-access-the-http-archive)
+
 The dataset is free and can be publicly accessed via Google BigQuery for fast, scalable analysis or download for offline research. Ideal for developers, analysts, and researchers exploring web performance trends and optimization opportunities.
 
 ## Dataset Contents

infra/tf/attachments/documentation.md

@@ -49,10 +49,12 @@ resource "google_bigquery_analytics_hub_listing" "crawl" {
 }
 
 resource "google_bigquery_analytics_hub_listing_iam_member" "member" {
+  for_each = toset(["roles/analyticshub.viewer", "roles/analyticshub.subscriber"])
+
   project          = local.project
   location         = local.location
   data_exchange_id = google_bigquery_analytics_hub_data_exchange.default.data_exchange_id
   listing_id       = google_bigquery_analytics_hub_listing.crawl.listing_id
-  role             = "roles/analyticshub.viewer"
+  role             = each.value
   member           = "allUsers"
 }

infra/tf/data_exchange.tf

@@ -12,7 +12,7 @@ resource "google_storage_bucket_object" "dataform_trigger_build" {
   source = data.archive_file.dataform-trigger.output_path
 }
 
-resource "google_cloudfunctions2_function" "default" {
+resource "google_cloudfunctions2_function" "dataform_trigger" {
   name     = "dataform-trigger"
   location = local.region
   build_config {
@@ -36,12 +36,12 @@ resource "google_cloudfunctions2_function" "default" {
 }
 
 locals {
-  function_uri = google_cloudfunctions2_function.default.service_config[0].uri
+  function_uri = google_cloudfunctions2_function.dataform_trigger.service_config[0].uri
 }
 
 resource "google_cloud_run_service_iam_member" "member" {
-  location = google_cloudfunctions2_function.default.location
-  service  = google_cloudfunctions2_function.default.name
+  location = google_cloudfunctions2_function.dataform_trigger.location
+  service  = google_cloudfunctions2_function.dataform_trigger.name
   role     = "roles/run.invoker"
   member   = "serviceAccount:${local.function_identity}"
 }

infra/tf/function_dataform_trigger.tf

@@ -0,0 +1,67 @@
+# Documentation:
+# https://docs.mastheadata.com/saas-manual-resource-creation-google-cloud-+-bigquery
+
+# 1. Create Pub/Sub resources
+resource "google_pubsub_topic" "masthead_topic" {
+  project = local.project
+  name    = "masthead-topic"
+}
+
+resource "google_pubsub_subscription" "masthead_agent_subscription" {
+  project              = local.project
+  ack_deadline_seconds = 60
+  expiration_policy {
+    ttl = "2678400s"
+  }
+  message_retention_duration = "604800s"
+  name                       = "masthead-agent-subscription"
+  topic                      = "projects/${local.project}/topics/masthead-topic"
+}
+
+# 2. Create Logs Router
+resource "google_logging_project_sink" "masthead_agent_sink" {
+  destination = "pubsub.googleapis.com/projects/${local.project}/topics/masthead-topic"
+  filter      = <<EOT
+  protoPayload.methodName="google.cloud.bigquery.v2.JobService.InsertJob" OR "google.cloud.bigquery.v2.TableService.InsertTable" OR "google.cloud.bigquery.v2.JobService.Query" OR
+    resource.type="bigquery_dataset" OR "bigquery_project" OR
+    (resource.type="bigquery_table" AND protoPayload.methodName!="google.cloud.bigquery.storage.v1.BigQueryWrite.AppendRows") OR
+    (resource.type="bigquery_table" AND protoPayload.methodName="google.cloud.bigquery.storage.v1.BigQueryWrite.AppendRows" AND sample(insertId, 0.0001))
+  EOT
+  name        = "masthead-agent-sink"
+}
+
+
+resource "google_project_iam_member" "masthead_pubsub_publisherer_member" {
+  role    = "roles/pubsub.publisher"
+  member  = "serviceAccount:[email protected]"
+  project = local.project
+}
+
+# 3. Grant Masthead Service Account roles
+resource "google_project_iam_custom_role" "masthead_bq_meta_reader" {
+  project     = local.project
+  description = "Masthead BigQuery assets metadata reader"
+  permissions = ["bigquery.datasets.get", "bigquery.tables.get", "bigquery.tables.list"]
+  role_id     = "masthead_bq_meta_reader"
+  stage       = "GA"
+  title       = "masthead_bq_meta_reader"
+}
+
+resource "google_project_iam_binding" "masthead_bq_meta_reader_binding" {
+  role    = google_project_iam_custom_role.masthead_bq_meta_reader.id
+  members = ["serviceAccount:[email protected]"]
+  project = local.project
+}
+
+resource "google_project_iam_member" "masthead_pubsub_subscriber_member" {
+  role    = "roles/pubsub.subscriber"
+  member  = "serviceAccount:[email protected]"
+  project = local.project
+}
+
+# 4. Grant Masthead Service Account to quickly onboard from retrospective data
+resource "google_project_iam_binding" "private_logs_viewer_binding" {
+  role    = "roles/logging.privateLogViewer"
+  members = ["serviceAccount:[email protected]"]
+  project = local.project
+}