feat: per module requirements configs for bigquery

Makefile

@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.23
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.25.4
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 ENABLE_BPMETADATA := 1
@@ -82,7 +82,7 @@ docker_generate_docs:
 		-e ENABLE_BPMETADATA \
 		-v $(CURDIR):/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
-		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs -d'
+		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs -d --per-module-requirements'
 
 # Alias for backwards compatibility
 .PHONY: generate_docs

metadata.yaml

@@ -386,28 +386,28 @@ spec:
     roles:
       - level: Project
         roles:
+          - roles/serviceusage.serviceUsageAdmin
+          - roles/storage.admin
           - roles/bigquery.admin
           - roles/aiplatform.admin
-          - roles/cloudfunctions.admin
           - roles/dataform.admin
-          - roles/datalineage.viewer
           - roles/iam.serviceAccountAdmin
-          - roles/iam.serviceAccountTokenCreator
-          - roles/iam.serviceAccountUser
           - roles/logging.configWriter
           - roles/resourcemanager.projectIamAdmin
           - roles/run.invoker
-          - roles/serviceusage.serviceUsageAdmin
-          - roles/storage.admin
           - roles/workflows.admin
+          - roles/cloudfunctions.admin
+          - roles/datalineage.viewer
+          - roles/iam.serviceAccountTokenCreator
+          - roles/iam.serviceAccountUser
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
-      - bigquerystorage.googleapis.com
       - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - bigquerystorage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
+      - serviceusage.googleapis.com
     providerVersions:
       - source: hashicorp/google
         version: ">= 5.39, < 7"

modules/authorization/metadata.yaml

@@ -93,28 +93,28 @@ spec:
     roles:
       - level: Project
         roles:
-          - roles/bigquery.admin
           - roles/aiplatform.admin
           - roles/cloudfunctions.admin
+          - roles/iam.serviceAccountAdmin
+          - roles/serviceusage.serviceUsageAdmin
+          - roles/storage.admin
+          - roles/workflows.admin
+          - roles/bigquery.admin
           - roles/dataform.admin
           - roles/datalineage.viewer
-          - roles/iam.serviceAccountAdmin
           - roles/iam.serviceAccountTokenCreator
           - roles/iam.serviceAccountUser
           - roles/logging.configWriter
           - roles/resourcemanager.projectIamAdmin
           - roles/run.invoker
-          - roles/serviceusage.serviceUsageAdmin
-          - roles/storage.admin
-          - roles/workflows.admin
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
-      - bigquerystorage.googleapis.com
       - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - bigquerystorage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
+      - serviceusage.googleapis.com
     providerVersions:
       - source: hashicorp/google
         version: ">= 4.44, < 7"

modules/data_warehouse/metadata.yaml

@@ -135,40 +135,40 @@ spec:
     roles:
       - level: Project
         roles:
-          - roles/bigquery.admin
-          - roles/aiplatform.admin
-          - roles/cloudfunctions.admin
-          - roles/dataform.admin
+          - roles/resourcemanager.projectIamAdmin
+          - roles/run.invoker
           - roles/datalineage.viewer
-          - roles/iam.serviceAccountAdmin
           - roles/iam.serviceAccountTokenCreator
-          - roles/iam.serviceAccountUser
           - roles/logging.configWriter
-          - roles/resourcemanager.projectIamAdmin
-          - roles/run.invoker
           - roles/serviceusage.serviceUsageAdmin
           - roles/storage.admin
           - roles/workflows.admin
+          - roles/bigquery.admin
+          - roles/aiplatform.admin
+          - roles/cloudfunctions.admin
+          - roles/dataform.admin
+          - roles/iam.serviceAccountAdmin
+          - roles/iam.serviceAccountUser
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
-      - bigquerystorage.googleapis.com
       - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - bigquerystorage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
+      - serviceusage.googleapis.com
     providerVersions:
       - source: hashicorp/archive
-        version: 10.1.1
+        version: ">= 2.4.2"
       - source: hashicorp/google
         version: ">= 6.11, < 7"
       - source: hashicorp/google-beta
         version: ">= 6.11, < 7"
       - source: hashicorp/http
         version: ">= 2"
       - source: hashicorp/local
-        version: ">=2.4"
+        version: ">= 2.4"
       - source: hashicorp/random
-        version: 10.1.1
+        version: ">= 3.6.2"
       - source: hashicorp/time
         version: ">= 0.9.1"

modules/scheduled_queries/metadata.yaml

@@ -59,28 +59,28 @@ spec:
     roles:
       - level: Project
         roles:
+          - roles/storage.admin
+          - roles/workflows.admin
           - roles/bigquery.admin
           - roles/aiplatform.admin
           - roles/cloudfunctions.admin
-          - roles/dataform.admin
           - roles/datalineage.viewer
           - roles/iam.serviceAccountAdmin
           - roles/iam.serviceAccountTokenCreator
           - roles/iam.serviceAccountUser
+          - roles/run.invoker
+          - roles/dataform.admin
           - roles/logging.configWriter
           - roles/resourcemanager.projectIamAdmin
-          - roles/run.invoker
           - roles/serviceusage.serviceUsageAdmin
-          - roles/storage.admin
-          - roles/workflows.admin
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
-      - bigquerystorage.googleapis.com
       - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - bigquerystorage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
+      - serviceusage.googleapis.com
     providerVersions:
       - source: hashicorp/google
         version: ">= 4.0, < 7"

modules/udf/metadata.yaml

@@ -63,28 +63,28 @@ spec:
     roles:
       - level: Project
         roles:
-          - roles/bigquery.admin
+          - roles/workflows.admin
           - roles/aiplatform.admin
           - roles/cloudfunctions.admin
-          - roles/dataform.admin
           - roles/datalineage.viewer
           - roles/iam.serviceAccountAdmin
           - roles/iam.serviceAccountTokenCreator
           - roles/iam.serviceAccountUser
+          - roles/run.invoker
+          - roles/storage.admin
+          - roles/bigquery.admin
+          - roles/dataform.admin
           - roles/logging.configWriter
           - roles/resourcemanager.projectIamAdmin
-          - roles/run.invoker
           - roles/serviceusage.serviceUsageAdmin
-          - roles/storage.admin
-          - roles/workflows.admin
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
-      - bigquerystorage.googleapis.com
       - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - bigquerystorage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
+      - serviceusage.googleapis.com
     providerVersions:
       - source: hashicorp/google
         version: ">= 3.53, < 7"

test/setup/iam.tf

@@ -15,22 +15,90 @@
  */
 
 locals {
-  int_required_roles = [
-    "roles/bigquery.admin",
-    "roles/aiplatform.admin",
-    "roles/cloudfunctions.admin",
-    "roles/dataform.admin",
-    "roles/datalineage.viewer",
-    "roles/iam.serviceAccountAdmin",
-    "roles/iam.serviceAccountTokenCreator",
-    "roles/iam.serviceAccountUser",
-    "roles/logging.configWriter",
-    "roles/resourcemanager.projectIamAdmin",
-    "roles/run.invoker",
-    "roles/serviceusage.serviceUsageAdmin",
-    "roles/storage.admin",
-    "roles/workflows.admin"
-  ]
+  per_module_roles = {
+    root = [
+      "roles/bigquery.admin",
+      "roles/aiplatform.admin",
+      "roles/cloudfunctions.admin",
+      "roles/dataform.admin",
+      "roles/datalineage.viewer",
+      "roles/iam.serviceAccountAdmin",
+      "roles/iam.serviceAccountTokenCreator",
+      "roles/iam.serviceAccountUser",
+      "roles/logging.configWriter",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/run.invoker",
+      "roles/serviceusage.serviceUsageAdmin",
+      "roles/storage.admin",
+      "roles/workflows.admin"
+    ]
+    authorization = [
+      "roles/bigquery.admin",
+      "roles/aiplatform.admin",
+      "roles/cloudfunctions.admin",
+      "roles/dataform.admin",
+      "roles/datalineage.viewer",
+      "roles/iam.serviceAccountAdmin",
+      "roles/iam.serviceAccountTokenCreator",
+      "roles/iam.serviceAccountUser",
+      "roles/logging.configWriter",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/run.invoker",
+      "roles/serviceusage.serviceUsageAdmin",
+      "roles/storage.admin",
+      "roles/workflows.admin"
+    ]
+    data_warehouse = [
+      "roles/bigquery.admin",
+      "roles/aiplatform.admin",
+      "roles/cloudfunctions.admin",
+      "roles/dataform.admin",
+      "roles/datalineage.viewer",
+      "roles/iam.serviceAccountAdmin",
+      "roles/iam.serviceAccountTokenCreator",
+      "roles/iam.serviceAccountUser",
+      "roles/logging.configWriter",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/run.invoker",
+      "roles/serviceusage.serviceUsageAdmin",
+      "roles/storage.admin",
+      "roles/workflows.admin"
+    ]
+    scheduled_queries = [
+      "roles/bigquery.admin",
+      "roles/aiplatform.admin",
+      "roles/cloudfunctions.admin",
+      "roles/dataform.admin",
+      "roles/datalineage.viewer",
+      "roles/iam.serviceAccountAdmin",
+      "roles/iam.serviceAccountTokenCreator",
+      "roles/iam.serviceAccountUser",
+      "roles/logging.configWriter",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/run.invoker",
+      "roles/serviceusage.serviceUsageAdmin",
+      "roles/storage.admin",
+      "roles/workflows.admin"
+    ]
+    udf = [
+      "roles/bigquery.admin",
+      "roles/aiplatform.admin",
+      "roles/cloudfunctions.admin",
+      "roles/dataform.admin",
+      "roles/datalineage.viewer",
+      "roles/iam.serviceAccountAdmin",
+      "roles/iam.serviceAccountTokenCreator",
+      "roles/iam.serviceAccountUser",
+      "roles/logging.configWriter",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/run.invoker",
+      "roles/serviceusage.serviceUsageAdmin",
+      "roles/storage.admin",
+      "roles/workflows.admin"
+    ]
+  }
+
+  int_required_roles = tolist(toset(flatten(values(local.per_module_roles))))
 }
 
 resource "google_service_account" "int_test" {