feat: per module requirements configs for bigquery (#405)

qz267 · Zheng Qin · web-flow · commit eec2567df620 · 2025-07-30T00:04:31.000+05:30
Co-authored-by: Zheng Qin &lt;zhengqin@google.com&gt;
diff --git a/.terraform.lock b/.terraform.lock
diff --git a/Makefile b/Makefile
@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.23
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.25
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 ENABLE_BPMETADATA := 1
@@ -82,7 +82,7 @@ docker_generate_docs:
 		-e ENABLE_BPMETADATA \
 		-v $(CURDIR):/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
-		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs -d'
+		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs -d --per-module-requirements'
 
 # Alias for backwards compatibility
 .PHONY: generate_docs
diff --git a/metadata.yaml b/metadata.yaml
@@ -387,26 +387,13 @@ spec:
       - level: Project
         roles:
           - roles/bigquery.admin
-          - roles/aiplatform.admin
-          - roles/cloudfunctions.admin
-          - roles/dataform.admin
-          - roles/datalineage.viewer
-          - roles/iam.serviceAccountAdmin
-          - roles/iam.serviceAccountTokenCreator
-          - roles/iam.serviceAccountUser
-          - roles/logging.configWriter
-          - roles/resourcemanager.projectIamAdmin
-          - roles/run.invoker
-          - roles/serviceusage.serviceUsageAdmin
           - roles/storage.admin
-          - roles/workflows.admin
+          - roles/cloudkms.cryptoKeyEncrypterDecrypter
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
       - bigquerystorage.googleapis.com
-      - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
     providerVersions:
       - source: hashicorp/google
diff --git a/modules/authorization/metadata.yaml b/modules/authorization/metadata.yaml
@@ -93,28 +93,28 @@ spec:
     roles:
       - level: Project
         roles:
+          - roles/run.invoker
+          - roles/storage.admin
+          - roles/workflows.admin
           - roles/bigquery.admin
           - roles/aiplatform.admin
           - roles/cloudfunctions.admin
           - roles/dataform.admin
-          - roles/datalineage.viewer
-          - roles/iam.serviceAccountAdmin
           - roles/iam.serviceAccountTokenCreator
           - roles/iam.serviceAccountUser
           - roles/logging.configWriter
-          - roles/resourcemanager.projectIamAdmin
-          - roles/run.invoker
           - roles/serviceusage.serviceUsageAdmin
-          - roles/storage.admin
-          - roles/workflows.admin
+          - roles/datalineage.viewer
+          - roles/iam.serviceAccountAdmin
+          - roles/resourcemanager.projectIamAdmin
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
-      - bigquerystorage.googleapis.com
       - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - bigquerystorage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
+      - serviceusage.googleapis.com
     providerVersions:
       - source: hashicorp/google
         version: ">= 4.44, < 7"
diff --git a/modules/data_warehouse/metadata.yaml b/modules/data_warehouse/metadata.yaml
@@ -38,23 +38,23 @@ spec:
       description: cost of this solution is $0.65
       url: https://cloud.google.com/products/calculator/#id=857776c6-49e8-4c6a-adc5-42a15b8fb67d
     cloudProducts:
-    - productId: search_BIGQUERY_SECTION
-      pageUrl: ""
-    - productId: WORKFLOWS_SECTION
-      pageUrl: ""
-    - productId: STORAGE_SECTION
-      pageUrl: ""
-    - productId: ai-platform
-      pageUrl: ""
-    - productId: LOOKER_STUDIO_SECTION
-      pageUrl: lookerstudio.google.com
-      isExternal: true
-    - productId: CLOUD_DMS_SECTION
-      pageUrl: ""
-    - productId: FUNCTIONS_SECTION
-      pageUrl: ""
-    - productId: DATAFORM_SECTION
-      pageUrl: ""
+      - productId: search_BIGQUERY_SECTION
+        pageUrl: ""
+      - productId: WORKFLOWS_SECTION
+        pageUrl: ""
+      - productId: STORAGE_SECTION
+        pageUrl: ""
+      - productId: ai-platform
+        pageUrl: ""
+      - productId: LOOKER_STUDIO_SECTION
+        pageUrl: lookerstudio.google.com
+        isExternal: true
+      - productId: CLOUD_DMS_SECTION
+        pageUrl: ""
+      - productId: FUNCTIONS_SECTION
+        pageUrl: ""
+      - productId: DATAFORM_SECTION
+        pageUrl: ""
   content:
     architecture:
       diagramUrl: www.gstatic.com/pantheon/images/solutions/data-warehouse-architecture_v6.svg
@@ -135,40 +135,40 @@ spec:
     roles:
       - level: Project
         roles:
-          - roles/bigquery.admin
+          - roles/storage.admin
           - roles/aiplatform.admin
           - roles/cloudfunctions.admin
           - roles/dataform.admin
+          - roles/iam.serviceAccountUser
+          - roles/serviceusage.serviceUsageAdmin
+          - roles/workflows.admin
+          - roles/bigquery.admin
           - roles/datalineage.viewer
           - roles/iam.serviceAccountAdmin
           - roles/iam.serviceAccountTokenCreator
-          - roles/iam.serviceAccountUser
           - roles/logging.configWriter
           - roles/resourcemanager.projectIamAdmin
           - roles/run.invoker
-          - roles/serviceusage.serviceUsageAdmin
-          - roles/storage.admin
-          - roles/workflows.admin
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
-      - bigquerystorage.googleapis.com
       - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - bigquerystorage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
+      - serviceusage.googleapis.com
     providerVersions:
       - source: hashicorp/archive
-        version: 10.1.1
+        version: ">= 2.4.2"
       - source: hashicorp/google
         version: ">= 6.11, < 7"
       - source: hashicorp/google-beta
         version: ">= 6.11, < 7"
       - source: hashicorp/http
         version: ">= 2"
       - source: hashicorp/local
-        version: ">=2.4"
+        version: ">= 2.4"
       - source: hashicorp/random
-        version: 10.1.1
+        version: ">= 3.6.2"
       - source: hashicorp/time
         version: ">= 0.9.1"
diff --git a/modules/scheduled_queries/metadata.yaml b/modules/scheduled_queries/metadata.yaml
@@ -59,28 +59,28 @@ spec:
     roles:
       - level: Project
         roles:
-          - roles/bigquery.admin
-          - roles/aiplatform.admin
           - roles/cloudfunctions.admin
           - roles/dataform.admin
           - roles/datalineage.viewer
+          - roles/resourcemanager.projectIamAdmin
+          - roles/run.invoker
+          - roles/workflows.admin
+          - roles/bigquery.admin
+          - roles/aiplatform.admin
           - roles/iam.serviceAccountAdmin
           - roles/iam.serviceAccountTokenCreator
           - roles/iam.serviceAccountUser
           - roles/logging.configWriter
-          - roles/resourcemanager.projectIamAdmin
-          - roles/run.invoker
           - roles/serviceusage.serviceUsageAdmin
           - roles/storage.admin
-          - roles/workflows.admin
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
-      - bigquerystorage.googleapis.com
       - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - bigquerystorage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
+      - serviceusage.googleapis.com
     providerVersions:
       - source: hashicorp/google
         version: ">= 4.0, < 7"
diff --git a/modules/udf/metadata.yaml b/modules/udf/metadata.yaml
@@ -63,28 +63,28 @@ spec:
     roles:
       - level: Project
         roles:
-          - roles/bigquery.admin
-          - roles/aiplatform.admin
-          - roles/cloudfunctions.admin
-          - roles/dataform.admin
-          - roles/datalineage.viewer
           - roles/iam.serviceAccountAdmin
           - roles/iam.serviceAccountTokenCreator
           - roles/iam.serviceAccountUser
-          - roles/logging.configWriter
           - roles/resourcemanager.projectIamAdmin
-          - roles/run.invoker
           - roles/serviceusage.serviceUsageAdmin
+          - roles/bigquery.admin
+          - roles/cloudfunctions.admin
+          - roles/logging.configWriter
+          - roles/run.invoker
           - roles/storage.admin
           - roles/workflows.admin
+          - roles/aiplatform.admin
+          - roles/dataform.admin
+          - roles/datalineage.viewer
     services:
-      - cloudkms.googleapis.com
-      - cloudresourcemanager.googleapis.com
       - bigquery.googleapis.com
-      - bigquerystorage.googleapis.com
       - bigqueryconnection.googleapis.com
-      - serviceusage.googleapis.com
+      - bigquerystorage.googleapis.com
+      - cloudkms.googleapis.com
+      - cloudresourcemanager.googleapis.com
       - iam.googleapis.com
+      - serviceusage.googleapis.com
     providerVersions:
       - source: hashicorp/google
         version: ">= 3.53, < 7"
diff --git a/test/setup/iam.tf b/test/setup/iam.tf
@@ -15,22 +15,79 @@
  */
 
 locals {
-  int_required_roles = [
-    "roles/bigquery.admin",
-    "roles/aiplatform.admin",
-    "roles/cloudfunctions.admin",
-    "roles/dataform.admin",
-    "roles/datalineage.viewer",
-    "roles/iam.serviceAccountAdmin",
-    "roles/iam.serviceAccountTokenCreator",
-    "roles/iam.serviceAccountUser",
-    "roles/logging.configWriter",
-    "roles/resourcemanager.projectIamAdmin",
-    "roles/run.invoker",
-    "roles/serviceusage.serviceUsageAdmin",
-    "roles/storage.admin",
-    "roles/workflows.admin"
-  ]
+  per_module_roles = {
+    root = [
+      "roles/bigquery.admin",
+      "roles/storage.admin",
+      "roles/cloudkms.cryptoKeyEncrypterDecrypter",
+    ]
+    authorization = [
+      "roles/bigquery.admin",
+      "roles/aiplatform.admin",
+      "roles/cloudfunctions.admin",
+      "roles/dataform.admin",
+      "roles/datalineage.viewer",
+      "roles/iam.serviceAccountAdmin",
+      "roles/iam.serviceAccountTokenCreator",
+      "roles/iam.serviceAccountUser",
+      "roles/logging.configWriter",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/run.invoker",
+      "roles/serviceusage.serviceUsageAdmin",
+      "roles/storage.admin",
+      "roles/workflows.admin"
+    ]
+    data_warehouse = [
+      "roles/bigquery.admin",
+      "roles/aiplatform.admin",
+      "roles/cloudfunctions.admin",
+      "roles/dataform.admin",
+      "roles/datalineage.viewer",
+      "roles/iam.serviceAccountAdmin",
+      "roles/iam.serviceAccountTokenCreator",
+      "roles/iam.serviceAccountUser",
+      "roles/logging.configWriter",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/run.invoker",
+      "roles/serviceusage.serviceUsageAdmin",
+      "roles/storage.admin",
+      "roles/workflows.admin"
+    ]
+    scheduled_queries = [
+      "roles/bigquery.admin",
+      "roles/aiplatform.admin",
+      "roles/cloudfunctions.admin",
+      "roles/dataform.admin",
+      "roles/datalineage.viewer",
+      "roles/iam.serviceAccountAdmin",
+      "roles/iam.serviceAccountTokenCreator",
+      "roles/iam.serviceAccountUser",
+      "roles/logging.configWriter",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/run.invoker",
+      "roles/serviceusage.serviceUsageAdmin",
+      "roles/storage.admin",
+      "roles/workflows.admin"
+    ]
+    udf = [
+      "roles/bigquery.admin",
+      "roles/aiplatform.admin",
+      "roles/cloudfunctions.admin",
+      "roles/dataform.admin",
+      "roles/datalineage.viewer",
+      "roles/iam.serviceAccountAdmin",
+      "roles/iam.serviceAccountTokenCreator",
+      "roles/iam.serviceAccountUser",
+      "roles/logging.configWriter",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/run.invoker",
+      "roles/serviceusage.serviceUsageAdmin",
+      "roles/storage.admin",
+      "roles/workflows.admin"
+    ]
+  }
+
+  int_required_roles = tolist(toset(flatten(values(local.per_module_roles))))
 }
 
 resource "google_service_account" "int_test" {
diff --git a/test/setup/main.tf b/test/setup/main.tf
