feat: introduce new GCS and Spanner backends

Please apply the spanner database schema via `gcloud --project [PROJECT_ID] spanner databases ddl update --instance abfs abfs --ddl-file ../../files/schemas/0.0.31-schema.sql` once `terraform apply` is finished. The final command can be seen in the outputs.

PiperOrigin-RevId: 773700345

Author: Android Build Filesystem (ABFS) Team

README.md

@@ -26,7 +26,7 @@ This module is divided into two distinct submodules which creates the following:
 Systemd config and scripts are created on the VM by the built-in cloud-init
 tool, which pulls a cloud-init config file from the metatdata server.
 
-The cloud-init config is generated in the server's [main.tf](./modules/server/main.tf) and the uploaders [main.tf](./modules/uploaders/main.tf) and is assembled from this [systemd configuration](./cloud-init) and [associated bash scripts](./cloud-init/scripts).
+The cloud-init config is generated in the server's [main.tf](./modules/server/main.tf) and the uploaders [main.tf](./modules/uploaders/main.tf) and is assembled from this [systemd configuration](./files/cloud-init) and [associated bash scripts](./files/cloud-init/scripts).
 
 Systemd is highly configurable and parallelized to allow for optimal and rapid VM
 startup. The visualization below shows the order and dependencies of the relevant

examples/simple/iam.tf

@@ -23,12 +23,15 @@ module "project-iam-bindings" {
 
   projects = [data.google_project.project.project_id]
   mode     = "authoritative"
+
   bindings = {
     "roles/artifactregistry.reader"             = [data.google_service_account.abfs.member],
     "roles/logging.logWriter"                   = [data.google_service_account.abfs.member],
     "roles/monitoring.metricWriter"             = [data.google_service_account.abfs.member],
     "roles/monitoring.viewer"                   = [data.google_service_account.abfs.member],
+    "roles/spanner.databaseUser"                = [data.google_service_account.abfs.member],
     "roles/stackdriver.resourceMetadata.writer" = [data.google_service_account.abfs.member],
+    "roles/storage.objectAdmin"                 = [data.google_service_account.abfs.member],
   }
 
   depends_on = [

examples/simple/main.tf

@@ -12,21 +12,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-module "abfs-deployment" {
+moved {
+  from = module.abfs-deployment
+  to   = module.abfs_server
+}
+
+moved {
+  from = module.abfs-uploaders
+  to   = module.abfs_uploaders
+}
+
+module "abfs_server" {
   source = "../../modules/server"
 
-  project_id            = var.project_id
-  zone                  = var.zone
-  service_account_email = data.google_service_account.abfs.email
-  subnetwork            = module.abfs-vpc.subnets["${var.region}/abfs-subnet"].name
-  abfs_docker_image_uri = var.abfs_docker_image_uri
-  abfs_license          = var.abfs_license
+  project_id                   = data.google_project.project.project_id
+  zone                         = var.zone
+  service_account_email        = data.google_service_account.abfs.email
+  subnetwork                   = module.abfs-vpc.subnets["${var.region}/abfs-subnet"].name
+  abfs_docker_image_uri        = var.abfs_docker_image_uri
+  abfs_license                 = var.abfs_license
+  abfs_bucket_location         = var.abfs_bucket_location
+  abfs_spanner_instance_config = var.abfs_spanner_instance_config
 }
 
-module "abfs-uploaders" {
+module "abfs_uploaders" {
   source = "../../modules/uploaders"
 
-  project_id                           = var.project_id
+  project_id                           = data.google_project.project.project_id
   zone                                 = var.zone
   service_account_email                = data.google_service_account.abfs.email
   subnetwork                           = module.abfs-vpc.subnets["${var.region}/abfs-subnet"].name
@@ -36,7 +48,7 @@ module "abfs-uploaders" {
   abfs_manifest_project_name           = var.abfs_manifest_project_name
   abfs_manifest_file                   = var.abfs_manifest_file
   abfs_license                         = var.abfs_license
-  abfs_server_name                     = module.abfs-deployment.abfs_server_name
+  abfs_server_name                     = module.abfs_server.abfs_server_name
 }
 
 module "monitoring" {

examples/simple/network.tf

@@ -19,6 +19,7 @@ module "abfs-vpc" {
   project_id   = data.google_project.project.project_id
   network_name = "abfs-network"
   routing_mode = "GLOBAL"
+
   firewall_rules = [
     {
       description             = "Allow egress to Google APIs via Private Google Access"
@@ -76,6 +77,7 @@ module "abfs-vpc" {
       ]
     }
   ]
+
   subnets = [
     {
       subnet_name           = "abfs-subnet"

examples/simple/outputs.tf

@@ -20,3 +20,8 @@ output "license_information" {
     service_account_unique_id = data.google_service_account.abfs.unique_id
   }
 }
+
+output "spanner_database_schema_creation" {
+  description = "The CLI command for creating the Spanner database schema"
+  value       = "gcloud --project ${data.google_project.project.project_id} spanner databases ddl update --instance ${module.abfs_server.abfs_spanner_instance.name} ${module.abfs_server.abfs_spanner_database.name} --ddl-file ${path.module}/../../files/schemas/0.0.31-schema.sql"
+}

examples/simple/services.tf

@@ -32,7 +32,8 @@ module "project-services" {
   activate_apis = [
     "compute.googleapis.com",
     "dns.googleapis.com",
-    "monitoring.googleapis.com"
+    "monitoring.googleapis.com",
+    "spanner.googleapis.com",
   ]
 
   depends_on = [

examples/simple/variables.tf

@@ -29,7 +29,7 @@ variable "zone" {
 
 variable "abfs_docker_image_uri" {
   type        = string
-  description = "Docker image URI for main ABFS server"
+  description = "Docker image URI for ABFS"
 }
 
 variable "abfs_gerrit_uploader_git_branch" {
@@ -70,3 +70,13 @@ variable "abfs_service_account_id" {
   type        = string
   description = "ABFS service account ID (e.g. abfs@<project-id>.iam.gserviceaccount.com)"
 }
+
+variable "abfs_bucket_location" {
+  type        = string
+  description = "The location of the ABFS bucket (https://cloud.google.com/storage/docs/locations)."
+}
+
+variable "abfs_spanner_instance_config" {
+  type        = string
+  description = "The name of the instance's configuration (similar but not quite the same as a region) which defines the geographic placement and replication of your ABFS database in this instance (https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/spanner_instance.html#config-1)."
+}