Add cloud storage terraform

Author: jpraynaud

mithril-infra/assets/docker/docker-compose-aggregator.yaml

@@ -52,7 +52,8 @@ services:
       - RUN_INTERVAL=60000
       - URL_SNAPSHOT_MANIFEST=https://storage.googleapis.com/${SNAPSHOT_BUCKET_NAME}/snapshots.json
       - SNAPSHOT_STORE_TYPE=local
-      - SNAPSHOT_UPLOADER_TYPE=local
+      - SNAPSHOT_UPLOADER_TYPE=gcp
+      - SNAPSHOT_BUCKET_NAME=${SNAPSHOT_BUCKET_NAME}
       - DATA_STORES_DIRECTORY=/mithril-aggregator/mithril/stores
       - STORE_RETENTION_LIMIT=5
       - CARDANO_NODE_SOCKET_PATH=/ipc/node.socket

mithril-infra/main.cloud-storage.tf

@@ -0,0 +1,47 @@
+resource "google_storage_bucket" "cloud_storage" {
+  name          = "${local.environment_name}-cs"
+  location      = var.google_region
+  force_destroy = true
+
+  lifecycle_rule {
+    condition {
+      age = var.google_storage_bucket_max_age
+    }
+    action {
+      type = "Delete"
+    }
+  }
+}
+
+resource "google_service_account" "cloud_storage" {
+  account_id   = "${local.environment_name}-cs-sa"
+  display_name = "${local.environment_name}-cs-sa"
+  description  = "${local.environment_name} cloud storage service account"
+}
+
+resource "google_service_account_key" "cloud_storage" {
+  service_account_id = google_service_account.cloud_storage.name
+  public_key_type    = "TYPE_X509_PEM_FILE"
+}
+
+locals {
+  google_cloud_storage_credentials_json = base64decode(google_service_account_key.cloud_storage.private_key)
+}
+
+resource "google_storage_bucket_iam_member" "cloud_storage_viewer" {
+  bucket = google_storage_bucket.cloud_storage.name
+  role   = "roles/storage.objectViewer"
+  member = "serviceAccount:${google_service_account.cloud_storage.email}"
+}
+
+resource "google_storage_bucket_iam_member" "cloud_storage_creator" {
+  bucket = google_storage_bucket.cloud_storage.name
+  role   = "roles/storage.objectCreator"
+  member = "serviceAccount:${google_service_account.cloud_storage.email}"
+}
+
+resource "google_storage_bucket_iam_member" "legacy_bucket_writer" {
+  bucket = google_storage_bucket.cloud_storage.name
+  role   = "roles/storage.legacyBucketWriter"
+  member = "serviceAccount:${google_service_account.cloud_storage.email}"
+}

mithril-infra/mithril.aggregator.tf

@@ -30,7 +30,8 @@ resource "null_resource" "mithril_aggregator" {
       "export NETWORK=${var.cardano_network}",
       "export IMAGE_ID=${var.mithril_image_id}",
       "export AGGREGATOR_HOST=${local.mithril_aggregator_host}",
-      "export GOOGLE_APPLICATION_CREDENTIALS_JSON='${var.google_application_credentials_json}'",
+      "export GOOGLE_APPLICATION_CREDENTIALS_JSON='${local.google_cloud_storage_credentials_json}'",
+      "export SNAPSHOT_BUCKET_NAME='${google_storage_bucket.cloud_storage.name}'",
       "export GENESIS_VERIFICATION_KEY=$(wget -q -O - ${var.mithril_genesis_verification_key_url})",
       "export GENESIS_SECRET_KEY='${var.mithril_genesis_secret_key}'",
       "export CURRENT_UID=$(id -u)",

mithril-infra/output.tf

@@ -6,6 +6,10 @@ output "aggregator_endpoint" {
   value = local.mithril_aggregator_endpoint_url
 }
 
+output "storage_bucket" {
+  value = google_storage_bucket.cloud_storage.name
+}
+
 output "external-ip" {
   value = google_compute_address.mithril-external-address.address
 }

mithril-infra/variables.tf

@@ -38,9 +38,10 @@ variable "google_service_credentials_json" {
   description = "The credentials of the GCP service account"
 }
 
-variable "google_application_credentials_json" {
-  type        = string
-  description = "Service account JSON key file used by aggregator to upload files to gcloud storage"
+variable "google_storage_bucket_max_age" {
+  type        = number
+  description = "Number of days after which an object in the storage bucket expires"
+  default     = 14
 }
 
 locals {