main.tf

terraform {
  required_providers {
    google = {
      source = "hashicorp/google"
    }
  }
}

# The default service account applied to all cluster node pools
resource "google_service_account" "cluster_default" {
  account_id   = "${var.name}${local.service_account_suffix}"
  display_name = "${var.name} GKE Default"
  project      = var.project
}

resource "google_project_iam_member" "cluster" {
  for_each = merge({
    # Read access to the project GCR
    "gcr" = "roles/storage.objectViewer"
    # Read access to the project ACR
    "acr" = "roles/artifactregistry.reader"
    # Log writer access
    "log-writer" = "roles/logging.logWriter"
    # Metrics writer access
    "metrics-writer" = "roles/monitoring.metricWriter"
    # Monitoring viewer access
    "monitoring-viewer" = "roles/monitoring.viewer"
  }, var.extra_roles)

  project = var.project
  role    = each.value
  member  = "serviceAccount:${google_service_account.cluster_default.email}"
}

resource "google_service_account_iam_member" "terraform_gke_impersonation" {
  count              = var.service_account_impersonation_email == null ? 0 : 1
  service_account_id = google_service_account.cluster_default.id
  role               = "roles/iam.serviceAccountUser"
  member             = "serviceAccount:${var.service_account_impersonation_email}"
}

locals {
  // Default the node service account suffix to the region so clusters sharing
  // a project (e.g. the same name across regions) don't collide on this
  // project-global resource. Callers can override via var.service_account_suffix.
  service_account_suffix = var.service_account_suffix != null ? var.service_account_suffix : "-${var.region}"

  default_labels = {
    basename(abspath(path.module)) = var.name
    terraform-module               = basename(abspath(path.module))
  }

  // Extract the project that owns the VPC. When network is a full self_link
  // (projects/{project}/global/networks/...) the project is at index [1];
  // bare names imply the cluster's own project.
  network_project = length(split("/", var.network)) > 1 ? split("/", var.network)[1] : var.project

  squad_label = {
    "squad" = var.team
    "team"  = var.team
  }

  product_label = var.product != "" ? {
    product = var.product
  } : {}
}

resource "google_container_cluster" "this" {
  name    = var.name
  project = var.project

  network    = var.network
  subnetwork = var.subnetwork

  location       = var.region
  node_locations = var.zones

  deletion_protection = var.deletion_protection

  enable_intranode_visibility = true

  remove_default_node_pool = true
  initial_node_count       = 1

  resource_labels = var.labels

  # Use Dataplane V2 (eBPF based networking)
  datapath_provider = "ADVANCED_DATAPATH"

  enable_fqdn_network_policy = var.enable_fqdn_network_policy

  networking_mode = "VPC_NATIVE"
  // When both names are null, GKE auto-allocates the secondary pod/service
  // ranges. On a Shared VPC the ranges are pre-created on the host subnet and
  // referenced by name, since GKE cannot create ranges on a subnet it does
  // not own.
  ip_allocation_policy {
    cluster_secondary_range_name  = var.cluster_secondary_range_name
    services_secondary_range_name = var.services_secondary_range_name
  }

  workload_identity_config {
    workload_pool = "${var.project}.svc.id.goog"
  }

  # Application-layer Secrets encryption. When a key is provided, GKE
  # envelope-encrypts Kubernetes Secret objects with the customer-managed key
  # before writing them to etcd. The GKE service agent must hold
  # roles/cloudkms.cryptoKeyEncrypterDecrypter on the key. Destroying the key
  # version while in use renders the cluster state DB undecryptable, so the key
  # must outlive the cluster.
  dynamic "database_encryption" {
    for_each = var.database_encryption_key == null ? [] : ["placeholder"]
    content {
      state    = "ENCRYPTED"
      key_name = var.database_encryption_key
    }
  }

  release_channel {
    # NOTE: Toggle to "RAPID" when we want to start playing with things like gcsfuse
    channel = var.release_channel
  }

  # Configured with separate node_pool resources
  # node_config {}

  # Emitted when node auto-provisioning is enabled OR when an autoscaling
  # profile is set: the profile governs the standard per-node-pool cluster
  # autoscaler's binpacking and scale-down aggressiveness regardless of NAP,
  # so it must be settable with `enabled = false`. The resource_limits and
  # auto_provisioning_defaults sub-blocks configure NAP itself and are only
  # emitted when NAP is enabled.
  dynamic "cluster_autoscaling" {
    for_each = var.cluster_autoscaling == false && var.cluster_autoscaling_profile == null ? [] : ["placeholder"]

    content {
      enabled                       = var.cluster_autoscaling
      default_compute_class_enabled = var.default_compute_class_enabled
      dynamic "resource_limits" {
        for_each = var.cluster_autoscaling ? ["cpu"] : []
        content {
          resource_type = var.cluster_autoscaling_cpu_limits.resource_type
          minimum       = var.cluster_autoscaling_cpu_limits.minimum
          maximum       = var.cluster_autoscaling_cpu_limits.maximum
        }
      }
      dynamic "resource_limits" {
        for_each = var.cluster_autoscaling ? ["memory"] : []
        content {
          resource_type = var.cluster_autoscaling_memory_limits.resource_type
          minimum       = var.cluster_autoscaling_memory_limits.minimum
          maximum       = var.cluster_autoscaling_memory_limits.maximum
        }
      }
      dynamic "resource_limits" {
        for_each = var.cluster_autoscaling ? var.cluster_autoscaling_gpu_limits : []
        content {
          resource_type = resource_limits.value.resource_type
          minimum       = resource_limits.value.minimum
          maximum       = resource_limits.value.maximum
        }
      }
      dynamic "auto_provisioning_defaults" {
        for_each = var.cluster_autoscaling && var.cluster_autoscaling_provisioning_defaults != null ? ["placeholder"] : []

        content {
          service_account = google_service_account.cluster_default.email
          disk_size       = var.cluster_autoscaling_provisioning_defaults.disk_size
          disk_type       = var.cluster_autoscaling_provisioning_defaults.disk_type

          dynamic "shielded_instance_config" {
            for_each = var.cluster_autoscaling_provisioning_defaults.shielded_instance_config == null ? [] : ["placeholder"]

            content {
              enable_secure_boot          = var.cluster_autoscaling_provisioning_defaults.shielded_instance_config.enable_secure_boot
              enable_integrity_monitoring = var.cluster_autoscaling_provisioning_defaults.shielded_instance_config.enable_integrity_monitoring
            }
          }
          dynamic "management" {
            for_each = var.cluster_autoscaling_provisioning_defaults.management == null ? [] : ["placeholder"]

            content {
              auto_upgrade = var.cluster_autoscaling_provisioning_defaults.management.auto_upgrade
              auto_repair  = var.cluster_autoscaling_provisioning_defaults.management.auto_repair
            }
          }
        }
      }
      autoscaling_profile = var.cluster_autoscaling_profile
    }
  }

  master_authorized_networks_config {
    # gcp_public_cidrs_access_enabled = true
    cidr_blocks {
      display_name = "Everywhere"
      cidr_block   = "0.0.0.0/0"
    }

    # TODO: Pin this to https://api.github.com/meta
    # Github recommends against doing this, so maybe there's a more effective way, perhaps a certain scale with a tail?
    # cidr_blocks {}
  }

  private_cluster_config {
    enable_private_nodes = var.enable_private_nodes
    master_global_access_config {
      enabled = true
    }
    # This doesn't do what you think it does
    # private_endpoint_subnetwork = var.subnetwork
  }

  dns_config {
    # Enable more efficient DNS resolution by leveraging the GCP backplane (instead of kube-dns)
    # Technically this adds cloud DNS billing, but the cost is negligible
    # https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns
    cluster_dns       = "CLOUD_DNS"
    cluster_dns_scope = "CLUSTER_SCOPE"
  }

  # TODO: These probably could be configurable
  addons_config {
    http_load_balancing {
      disabled = false
    }
    gke_backup_agent_config {
      enabled = false
    }
    config_connector_config {
      enabled = false
    }
    gcs_fuse_csi_driver_config {
      enabled = true
    }
    dns_cache_config {
      enabled = var.enable_dns_cache
    }
    dynamic "horizontal_pod_autoscaling" {
      for_each = var.disable_horizontal_pod_autoscaling != null ? ["placeholder"] : []
      content {
        disabled = var.disable_horizontal_pod_autoscaling
      }
    }
  }

  monitoring_config {
    enable_components = ["SYSTEM_COMPONENTS", "APISERVER", "SCHEDULER", "CONTROLLER_MANAGER", "STORAGE", "POD"]
    managed_prometheus { enabled = true }

    dynamic "advanced_datapath_observability_config" {
      for_each = var.advanced_datapath_observability_config.enable ? ["placeholder"] : []

      content {
        enable_metrics = var.advanced_datapath_observability_config.enable_metrics
        enable_relay   = var.advanced_datapath_observability_config.enable_relay
      }
    }
  }

  # This can't hurt... right?
  cost_management_config {
    enabled = true
  }

  dynamic "resource_usage_export_config" {
    for_each = var.resource_usage_export_config.bigquery_dataset_id == "" ? [] : ["placeholder"]

    content {
      enable_network_egress_metering       = var.resource_usage_export_config.enable_network_egress_metering
      enable_resource_consumption_metering = var.resource_usage_export_config.enable_resource_consumption_metering
      bigquery_destination {
        dataset_id = var.resource_usage_export_config.bigquery_dataset_id
      }
    }
  }

  timeouts {
    create = "30m"
    update = "30m"
    delete = "30m"
  }

  lifecycle {
    # https://github.com/hashicorp/terraform-provider-google/issues/6901
    ignore_changes = [initial_node_count]
  }

  depends_on = [google_service_account_iam_member.terraform_gke_impersonation]
}

resource "google_container_node_pool" "pools" {
  for_each = var.pools
  provider = google-beta

  name           = each.key
  cluster        = google_container_cluster.this.name
  project        = var.project
  location       = google_container_cluster.this.location
  node_locations = each.value.node_locations != null ? each.value.node_locations : var.zones

  dynamic "network_config" {
    for_each = each.value.network_config != null ? [1] : []
    content {
      enable_private_nodes = each.value.network_config.enable_private_nodes
      create_pod_range     = each.value.network_config.create_pod_range
      pod_ipv4_cidr_block  = each.value.network_config.pod_ipv4_cidr_block
    }
  }

  node_config {
    service_account = google_service_account.cluster_default.email
    image_type      = "COS_CONTAINERD"
    machine_type    = each.value.machine_type
    workload_metadata_config {
      # Run the GKE metadata server on these nodes (required for workload identity)
      mode = "GKE_METADATA"
    }
    metadata = {
      disable-legacy-endpoints = true
      block-project-ssh-keys   = true
    }

    disk_type    = each.value.disk_type
    disk_size_gb = each.value.disk_size

    dynamic "ephemeral_storage_local_ssd_config" {
      for_each = each.value.ephemeral_storage_local_ssd_count > 0 ? [1] : []
      content {
        local_ssd_count = each.value.ephemeral_storage_local_ssd_count
      }
    }

    dynamic "guest_accelerator" {
      for_each = each.value.guest_accelerator != null ? each.value.guest_accelerator : []
      content {
        type  = guest_accelerator.value.type
        count = guest_accelerator.value.count

        dynamic "gpu_driver_installation_config" {
          for_each = guest_accelerator.value.gpu_driver_installation_config
          content {
            gpu_driver_version = gpu_driver_installation_config.value.gpu_driver_version
          }
        }
      }
    }

    # Don't set legacy scopes
    # oauth_scopes = []

    # Enable google vNIC driver
    gvnic {
      enabled = true
    }

    # Enable google container filesystem (required for image streaming)
    gcfs_config {
      enabled = true
    }

    dynamic "sandbox_config" {
      for_each = each.value.gvisor ? [1] : []
      content {
        sandbox_type = "gvisor"
      }
    }

    spot            = each.value.spot
    labels          = each.value.labels
    tags            = each.value.tags
    resource_labels = merge(local.default_labels, local.squad_label, local.product_label, var.labels)

    dynamic "advanced_machine_features" {
      for_each = each.value.enable_nested_virtualization != null ? [1] : []
      content {
        threads_per_core             = 0 // Must be set, but we want the default.
        enable_nested_virtualization = each.value.enable_nested_virtualization
      }
    }

    shielded_instance_config {
      enable_secure_boot          = each.value.enable_secure_boot
      enable_integrity_monitoring = each.value.enable_integrity_monitoring
    }

    dynamic "taint" {
      for_each = each.value.taints
      content {
        key    = taint.value.key
        value  = taint.value.value
        effect = taint.value.effect
      }
    }
  }

  autoscaling {
    min_node_count = each.value.min_node_count
    max_node_count = each.value.max_node_count
  }

  management {
    auto_repair  = true
    auto_upgrade = true
  }
}

# Allow GKE master to hit non 443 ports for webhook/admission controllers
#
# https://github.com/kubernetes/kubernetes/issues/79739
resource "google_compute_firewall" "master_webhook" {
  project = local.network_project
  network = var.network

  # Firewall rules are global, so include the region to keep the name unique
  # when multiple clusters share a project (e.g. same name across regions).
  name        = "${var.name}-${var.region}-master-webhook"
  description = "Allow GKE master to hit non 443 ports for webhook/admission controllers"
  direction   = "INGRESS"

  source_ranges = ["${google_container_cluster.this.endpoint}/32"]
  source_tags   = []
  target_tags   = ["gke-${google_container_cluster.this.name}"]

  allow {
    protocol = "tcp"
    ports = [
      "8443",
      "9443",
      "15017",
    ]
  }

  depends_on = [google_container_cluster.this]
}