diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 3fac2124f7..f8e1a14bb7 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -83,6 +83,9 @@ resource "google_container_cluster" "primary" { for_each = local.confidential_node_config content { enabled = confidential_nodes.value.enabled + {% if autopilot_cluster != true %} + confidential_instance_type = lookup(var.node_pools[0], "confidential_instance_type", null) + {% endif %} } } @@ -594,6 +597,8 @@ resource "google_container_cluster" "primary" { min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) disk_type = lookup(var.node_pools[0], "disk_type", null) + preemptible = lookup(var.node_pools[0], "preemptible", false) + spot = lookup(var.node_pools[0], "spot", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", null) != null ? [var.node_pools[0].enable_gcfs] : [] content { @@ -601,6 +606,30 @@ resource "google_container_cluster" "primary" { } } + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[0], "accelerator_count", 0) > 0 ? [1] : [] + content { + type = lookup(var.node_pools[0], "accelerator_type", "") + count = lookup(var.node_pools[0], "accelerator_count", 0) + gpu_partition_size = lookup(var.node_pools[0], "gpu_partition_size", null) + + dynamic "gpu_driver_installation_config" { + for_each = lookup(var.node_pools[0], "gpu_driver_version", "") != "" ? [1] : [] + content { + gpu_driver_version = lookup(var.node_pools[0], "gpu_driver_version", "") + } + } + + dynamic "gpu_sharing_config" { + for_each = lookup(var.node_pools[0], "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(var.node_pools[0], "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(var.node_pools[0], "max_shared_clients_per_gpu", 2) + } + } + } + } + dynamic "gvnic" { for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] content { @@ -1300,9 +1329,10 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } diff --git a/cluster.tf b/cluster.tf index eb9ca74a5b..86967d4887 100644 --- a/cluster.tf +++ b/cluster.tf @@ -76,7 +76,8 @@ resource "google_container_cluster" "primary" { dynamic "confidential_nodes" { for_each = local.confidential_node_config content { - enabled = confidential_nodes.value.enabled + enabled = confidential_nodes.value.enabled + confidential_instance_type = lookup(var.node_pools[0], "confidential_instance_type", null) } } @@ -454,6 +455,8 @@ resource "google_container_cluster" "primary" { min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) disk_type = lookup(var.node_pools[0], "disk_type", null) + preemptible = lookup(var.node_pools[0], "preemptible", false) + spot = lookup(var.node_pools[0], "spot", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", null) != null ? [var.node_pools[0].enable_gcfs] : [] content { @@ -461,6 +464,30 @@ resource "google_container_cluster" "primary" { } } + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[0], "accelerator_count", 0) > 0 ? [1] : [] + content { + type = lookup(var.node_pools[0], "accelerator_type", "") + count = lookup(var.node_pools[0], "accelerator_count", 0) + gpu_partition_size = lookup(var.node_pools[0], "gpu_partition_size", null) + + dynamic "gpu_driver_installation_config" { + for_each = lookup(var.node_pools[0], "gpu_driver_version", "") != "" ? [1] : [] + content { + gpu_driver_version = lookup(var.node_pools[0], "gpu_driver_version", "") + } + } + + dynamic "gpu_sharing_config" { + for_each = lookup(var.node_pools[0], "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(var.node_pools[0], "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(var.node_pools[0], "max_shared_clients_per_gpu", 2) + } + } + } + } + dynamic "gvnic" { for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] content { @@ -984,9 +1011,10 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } @@ -1317,9 +1345,10 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } diff --git a/examples/confidential_gpu_public/README.md b/examples/confidential_gpu_public/README.md new file mode 100644 index 0000000000..2721f54755 --- /dev/null +++ b/examples/confidential_gpu_public/README.md @@ -0,0 +1,43 @@ +# Confidential GPU GKE Cluster + +This example illustrates how to instantiate the Beta Public Cluster module +with confidential nodes enabled, database encrypted with KMS key +and encrypted GPU Workload with NVIDIA Confidential Computing. +This module also installs the NVIDIA drivers on the GPU, so it's +ready to receive workloads. +See more: https://cloud.google.com/kubernetes-engine/docs/how-to/gpus-confidential-nodes. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to host the cluster in. | `string` | n/a | yes | +| region | The region to host the cluster in. | `string` | `"us-central1"` | no | +| zones | The zones to host the nodes in. The nodes must be in a zone that supports NVIDIA Confidential Computing. For more information, [view supported zones](https://cloud.google.com/confidential-computing/confidential-vm/docs/supported-configurations#nvidia-confidential-computing_1). | `list(string)` | 
[
  "us-central1-a"
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | The cluster ca certificate (base64 encoded). | +| cluster\_name | Cluster name. | +| keyring | The name of the keyring. | +| kms\_key\_name | KMS Key Name. | +| kubernetes\_endpoint | The cluster endpoint. | +| location | n/a | +| master\_kubernetes\_version | Kubernetes version of the master. | +| network\_name | The name of the VPC being created. | +| project\_id | The project ID the cluster is in. | +| region | The region in which the cluster resides. | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created. | +| zones | List of zones in which the cluster resides. | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/confidential_gpu_public/kms.tf b/examples/confidential_gpu_public/kms.tf new file mode 100644 index 0000000000..c745524ba8 --- /dev/null +++ b/examples/confidential_gpu_public/kms.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + key_name = "gke-key-${random_string.suffix.result}" +} + +module "kms" { + source = "terraform-google-modules/kms/google" + version = "~> 4.0" + project_id = var.project_id + location = var.region + keyring = "gke-keyring-${random_string.suffix.result}" + keys = [local.key_name] + prevent_destroy = false +} + +resource "google_project_service_identity" "container_identity" { + provider = google-beta + project = var.project_id + service = "container.googleapis.com" +} + +resource "google_kms_crypto_key_iam_member" "sm_sa_encrypter_decrypter" { + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = google_project_service_identity.container_identity.member + crypto_key_id = module.kms.keys[local.key_name] +} diff --git a/examples/confidential_gpu_public/main.tf b/examples/confidential_gpu_public/main.tf new file mode 100644 index 0000000000..876f8463a0 --- /dev/null +++ b/examples/confidential_gpu_public/main.tf @@ -0,0 +1,110 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "confidential-gpu" + network_name = "confidential-gpu-network-${random_string.suffix.result}" + subnet_name = "confidential-gpu-subnet" + master_auth_subnetwork = "confidential-gpu-master-subnet" + pods_range_name = "ip-range-pods-${random_string.suffix.result}" + svc_range_name = "ip-range-svc-${random_string.suffix.result}" + subnet_names = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]] +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +data "google_project" "main" { + project_id = var.project_id +} + +resource "google_kms_crypto_key_iam_member" "main" { + crypto_key_id = module.kms.keys[local.key_name] + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.main.number}@compute-system.iam.gserviceaccount.com" +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster" + version = "~> 37.0" + + project_id = var.project_id + name = "${local.cluster_type}-cluster-${random_string.suffix.result}" + region = var.region + zones = var.zones + network = module.gcp-network.network_name + subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name + create_service_account = false + initial_node_count = 1 + remove_default_node_pool = true + disable_legacy_metadata_endpoints = false + deletion_protection = false + service_account = "default" + logging_variant = "MAX_THROUGHPUT" + dns_allow_external_traffic = true + + enable_confidential_nodes = true + + database_encryption = [ + { + "key_name" : module.kms.keys[local.key_name], + "state" : "ENCRYPTED" + } + ] + + node_pools = [ + { + name = "default" + machine_type = "a3-highgpu-1g" + confidential_instance_type = "TDX" + spot = true + disk_type = "hyperdisk-balanced" + boot_disk_kms_key = module.kms.keys[local.key_name] + enable_confidential_storage = true + accelerator_count = 1 + accelerator_type = "nvidia-h100-80gb" + gpu_driver_version = "INSTALLATION_DISABLED" + node_locations = join(",", var.zones) + local_ssd_ephemeral_storage_count = 2 + }, + ] +} + +module "kubectl" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.0" + + project_id = var.project_id + cluster_name = module.gke.name + cluster_location = module.gke.location + module_depends_on = [module.gke.endpoint] + kubectl_create_command = "kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/refs/heads/master/nvidia-driver-installer/cos/daemonset-confidential.yaml" + kubectl_destroy_command = "kubectl delete -f https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/refs/heads/master/nvidia-driver-installer/cos/daemonset-confidential.yaml" + skip_download = true +} diff --git a/examples/confidential_gpu_public/network.tf b/examples/confidential_gpu_public/network.tf new file mode 100644 index 0000000000..bdcbed3092 --- /dev/null +++ b/examples/confidential_gpu_public/network.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = "~> 11.0" + + project_id = var.project_id + network_name = local.network_name + routing_mode = "GLOBAL" + + subnets = [ + { + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = var.region + subnet_private_access = true + }, + { + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" + subnet_region = var.region + subnet_private_access = true + }, + ] + + secondary_ranges = { + (local.subnet_name) = [ + { + range_name = local.pods_range_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = local.svc_range_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/confidential_gpu_public/outputs.tf b/examples/confidential_gpu_public/outputs.tf new file mode 100644 index 0000000000..bd3610c9d8 --- /dev/null +++ b/examples/confidential_gpu_public/outputs.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint." + sensitive = true + value = module.gke.endpoint +} + +output "cluster_name" { + description = "Cluster name." + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master." + value = module.gke.master_version +} + +output "ca_certificate" { + description = "The cluster ca certificate (base64 encoded)." + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + +output "network_name" { + description = "The name of the VPC being created." + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created." + value = module.gcp-network.subnets_names +} + +output "region" { + description = "The region in which the cluster resides." + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides." + value = module.gke.zones +} + +output "project_id" { + description = "The project ID the cluster is in." + value = var.project_id +} + +output "keyring" { + description = "The name of the keyring." + value = module.kms.keyring +} + +output "kms_key_name" { + description = "KMS Key Name." + value = module.kms.keys[local.key_name] +} diff --git a/examples/confidential_gpu_public/variables.tf b/examples/confidential_gpu_public/variables.tf new file mode 100644 index 0000000000..add12fbbda --- /dev/null +++ b/examples/confidential_gpu_public/variables.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project ID to host the cluster in." +} + +variable "region" { + type = string + description = "The region to host the cluster in." + default = "us-central1" +} + +variable "zones" { + type = list(string) + description = "The zones to host the nodes in. The nodes must be in a zone that supports NVIDIA Confidential Computing. For more information, [view supported zones](https://cloud.google.com/confidential-computing/confidential-vm/docs/supported-configurations#nvidia-confidential-computing_1)." + default = ["us-central1-a"] +} diff --git a/examples/confidential_gpu_public/versions.tf b/examples/confidential_gpu_public/versions.tf new file mode 100644 index 0000000000..8279b4edc8 --- /dev/null +++ b/examples/confidential_gpu_public/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.3" + required_providers { + google = { + source = "hashicorp/google" + } + google-beta = { + source = "hashicorp/google-beta" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + } +} diff --git a/metadata.yaml b/metadata.yaml index cd845d42d2..c2415612e5 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -76,6 +76,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/auth/metadata.yaml b/modules/auth/metadata.yaml index e426b1b803..9e986107be 100644 --- a/modules/auth/metadata.yaml +++ b/modules/auth/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/beta-autopilot-private-cluster/metadata.yaml b/modules/beta-autopilot-private-cluster/metadata.yaml index daf91311b5..5e51290dd0 100644 --- a/modules/beta-autopilot-private-cluster/metadata.yaml +++ b/modules/beta-autopilot-private-cluster/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/beta-autopilot-public-cluster/metadata.yaml b/modules/beta-autopilot-public-cluster/metadata.yaml index 9bbfce2333..d5d214ffc8 100644 --- a/modules/beta-autopilot-public-cluster/metadata.yaml +++ b/modules/beta-autopilot-public-cluster/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index c3cfd5a8b4..63e86d9faf 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -76,7 +76,8 @@ resource "google_container_cluster" "primary" { dynamic "confidential_nodes" { for_each = local.confidential_node_config content { - enabled = confidential_nodes.value.enabled + enabled = confidential_nodes.value.enabled + confidential_instance_type = lookup(var.node_pools[0], "confidential_instance_type", null) } } @@ -491,6 +492,8 @@ resource "google_container_cluster" "primary" { min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) disk_type = lookup(var.node_pools[0], "disk_type", null) + preemptible = lookup(var.node_pools[0], "preemptible", false) + spot = lookup(var.node_pools[0], "spot", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", null) != null ? [var.node_pools[0].enable_gcfs] : [] content { @@ -498,6 +501,30 @@ resource "google_container_cluster" "primary" { } } + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[0], "accelerator_count", 0) > 0 ? [1] : [] + content { + type = lookup(var.node_pools[0], "accelerator_type", "") + count = lookup(var.node_pools[0], "accelerator_count", 0) + gpu_partition_size = lookup(var.node_pools[0], "gpu_partition_size", null) + + dynamic "gpu_driver_installation_config" { + for_each = lookup(var.node_pools[0], "gpu_driver_version", "") != "" ? [1] : [] + content { + gpu_driver_version = lookup(var.node_pools[0], "gpu_driver_version", "") + } + } + + dynamic "gpu_sharing_config" { + for_each = lookup(var.node_pools[0], "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(var.node_pools[0], "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(var.node_pools[0], "max_shared_clients_per_gpu", 2) + } + } + } + } + dynamic "gvnic" { for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] content { @@ -1150,9 +1177,10 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } @@ -1497,9 +1525,10 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } diff --git a/modules/beta-private-cluster-update-variant/metadata.yaml b/modules/beta-private-cluster-update-variant/metadata.yaml index eda6878b2d..10df71bbc8 100644 --- a/modules/beta-private-cluster-update-variant/metadata.yaml +++ b/modules/beta-private-cluster-update-variant/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 603d388abe..8ac3cac674 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -76,7 +76,8 @@ resource "google_container_cluster" "primary" { dynamic "confidential_nodes" { for_each = local.confidential_node_config content { - enabled = confidential_nodes.value.enabled + enabled = confidential_nodes.value.enabled + confidential_instance_type = lookup(var.node_pools[0], "confidential_instance_type", null) } } @@ -491,6 +492,8 @@ resource "google_container_cluster" "primary" { min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) disk_type = lookup(var.node_pools[0], "disk_type", null) + preemptible = lookup(var.node_pools[0], "preemptible", false) + spot = lookup(var.node_pools[0], "spot", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", null) != null ? [var.node_pools[0].enable_gcfs] : [] content { @@ -498,6 +501,30 @@ resource "google_container_cluster" "primary" { } } + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[0], "accelerator_count", 0) > 0 ? [1] : [] + content { + type = lookup(var.node_pools[0], "accelerator_type", "") + count = lookup(var.node_pools[0], "accelerator_count", 0) + gpu_partition_size = lookup(var.node_pools[0], "gpu_partition_size", null) + + dynamic "gpu_driver_installation_config" { + for_each = lookup(var.node_pools[0], "gpu_driver_version", "") != "" ? [1] : [] + content { + gpu_driver_version = lookup(var.node_pools[0], "gpu_driver_version", "") + } + } + + dynamic "gpu_sharing_config" { + for_each = lookup(var.node_pools[0], "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(var.node_pools[0], "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(var.node_pools[0], "max_shared_clients_per_gpu", 2) + } + } + } + } + dynamic "gvnic" { for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] content { @@ -1064,9 +1091,10 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } @@ -1410,9 +1438,10 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } diff --git a/modules/beta-private-cluster/metadata.yaml b/modules/beta-private-cluster/metadata.yaml index 501111fdad..caf6ccb4f4 100644 --- a/modules/beta-private-cluster/metadata.yaml +++ b/modules/beta-private-cluster/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index d521ca8ce1..49c762be8e 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -76,7 +76,8 @@ resource "google_container_cluster" "primary" { dynamic "confidential_nodes" { for_each = local.confidential_node_config content { - enabled = confidential_nodes.value.enabled + enabled = confidential_nodes.value.enabled + confidential_instance_type = lookup(var.node_pools[0], "confidential_instance_type", null) } } @@ -491,6 +492,8 @@ resource "google_container_cluster" "primary" { min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) disk_type = lookup(var.node_pools[0], "disk_type", null) + preemptible = lookup(var.node_pools[0], "preemptible", false) + spot = lookup(var.node_pools[0], "spot", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", null) != null ? [var.node_pools[0].enable_gcfs] : [] content { @@ -498,6 +501,30 @@ resource "google_container_cluster" "primary" { } } + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[0], "accelerator_count", 0) > 0 ? [1] : [] + content { + type = lookup(var.node_pools[0], "accelerator_type", "") + count = lookup(var.node_pools[0], "accelerator_count", 0) + gpu_partition_size = lookup(var.node_pools[0], "gpu_partition_size", null) + + dynamic "gpu_driver_installation_config" { + for_each = lookup(var.node_pools[0], "gpu_driver_version", "") != "" ? [1] : [] + content { + gpu_driver_version = lookup(var.node_pools[0], "gpu_driver_version", "") + } + } + + dynamic "gpu_sharing_config" { + for_each = lookup(var.node_pools[0], "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(var.node_pools[0], "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(var.node_pools[0], "max_shared_clients_per_gpu", 2) + } + } + } + } + dynamic "gvnic" { for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] content { @@ -1128,9 +1155,10 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } @@ -1475,9 +1503,10 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } diff --git a/modules/beta-public-cluster-update-variant/metadata.yaml b/modules/beta-public-cluster-update-variant/metadata.yaml index a4feebd885..6fa9d4f225 100644 --- a/modules/beta-public-cluster-update-variant/metadata.yaml +++ b/modules/beta-public-cluster-update-variant/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 0379a3f001..bef081299e 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -76,7 +76,8 @@ resource "google_container_cluster" "primary" { dynamic "confidential_nodes" { for_each = local.confidential_node_config content { - enabled = confidential_nodes.value.enabled + enabled = confidential_nodes.value.enabled + confidential_instance_type = lookup(var.node_pools[0], "confidential_instance_type", null) } } @@ -491,6 +492,8 @@ resource "google_container_cluster" "primary" { min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) disk_type = lookup(var.node_pools[0], "disk_type", null) + preemptible = lookup(var.node_pools[0], "preemptible", false) + spot = lookup(var.node_pools[0], "spot", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", null) != null ? [var.node_pools[0].enable_gcfs] : [] content { @@ -498,6 +501,30 @@ resource "google_container_cluster" "primary" { } } + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[0], "accelerator_count", 0) > 0 ? [1] : [] + content { + type = lookup(var.node_pools[0], "accelerator_type", "") + count = lookup(var.node_pools[0], "accelerator_count", 0) + gpu_partition_size = lookup(var.node_pools[0], "gpu_partition_size", null) + + dynamic "gpu_driver_installation_config" { + for_each = lookup(var.node_pools[0], "gpu_driver_version", "") != "" ? [1] : [] + content { + gpu_driver_version = lookup(var.node_pools[0], "gpu_driver_version", "") + } + } + + dynamic "gpu_sharing_config" { + for_each = lookup(var.node_pools[0], "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(var.node_pools[0], "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(var.node_pools[0], "max_shared_clients_per_gpu", 2) + } + } + } + } + dynamic "gvnic" { for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] content { @@ -1042,9 +1069,10 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } @@ -1388,9 +1416,10 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } diff --git a/modules/beta-public-cluster/metadata.yaml b/modules/beta-public-cluster/metadata.yaml index cd93cb5f55..109e1e4ab1 100644 --- a/modules/beta-public-cluster/metadata.yaml +++ b/modules/beta-public-cluster/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/binary-authorization/metadata.yaml b/modules/binary-authorization/metadata.yaml index d509b4a08e..0a4e9f173d 100644 --- a/modules/binary-authorization/metadata.yaml +++ b/modules/binary-authorization/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/fleet-app-operator-permissions/metadata.yaml b/modules/fleet-app-operator-permissions/metadata.yaml index 981260c6d9..b22024c116 100644 --- a/modules/fleet-app-operator-permissions/metadata.yaml +++ b/modules/fleet-app-operator-permissions/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/fleet-membership/metadata.yaml b/modules/fleet-membership/metadata.yaml index 7994816aee..06bc2f099c 100644 --- a/modules/fleet-membership/metadata.yaml +++ b/modules/fleet-membership/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/gke-autopilot-cluster/metadata.yaml b/modules/gke-autopilot-cluster/metadata.yaml index 9a9ddf02ed..09eb8812ff 100644 --- a/modules/gke-autopilot-cluster/metadata.yaml +++ b/modules/gke-autopilot-cluster/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/gke-node-pool/metadata.yaml b/modules/gke-node-pool/metadata.yaml index cd29dee409..e95d9f6331 100644 --- a/modules/gke-node-pool/metadata.yaml +++ b/modules/gke-node-pool/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/gke-standard-cluster/metadata.yaml b/modules/gke-standard-cluster/metadata.yaml index 3279346661..a9ead6515b 100644 --- a/modules/gke-standard-cluster/metadata.yaml +++ b/modules/gke-standard-cluster/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/hub-legacy/metadata.yaml b/modules/hub-legacy/metadata.yaml index e4360149c8..037fcdaff2 100644 --- a/modules/hub-legacy/metadata.yaml +++ b/modules/hub-legacy/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 14bff26bd5..cade51de5b 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -76,7 +76,8 @@ resource "google_container_cluster" "primary" { dynamic "confidential_nodes" { for_each = local.confidential_node_config content { - enabled = confidential_nodes.value.enabled + enabled = confidential_nodes.value.enabled + confidential_instance_type = lookup(var.node_pools[0], "confidential_instance_type", null) } } @@ -454,6 +455,8 @@ resource "google_container_cluster" "primary" { min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) disk_type = lookup(var.node_pools[0], "disk_type", null) + preemptible = lookup(var.node_pools[0], "preemptible", false) + spot = lookup(var.node_pools[0], "spot", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", null) != null ? [var.node_pools[0].enable_gcfs] : [] content { @@ -461,6 +464,30 @@ resource "google_container_cluster" "primary" { } } + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[0], "accelerator_count", 0) > 0 ? [1] : [] + content { + type = lookup(var.node_pools[0], "accelerator_type", "") + count = lookup(var.node_pools[0], "accelerator_count", 0) + gpu_partition_size = lookup(var.node_pools[0], "gpu_partition_size", null) + + dynamic "gpu_driver_installation_config" { + for_each = lookup(var.node_pools[0], "gpu_driver_version", "") != "" ? [1] : [] + content { + gpu_driver_version = lookup(var.node_pools[0], "gpu_driver_version", "") + } + } + + dynamic "gpu_sharing_config" { + for_each = lookup(var.node_pools[0], "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(var.node_pools[0], "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(var.node_pools[0], "max_shared_clients_per_gpu", 2) + } + } + } + } + dynamic "gvnic" { for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] content { @@ -1091,9 +1118,10 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } @@ -1425,9 +1453,10 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } diff --git a/modules/private-cluster-update-variant/metadata.yaml b/modules/private-cluster-update-variant/metadata.yaml index d2a5276ff8..1ffa86d85d 100644 --- a/modules/private-cluster-update-variant/metadata.yaml +++ b/modules/private-cluster-update-variant/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index bfc3b1b0b5..b51af00552 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -76,7 +76,8 @@ resource "google_container_cluster" "primary" { dynamic "confidential_nodes" { for_each = local.confidential_node_config content { - enabled = confidential_nodes.value.enabled + enabled = confidential_nodes.value.enabled + confidential_instance_type = lookup(var.node_pools[0], "confidential_instance_type", null) } } @@ -454,6 +455,8 @@ resource "google_container_cluster" "primary" { min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) disk_type = lookup(var.node_pools[0], "disk_type", null) + preemptible = lookup(var.node_pools[0], "preemptible", false) + spot = lookup(var.node_pools[0], "spot", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", null) != null ? [var.node_pools[0].enable_gcfs] : [] content { @@ -461,6 +464,30 @@ resource "google_container_cluster" "primary" { } } + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[0], "accelerator_count", 0) > 0 ? [1] : [] + content { + type = lookup(var.node_pools[0], "accelerator_type", "") + count = lookup(var.node_pools[0], "accelerator_count", 0) + gpu_partition_size = lookup(var.node_pools[0], "gpu_partition_size", null) + + dynamic "gpu_driver_installation_config" { + for_each = lookup(var.node_pools[0], "gpu_driver_version", "") != "" ? [1] : [] + content { + gpu_driver_version = lookup(var.node_pools[0], "gpu_driver_version", "") + } + } + + dynamic "gpu_sharing_config" { + for_each = lookup(var.node_pools[0], "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(var.node_pools[0], "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(var.node_pools[0], "max_shared_clients_per_gpu", 2) + } + } + } + } + dynamic "gvnic" { for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] content { @@ -1006,9 +1033,10 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } @@ -1339,9 +1367,10 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [{ enabled = each.value.enable_confidential_nodes, confidential_instance_type = lookup(each.value, "confidential_instance_type", null) }] : [] content { - enabled = confidential_nodes.value + enabled = confidential_nodes.enabled + confidential_instance_type = confidential_nodes.confidential_instance_type } } diff --git a/modules/private-cluster/metadata.yaml b/modules/private-cluster/metadata.yaml index 752648f672..cb78868ffc 100644 --- a/modules/private-cluster/metadata.yaml +++ b/modules/private-cluster/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/safer-cluster-update-variant/metadata.yaml b/modules/safer-cluster-update-variant/metadata.yaml index 186fb361a7..48274b396c 100644 --- a/modules/safer-cluster-update-variant/metadata.yaml +++ b/modules/safer-cluster-update-variant/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/safer-cluster/metadata.yaml b/modules/safer-cluster/metadata.yaml index d5ca5fa743..e1d786a562 100644 --- a/modules/safer-cluster/metadata.yaml +++ b/modules/safer-cluster/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/modules/workload-identity/metadata.yaml b/modules/workload-identity/metadata.yaml index bb6d3efbf2..274456a0ab 100644 --- a/modules/workload-identity/metadata.yaml +++ b/modules/workload-identity/metadata.yaml @@ -36,6 +36,8 @@ spec: location: examples/autopilot_private_firewalls - name: confidential_autopilot_private location: examples/confidential_autopilot_private + - name: confidential_gpu_public + location: examples/confidential_gpu_public - name: confidential_safer_cluster location: examples/confidential_safer_cluster - name: deploy_service diff --git a/test/integration/confidential_gpu_public/confidential_gpu_public_test.go b/test/integration/confidential_gpu_public/confidential_gpu_public_test.go new file mode 100644 index 0000000000..2df4e95167 --- /dev/null +++ b/test/integration/confidential_gpu_public/confidential_gpu_public_test.go @@ -0,0 +1,99 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confidential_gpu_public + +import ( + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" +) + +func TestConfidentialGpuPublic(t *testing.T) { + projectID := testutils.GetTestProjectFromSetup(t, 1) + bpt := tft.NewTFBlueprintTest(t, + tft.WithVars(map[string]interface{}{"project_id": projectID}), + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + testutils.TGKEVerify(t, bpt, assert) + + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + serviceAccount := bpt.GetStringOutput("service_account") + keyName := strings.Split(bpt.GetStringOutput("kms_key_name"), "/") + keyNameStr := strings.Join(keyName[len(keyName)-4:], "/") + + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + g := golden.NewOrUpdate(t, op.String(), + golden.WithSanitizer(golden.StringSanitizer(keyNameStr, "KEY_NAME")), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")), + golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")), + ) + validateJSONPaths := []string{ + "status", + "location", + "confidentialNodes.enabled", + "databaseEncryption.keyName", + "databaseEncryption.state", + "privateClusterConfig.enablePrivateEndpoint", + "privateClusterConfig.enablePrivateNodes", + "addonsConfig.horizontalPodAutoscaling", + "addonsConfig.kubernetesDashboard", + "addonsConfig.networkPolicyConfig", + "networkPolicy", + "networkConfig.datapathProvider", + "binaryAuthorization.evaluationMode", + "legacyAbac", + "meshCertificates.enableCertificates", + "nodeConfig.bootDiskKmsKey", + "nodeConfig.diskType", + "nodeConfig.enableConfidentialStorage", + "nodeConfig.machineType", + "nodeConfig.diskType", + "nodeConfig.accelerators", + "nodeConfig.accelerators.acceleratorType", + "nodeConfig.accelerators.acceleratorCount", + "nodePools.enableConfidentialStorage", + "nodePools.diskType", + "nodePools.bootDiskKmsKey", + "nodePools.autoscaling", + "nodePools.config.machineType", + "nodePools.config.diskSizeGb", + "nodePools.config.labels", + "nodePools.config.tags", + "nodePools.config.accelerators", + "nodePools.config.accelerators.acceleratorType", + "nodePools.config.accelerators.acceleratorCount", + "nodePools.management.autoRepair", + "nodePools.shieldedInstanceConfig", + } + for _, pth := range validateJSONPaths { + g.JSONEq(assert, op, pth) + } + }) + + bpt.Test() +} diff --git a/test/integration/confidential_gpu_public/testdata/TestConfidentialGpuPublic.json b/test/integration/confidential_gpu_public/testdata/TestConfidentialGpuPublic.json new file mode 100644 index 0000000000..cde2fb1335 --- /dev/null +++ b/test/integration/confidential_gpu_public/testdata/TestConfidentialGpuPublic.json @@ -0,0 +1,367 @@ +{ + "addonsConfig": { + "configConnectorConfig": {}, + "dnsCacheConfig": {}, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + } + }, + "anonymousAuthenticationConfig": { + "mode": "ENABLED" + }, + "autopilot": {}, + "autoscaling": { + "autoscalingProfile": "BALANCED" + }, + "binaryAuthorization": {}, + "clusterIpv4Cidr": "192.168.0.0/18", + "confidentialNodes": { + "confidentialInstanceType": "TDX", + "enabled": true + }, + "controlPlaneEndpointsConfig": { + "dnsEndpointConfig": { + "allowExternalTraffic": true, + "endpoint": "gke-4d880c2015c144de8f6091f80aa72337b5d6-225014432059.us-central1.gke.goog" + }, + "ipEndpointsConfig": { + "authorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "enablePublicEndpoint": true, + "enabled": true, + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "34.132.168.89" + } + }, + "createTime": "2025-08-11T20:42:50+00:00", + "currentMasterVersion": "1.33.2-gke.1240000", + "currentNodeCount": 1, + "currentNodeVersion": "1.33.2-gke.1240000", + "databaseEncryption": { + "currentState": "CURRENT_STATE_DECRYPTED", + "state": "ENCRYPTED", + "keyName": "projects/PROJECT_ID/locations/us-central1/KEY_NAME" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "34.132.168.89", + "enterpriseConfig": { + "clusterTier": "STANDARD" + }, + "etag": "ec52e471-8245-4d00-843e-b599f38fed84", + "id": "4d880c2015c144de8f6091f80aa72337b5d6b6e9d14344c09690c68f55ca0770", + "identityServiceConfig": {}, + "initialClusterVersion": "1.33.2-gke.1240000", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gke-confidential-gpu-cluster--default-bcfac9d6-grp" + ], + "ipAllocationPolicy": { + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "ip-range-pods-sjvt", + "defaultPodIpv4RangeUtilization": 0.0156, + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "ip-range-svc-sjvt", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "78cdf2f6", + "legacyAbac": {}, + "location": "us-central1", + "locations": [ + "us-central1-a" + ], + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clientCertificateConfig": {}, + "clusterCaCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVMVENDQXBXZ0F3SUJBZ0lSQU14djZVLzZqcjkwaUlaVEMxb1JySDR3RFFZSktvWklodmNOQVFFTEJRQXcKTHpFdE1Dc0dBMVVFQXhNa1pUQXdaRFExTmpZdE1UbGxPQzAwWVROakxUazJOall0TnpZM1pqQmtORGhsT0dZMApNQ0FYRFRJMU1EZ3hNVEU1TkRJMU0xb1lEekl3TlRVd09EQTBNakEwTWpVeldqQXZNUzB3S3dZRFZRUURFeVJsCk1EQmtORFUyTmkweE9XVTRMVFJoTTJNdE9UWTJOaTAzTmpkbU1HUTBPR1U0WmpRd2dnR2lNQTBHQ1NxR1NJYjMKRFFFQkFRVUFBNElCandBd2dnR0tBb0lCZ1FESlB1Zlc5WlFXQlVFQ0N5QStWeXBmQnQvdWppRk1aWFlUVFNjVQpyQmNDRlR0a0o0b3pmb2hLMGtXNytwMWM4QWNtd3d0QVRGUlV2Lzh3REE0SjZNcHNqRkhQMFJJWlF4eWV6Z3IxCjFCOXY5cHFCY0h5bnY0cHFsUlIzOTZEWUdVK1A0SDUxckFlMTRLVXU5dktOaHlOVFc5TWIzS3B4cUZSeU5KaVgKNFhtT0pDamZESGhzcFM3bVNmT01IYkRCWDJobjhGKyt5dWF0Y0o5YzdtaU1ONkR0b2tyNXF3TGRBUG5ITTR1cwowQzFidHJjeE0wWk1wZWpxVkJjajlBSHdaemR2a1dwWmxGTVRVdzFTU1ZXZkhockNMckhqeFU1UUx2Y0tmdkRFCkkwRWtRY3pCWVZ5dGZ0U3FXSjhJbGJvdVJMWTAxZG9FZndua1NCN3J6VmZBRE5UN0wwTXBkZ25PWW0wNExDZkIKUTdnazZIKzBIdDRGRDJBYVhRaXZhZGswbVQ1RzFMWmJaMjdVZ1daNWZqYS9KcXZMUFdoS3g5ZG1QT1NFdUs1dQpaYVpMRCtSOEZSL2NjcWFnRlg5ellOWHZKdTI4QnF0UDdXUUp5VTdlaGFxcGNKYnFuRnFpb2JicTNwcmU5NSt2CnJJSlBsdC8yL09UVW1RKzU1MWNpMTJyL3lPVUNBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdJRU1BOEcKQTFVZEV3RUIvd1FGTUFNQkFmOHdIUVlEVlIwT0JCWUVGQ09sZVFFenVvU0U4SU9IOENkbUhNdCtqUlBWTUEwRwpDU3FHU0liM0RRRUJDd1VBQTRJQmdRQ3lKY0JDUXFVV3lZYmJORk9JbFNQT0xTZUZicDloTzNTcU5Sejd2Q3JoCjd0TUlpQmN3YVNseWpHWHpCS3E4RU93bEVsUUtERXZHSXZJbERoZ1pEbHIycVVBbnErN3NmT3hYM01XWENvQTQKaUdHSHZjdlFpUUlGZm5VUmlSVFJ3eUordXRvQ0tUdHNKYTltU2xVRGVCUkhjSWp2Q0IyQjRLT3NmakdQOXB4aQpRckRJcjV2djZNTG5HUkVUTnQ1aHVyZFRjYkk4bFI1OWZpOXdNWW9HV1RKaE9FUEtyOUZQeEFmWW9UNm9vWHhuClFKZE1rcHp4MjEwMjNLNlplL2xBZUhBVXA3KzF4MDhhU09mbUhHeU40VVpiVG5iMmZybmtHLzdSdjd3UGZxbUUKdm82NzdiTkRTQWVBME1nV3M1UVdrbWJDT0NzYnJzL0JRcFE2UVRZYVRyVmJRQVhWTU5TSWxueitTaEZUZksvcwpsS2FZR3YvUy9FbFJlSnMyYnp2YnhwRlNFbXpaeEFoOU9lV0JFMlZld2lld2V4dVVaRmJwbUtnd25VUVR1b2pGCkV1aHZJSFVMWG1TUWhIemdjTy9lRmZLbDJEQUNhK0k1SHVFKzJzWXNQMHRFWkVDcnNFWGRaTDBOREcxanpOaTIKY0N1cjBzYm05bzZ4MWhkdUZCc241cGs9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" + }, + "masterAuthorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "meshCertificates": { + "enableCertificates": false + }, + "monitoringConfig": { + "advancedDatapathObservabilityConfig": {}, + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "STATEFULSET", + "JOBSET", + "STORAGE", + "HPA", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "KUBELET", + "CADVISOR", + "DCGM" + ] + }, + "managedPrometheusConfig": { + "enabled": true + } + }, + "monitoringService": "monitoring.googleapis.com/kubernetes", + "name": "CLUSTER_NAME", + "network": "confidential-gpu-network-sjvt", + "networkConfig": { + "defaultSnatStatus": {}, + "network": "projects/PROJECT_ID/global/networks/confidential-gpu-network-sjvt", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/us-central1/subnetworks/confidential-gpu-subnet" + }, + "nodeConfig": { + "accelerators": [ + { + "acceleratorCount": "1", + "acceleratorType": "nvidia-h100-80gb", + "gpuDriverInstallationConfig": { + "gpuDriverVersion": "INSTALLATION_DISABLED" + } + } + ], + "bootDisk": { + "diskType": "hyperdisk-balanced", + "sizeGb": "100" + }, + "bootDiskKmsKey": "projects/PROJECT_ID/locations/us-central1/KEY_NAME", + "diskSizeGb": 100, + "diskType": "hyperdisk-balanced", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "enableConfidentialStorage": true, + "ephemeralStorageLocalSsdConfig": { + "localSsdCount": 2 + }, + "imageType": "COS_CONTAINERD", + "kubeletConfig": { + "insecureKubeletReadonlyPortEnabled": false, + "maxParallelImagePulls": 2 + }, + "labels": { + "cluster_name": "CLUSTER_NAME", + "node_pool": "default" + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "a3-highgpu-1g", + "metadata": { + "cluster_name": "CLUSTER_NAME", + "disable-legacy-endpoints": "false", + "node_pool": "default" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "resourceLabels": { + "goog-gke-accelerator-type": "nvidia-h100-80gb", + "goog-gke-node-pool-provisioning-model": "spot" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "spot": true, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolAutoConfig": { + "nodeKubeletConfig": { + "insecureKubeletReadonlyPortEnabled": false + } + }, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "gcfsConfig": {}, + "loggingConfig": { + "variantConfig": { + "variant": "MAX_THROUGHPUT" + } + }, + "nodeKubeletConfig": { + "insecureKubeletReadonlyPortEnabled": false + } + } + }, + "nodePools": [ + { + "autoscaling": { + "enabled": true, + "locationPolicy": "ANY", + "maxNodeCount": 100, + "minNodeCount": 1 + }, + "config": { + "accelerators": [ + { + "acceleratorCount": "1", + "acceleratorType": "nvidia-h100-80gb", + "gpuDriverInstallationConfig": { + "gpuDriverVersion": "INSTALLATION_DISABLED" + } + } + ], + "bootDisk": { + "diskType": "hyperdisk-balanced", + "sizeGb": "100" + }, + "bootDiskKmsKey": "projects/PROJECT_ID/locations/us-central1/KEY_NAME", + "diskSizeGb": 100, + "diskType": "hyperdisk-balanced", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "enableConfidentialStorage": true, + "ephemeralStorageLocalSsdConfig": { + "localSsdCount": 2 + }, + "imageType": "COS_CONTAINERD", + "kubeletConfig": { + "insecureKubeletReadonlyPortEnabled": false, + "maxParallelImagePulls": 2 + }, + "labels": { + "cluster_name": "CLUSTER_NAME", + "node_pool": "default" + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "a3-highgpu-1g", + "metadata": { + "cluster_name": "CLUSTER_NAME", + "disable-legacy-endpoints": "false", + "node_pool": "default" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "resourceLabels": { + "goog-gke-accelerator-type": "nvidia-h100-80gb", + "goog-gke-node-pool-provisioning-model": "spot" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "spot": true, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "ca100088-959d-40a1-b2ce-5e7e192d2d2c", + "initialNodeCount": 1, + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gke-confidential-gpu-cluster--default-bcfac9d6-grp" + ], + "locations": [ + "us-central1-a" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "default", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0156, + "podRange": "ip-range-pods-sjvt", + "subnetwork": "projects/PROJECT_ID/regions/us-central1/subnetworks/confidential-gpu-subnet" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME/nodePools/default", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.33.2-gke.1240000" + } + ], + "notificationConfig": { + "pubsub": {} + }, + "podAutoscaling": { + "hpaProfile": "HPA_PROFILE_UNSPECIFIED" + }, + "privateClusterConfig": { + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "34.132.168.89" + }, + "rbacBindingConfig": { + "enableInsecureBindingSystemAuthenticated": true, + "enableInsecureBindingSystemUnauthenticated": true + }, + "releaseChannel": { + "channel": "REGULAR" + }, + "resourceLabels": { + "goog-terraform-provisioned": "true" + }, + "securityPostureConfig": { + "mode": "DISABLED", + "vulnerabilityMode": "VULNERABILITY_DISABLED" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "confidential-gpu-subnet", + "userManagedKeysConfig": {}, + "verticalPodAutoscaling": {}, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "us-central1" +}