diff --git a/autogen/safer-cluster/main.tf.tmpl b/autogen/safer-cluster/main.tf.tmpl index 3a06a87e3f..0fb45c6773 100644 --- a/autogen/safer-cluster/main.tf.tmpl +++ b/autogen/safer-cluster/main.tf.tmpl @@ -110,6 +110,8 @@ module "gke" { monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus monitoring_enabled_components = var.monitoring_enabled_components + enable_confidential_nodes = var.enable_confidential_nodes + // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever // compromised. diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl index 678eaa2a3a..bcf8795072 100644 --- a/autogen/safer-cluster/variables.tf.tmpl +++ b/autogen/safer-cluster/variables.tf.tmpl @@ -544,3 +544,9 @@ variable "deletion_protection" { description = "Whether or not to allow Terraform to destroy the cluster." default = true } + +variable "enable_confidential_nodes" { + type = bool + description = "An optional flag to enable confidential node config." + default = false +} diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 2658f5e030..6e2675b71b 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -466,6 +466,21 @@ steps: - verify simple-fleet-app-operator-permissions name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage teardown --verbose'] +- id: apply test-confidential-safer-cluster + waitFor: + - init-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestConfidentialSaferCluster --stage apply --verbose'] +- id: verify test-confidential-safer-cluster + waitFor: + - apply test-confidential-safer-cluster + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestConfidentialSaferCluster --stage verify --verbose'] +- id: teardown test-confidential-safer-cluster + waitFor: + - verify test-confidential-safer-cluster + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestConfidentialSaferCluster --stage teardown --verbose'] tags: - 'ci' - 'integration' diff --git a/examples/confidential_safer_cluster/README.md b/examples/confidential_safer_cluster/README.md new file mode 100644 index 0000000000..9fe41f30e4 --- /dev/null +++ b/examples/confidential_safer_cluster/README.md @@ -0,0 +1,40 @@ +# Confidential Safer GKE Cluster + +This example illustrates how to instantiate the Safer Cluster module +with confidential nodes enabled and database encrypted with KMS key. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to host the cluster in. | `string` | n/a | yes | +| region | The region to host the cluster in. | `string` | `"us-central1"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | The cluster ca certificate (base64 encoded). | +| client\_token | The bearer token for auth. | +| cluster\_name | Cluster name. | +| explicit\_k8s\_version | Explicit version used for cluster creation. | +| keyring | The name of the keyring. | +| kms\_key\_name | KMS Key Name. | +| kubernetes\_endpoint | The cluster endpoint. | +| location | n/a | +| master\_kubernetes\_version | Kubernetes version of the master. | +| network\_name | The name of the VPC being created. | +| project\_id | The project ID the cluster is in. | +| region | The region in which the cluster resides. | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created. | +| zones | List of zones in which the cluster resides. | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/confidential_safer_cluster/kms.tf b/examples/confidential_safer_cluster/kms.tf new file mode 100644 index 0000000000..21fb456c7e --- /dev/null +++ b/examples/confidential_safer_cluster/kms.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + key_name = "gke-key-${random_string.suffix.result}" +} + +module "kms" { + source = "terraform-google-modules/kms/google" + version = "~> 3.0" + project_id = var.project_id + location = var.region + keyring = "gke-keyring-${random_string.suffix.result}" + keys = [local.key_name] + prevent_destroy = false +} + +resource "google_project_service_identity" "container_identity" { + provider = google-beta + project = var.project_id + service = "container.googleapis.com" +} + +resource "google_kms_crypto_key_iam_member" "sm_sa_encrypter_decrypter" { + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = google_project_service_identity.container_identity.member + crypto_key_id = module.kms.keys[local.key_name] +} diff --git a/examples/confidential_safer_cluster/main.tf b/examples/confidential_safer_cluster/main.tf new file mode 100644 index 0000000000..96eef1d1b0 --- /dev/null +++ b/examples/confidential_safer_cluster/main.tf @@ -0,0 +1,102 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +locals { + cluster_type = "confidential-safer" + network_name = "confidential-safer-network-${random_string.suffix.result}" + subnet_name = "confidential-safer-subnet" + master_auth_subnetwork = "confidential-safer-master-subnet" + pods_range_name = "ip-range-pods-${random_string.suffix.result}" + svc_range_name = "ip-range-svc-${random_string.suffix.result}" + subnet_names = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]] +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +// A random valid k8s version is retrived +// to specify as an explicit version. +data "google_container_engine_versions" "current" { + project = var.project_id + location = var.region +} + +resource "random_shuffle" "version" { + input = data.google_container_engine_versions.current.valid_master_versions + result_count = 1 +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/safer-cluster" + version = "~> 35.0" + + project_id = var.project_id + name = "${local.cluster_type}-cluster-${random_string.suffix.result}" + regional = true + region = var.region + network = module.gcp-network.network_name + subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name + master_ipv4_cidr_block = "172.16.0.0/28" + add_cluster_firewall_rules = true + firewall_inbound_ports = ["9443", "15017"] + kubernetes_version = random_shuffle.version.result[0] + release_channel = "UNSPECIFIED" + deletion_protection = false + enable_private_endpoint = true + enable_confidential_nodes = true + + master_authorized_networks = [ + { + cidr_block = "10.60.0.0/17" + display_name = "VPC" + }, + ] + + database_encryption = [ + { + "key_name" : module.kms.keys[local.key_name], + "state" : "ENCRYPTED" + } + ] + + node_pools = [ + { + name = "default" + machine_type = "n2d-standard-2" + enable_secure_boot = true + }, + ] + + notification_config_topic = google_pubsub_topic.updates.id +} + +resource "google_pubsub_topic" "updates" { + name = "cluster-updates-${random_string.suffix.result}" + project = var.project_id +} diff --git a/examples/confidential_safer_cluster/network.tf b/examples/confidential_safer_cluster/network.tf new file mode 100644 index 0000000000..5f12f52936 --- /dev/null +++ b/examples/confidential_safer_cluster/network.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = "~> 10.0" + + project_id = var.project_id + network_name = local.network_name + routing_mode = "GLOBAL" + + subnets = [ + { + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = var.region + subnet_private_access = true + }, + { + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" + subnet_region = var.region + subnet_private_access = true + }, + ] + + secondary_ranges = { + (local.subnet_name) = [ + { + range_name = local.pods_range_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = local.svc_range_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/confidential_safer_cluster/outputs.tf b/examples/confidential_safer_cluster/outputs.tf new file mode 100644 index 0000000000..d2a448ba9f --- /dev/null +++ b/examples/confidential_safer_cluster/outputs.tf @@ -0,0 +1,91 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint." + sensitive = true + value = module.gke.endpoint +} + +output "cluster_name" { + description = "Cluster name." + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master." + value = module.gke.master_version +} + +output "client_token" { + description = "The bearer token for auth." + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + description = "The cluster ca certificate (base64 encoded)." + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + +output "network_name" { + description = "The name of the VPC being created." + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created." + value = module.gcp-network.subnets_names +} + +output "region" { + description = "The region in which the cluster resides." + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides." + value = module.gke.zones +} + +output "project_id" { + description = "The project ID the cluster is in." + value = var.project_id +} + +output "explicit_k8s_version" { + description = "Explicit version used for cluster creation." + value = random_shuffle.version.result[0] +} + +output "keyring" { + description = "The name of the keyring." + value = module.kms.keyring +} + +output "kms_key_name" { + description = "KMS Key Name." + value = module.kms.keys[local.key_name] +} diff --git a/examples/confidential_safer_cluster/variables.tf b/examples/confidential_safer_cluster/variables.tf new file mode 100644 index 0000000000..4254bb8138 --- /dev/null +++ b/examples/confidential_safer_cluster/variables.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project ID to host the cluster in." +} + +variable "region" { + type = string + description = "The region to host the cluster in." + default = "us-central1" +} diff --git a/examples/confidential_safer_cluster/versions.tf b/examples/confidential_safer_cluster/versions.tf new file mode 100644 index 0000000000..9cb767a518 --- /dev/null +++ b/examples/confidential_safer_cluster/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.3" + required_providers { + google = { + source = "hashicorp/google" + } + google-beta = { + source = "hashicorp/google-beta" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + } +} diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index ee0e3c39e8..5b13a898e2 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -219,6 +219,7 @@ For simplicity, we suggest using `roles/container.admin` and | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | diff --git a/modules/safer-cluster-update-variant/main.tf b/modules/safer-cluster-update-variant/main.tf index a13fafe5fe..ab5f5abce9 100644 --- a/modules/safer-cluster-update-variant/main.tf +++ b/modules/safer-cluster-update-variant/main.tf @@ -106,6 +106,8 @@ module "gke" { monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus monitoring_enabled_components = var.monitoring_enabled_components + enable_confidential_nodes = var.enable_confidential_nodes + // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever // compromised. diff --git a/modules/safer-cluster-update-variant/variables.tf b/modules/safer-cluster-update-variant/variables.tf index 02d6f8e526..cb0a185ce6 100644 --- a/modules/safer-cluster-update-variant/variables.tf +++ b/modules/safer-cluster-update-variant/variables.tf @@ -544,3 +544,9 @@ variable "deletion_protection" { description = "Whether or not to allow Terraform to destroy the cluster." default = true } + +variable "enable_confidential_nodes" { + type = bool + description = "An optional flag to enable confidential node config." + default = false +} diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index ee0e3c39e8..5b13a898e2 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -219,6 +219,7 @@ For simplicity, we suggest using `roles/container.admin` and | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index e113c09a6a..a9ddb7d504 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -106,6 +106,8 @@ module "gke" { monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus monitoring_enabled_components = var.monitoring_enabled_components + enable_confidential_nodes = var.enable_confidential_nodes + // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever // compromised. diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 02d6f8e526..cb0a185ce6 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -544,3 +544,9 @@ variable "deletion_protection" { description = "Whether or not to allow Terraform to destroy the cluster." default = true } + +variable "enable_confidential_nodes" { + type = bool + description = "An optional flag to enable confidential node config." + default = false +} diff --git a/test/fixtures/confidential_safer_cluster/example.tf b/test/fixtures/confidential_safer_cluster/example.tf new file mode 100644 index 0000000000..07db6c9cb7 --- /dev/null +++ b/test/fixtures/confidential_safer_cluster/example.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/confidential_safer_cluster" + + project_id = var.project_ids[1] + region = var.region +} diff --git a/test/fixtures/confidential_safer_cluster/outputs.tf b/test/fixtures/confidential_safer_cluster/outputs.tf new file mode 100644 index 0000000000..3f8f89b027 --- /dev/null +++ b/test/fixtures/confidential_safer_cluster/outputs.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = module.example.project_id +} + +output "cluster_name" { + description = "Cluster name." + value = module.example.cluster_name +} + +output "location" { + value = module.example.location +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.example.service_account +} + +output "kms_key_name" { + description = "KMS Key Name." + value = module.example.kms_key_name +} diff --git a/test/fixtures/confidential_safer_cluster/variables.tf b/test/fixtures/confidential_safer_cluster/variables.tf new file mode 100644 index 0000000000..4cdbff699a --- /dev/null +++ b/test/fixtures/confidential_safer_cluster/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests." +} + +variable "region" { + description = "The GCP region to create and test resources in." + default = "us-central1" +} diff --git a/test/integration/confidential_safer_cluster/confidential_safer_cluster_test.go b/test/integration/confidential_safer_cluster/confidential_safer_cluster_test.go new file mode 100644 index 0000000000..fefe91144c --- /dev/null +++ b/test/integration/confidential_safer_cluster/confidential_safer_cluster_test.go @@ -0,0 +1,80 @@ +// Copyright 2022-2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package confidential_safer_cluster + +import ( + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" +) + +func TestConfidentialSaferCluster(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + testutils.TGKEVerify(t, bpt, assert) + + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + serviceAccount := bpt.GetStringOutput("service_account") + keyName := bpt.GetStringOutput("kms_key_name") + + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + g := golden.NewOrUpdate(t, op.String(), + golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")), + golden.WithSanitizer(golden.StringSanitizer(keyName, "KEY_NAME")), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")), + ) + validateJSONPaths := []string{ + "status", + "location", + "confidentialNodes.enabled", + "databaseEncryption.keyName", + "databaseEncryption.state", + "privateClusterConfig.enablePrivateEndpoint", + "privateClusterConfig.enablePrivateNodes", + "addonsConfig.horizontalPodAutoscaling", + "addonsConfig.kubernetesDashboard", + "addonsConfig.networkPolicyConfig", + "networkPolicy", + "networkConfig.datapathProvider", + "binaryAuthorization.evaluationMode", + "legacyAbac", + "meshCertificates.enableCertificates", + "nodePools.autoscaling", + "nodePools.config.machineType", + "nodePools.config.diskSizeGb", + "nodePools.config.labels", + "nodePools.config.tags", + "nodePools.management.autoRepair", + "nodePools.shieldedInstanceConfig", + } + for _, pth := range validateJSONPaths { + g.JSONEq(assert, op, pth) + } + }) + + bpt.Test() +} diff --git a/test/integration/confidential_safer_cluster/testdata/TestConfidentialSaferCluster.json b/test/integration/confidential_safer_cluster/testdata/TestConfidentialSaferCluster.json new file mode 100644 index 0000000000..98175b78c0 --- /dev/null +++ b/test/integration/confidential_safer_cluster/testdata/TestConfidentialSaferCluster.json @@ -0,0 +1,326 @@ +{ + "addonsConfig": { + "configConnectorConfig": {}, + "dnsCacheConfig": {}, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + } + }, + "autopilot": {}, + "autoscaling": { + "autoscalingProfile": "BALANCED" + }, + "binaryAuthorization": { + "evaluationMode": "PROJECT_SINGLETON_POLICY_ENFORCE" + }, + "clusterIpv4Cidr": "192.168.0.0/18", + "confidentialNodes": { + "enabled": true + }, + "controlPlaneEndpointsConfig": { + "dnsEndpointConfig": { + "allowExternalTraffic": true, + "endpoint": "gke-c9866da76079445680c7b1d11c7b3690c08a-225014432059.us-central1.gke.goog" + }, + "ipEndpointsConfig": { + "authorizedNetworksConfig": { + "cidrBlocks": [ + { + "cidrBlock": "10.60.0.0/17", + "displayName": "VPC" + } + ], + "enabled": true, + "privateEndpointEnforcementEnabled": true + }, + "enablePublicEndpoint": false, + "enabled": true, + "globalAccess": true, + "privateEndpoint": "172.16.0.2", + "privateEndpointSubnetwork": "projects/PROJECT_ID/regions/us-central1/subnetworks/gke-CLUSTER_NAME-5b98b0e0-pe-subnet" + } + }, + "createTime": "2024-12-23T20:31:14+00:00", + "currentMasterVersion": "1.30.7-gke.1084000", + "currentNodeCount": 3, + "currentNodeVersion": "1.30.7-gke.1084000", + "databaseEncryption": { + "currentState": "CURRENT_STATE_ENCRYPTED", + "keyName": "KEY_NAME", + "state": "ENCRYPTED" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "172.16.0.2", + "enterpriseConfig": { + "clusterTier": "STANDARD" + }, + "etag": "0af4b53d-a1bb-440a-92f3-d90c2f976cb8", + "id": "c9866da76079445680c7b1d11c7b3690c08af02ff41b42078e424b33f1b3de9d", + "identityServiceConfig": {}, + "initialClusterVersion": "1.30.7-gke.1084000", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gke-confidential-safer-cluste-default-fa922279-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-b/instanceGroupManagers/gke-confidential-safer-cluste-default-3de090d9-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gke-confidential-safer-cluste-default-f802dd9a-grp" + ], + "ipAllocationPolicy": { + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "ip-range-pods-4j29", + "defaultPodIpv4RangeUtilization": 0.0469, + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "ip-range-svc-4j29", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "78cdf2f6", + "legacyAbac": {}, + "location": "us-central1", + "locations": [ + "us-central1-a", + "us-central1-b", + "us-central1-c" + ], + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clientCertificateConfig": {}, + "clusterCaCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVMVENDQXBXZ0F3SUJBZ0lSQU40S1I3Q1J0MHg5L2Z4b1dMa0JjS0l3RFFZSktvWklodmNOQVFFTEJRQXcKTHpFdE1Dc0dBMVVFQXhNa016VmlOREExWldVdFlXRTBZaTAwTkRJMExXSXdNalV0TW1RM1l6UTBORFEzWmpjegpNQ0FYRFRJME1USXlNekU1TXpFeE5Wb1lEekl3TlRReE1qRTJNakF6TVRFMVdqQXZNUzB3S3dZRFZRUURFeVF6Ck5XSTBNRFZsWlMxaFlUUmlMVFEwTWpRdFlqQXlOUzB5WkRkak5EUTBORGRtTnpNd2dnR2lNQTBHQ1NxR1NJYjMKRFFFQkFRVUFBNElCandBd2dnR0tBb0lCZ1FDdVkwWGNsSFdGTmRTSktiYTgrL29KazZYUmo0a0t5UUppUXAveQozaC9nSWlsQUVmd2FpVmorYlNNTCtrMTV6RlhTd2thaFRiKzV3T3d4emFvMXJ3OS8wQWxJTm1qM1cxUXlwazBMCktGa3lDK0c3dHR6VHlVMkxDMi90Y3oycjRSYWV0UjNacjJ5a282NmVDTUZTZXR4N0E3cUtka01UbWVJL2lCVDAKSWcvOHpEMjMrVTJ5eVpvTHhOOVJrTDZHYnZTS1REa1orUDdwSlBDOFcyVE90UnVzZlBmRGlueGh5elpQU2FGWgpWT1h4V2RDNWJwQlB3dkJhOTEwWWdOMklXVjZGaG1VU0pSTnk2QWJKbTBXakFJWUI0TS9iVWRGTndnSFBFS0F2CjY1cGdFanVmLy9aQnh3TlRTY1lrZG93VElLNm9aUFl3clpxRUgyMDUwZzNIem1nTHM5R21NN3ZyNmlkR2FIY0QKU0phNHAxN3F4SjVnSkRjcUtOSzhYYVhFL0FxUGIxc0VZcG1MSDlqZmMwU3FTdEJVTi9tMTNobmxzd0tYMDFXeApGbVQ1bE1nVTBSS3FEeThqM3pmVWNlTzFlWUZoOVF1OW51aGFqbmN4WUpjZFUzWk5xU2tsUlFZVWlrRGt6TTVQClA3anlsU1ByUzdvbWhOZmR4UnN6Tkt6WGFUY0NBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdJRU1BOEcKQTFVZEV3RUIvd1FGTUFNQkFmOHdIUVlEVlIwT0JCWUVGUDBMK0c4Qk80ai9renRlUGVzYzV5ckR0ckNrTUEwRwpDU3FHU0liM0RRRUJDd1VBQTRJQmdRQUQxaHZydklmVkV6bHhYNk0wcWFYVjR6SmdjOWV1RzdOT2tQUEU2OGh0CkdTK0l3S045TGl4R1NLWkZnMUM4RVR1cnlmNEJXdktSOW4va1VlbkozMUQ0ZklReGo4cWJJQmZuS1U3aGhmdGMKeHVUVnNTWW5sVWI5SXdONjRnWVM4QjJteS92OUtOdW1ISktDS1VGcnNFU2NCeExrTDRGeHplRVEzSnVrVGMrawo0dEw2R0tTclZkYUtTQmNCeEZZNlBjL1lzWjg1c2RIWjlQRmxUaVFkZ0tKQlNWTDJIUWpGbXdDUFIyWFEvc0dNCkE0RUFNWVYvTG9sOS82QzVnQ3ZSUU5SQjlrV3hPbzU5anRBYkZiT2JEWkhtQW1ZWUY4N0Qrd1k1ZUhYK0QwMDUKdlZCamFleXRLTVZ2cXVuVTFRc0pNMmtCMEZJZ0E1cWZ4K2pyVTJFVG5ONkhYNHduYVJ0V0QrN1dLemZzRjlvLwpTenQ4L1dOVkVEYjlGNGZXWFFrcVA4UVdiT3VTd0hSVkhlT0l6VFYzZ0t5aXFBRWt6V1RCSkNrdWRZdFA5MUR3CnM0Ukt6RFpFUmQ1c1VZZFRIOWdScHh6elFUMUllRVA0OUwzY1RLc0FsSnRqTWNQNWx0SFdGL2ZHNlg3bitYNzgKc0tnZG5WdjR6dnNkR2VEdEJHMElZeGs9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" + }, + "masterAuthorizedNetworksConfig": { + "cidrBlocks": [ + { + "cidrBlock": "10.60.0.0/17", + "displayName": "VPC" + } + ], + "enabled": true, + "privateEndpointEnforcementEnabled": true + }, + "meshCertificates": { + "enableCertificates": false + }, + "monitoringConfig": { + "advancedDatapathObservabilityConfig": { + "enableRelay": false + }, + "componentConfig": {}, + "managedPrometheusConfig": {} + }, + "monitoringService": "none", + "name": "CLUSTER_NAME", + "network": "confidential-safer-network-4j29", + "networkConfig": { + "datapathProvider": "ADVANCED_DATAPATH", + "defaultEnablePrivateNodes": true, + "defaultSnatStatus": {}, + "network": "projects/PROJECT_ID/global/networks/confidential-safer-network-4j29", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/us-central1/subnetworks/confidential-safer-subnet" + }, + "nodeConfig": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "kubeletConfig": { + "insecureKubeletReadonlyPortEnabled": true + }, + "labels": { + "cluster_name": "CLUSTER_NAME", + "node_pool": "default" + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "n2d-standard-2", + "metadata": { + "cluster_name": "CLUSTER_NAME", + "disable-legacy-endpoints": "true", + "node_pool": "default" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true, + "enableSecureBoot": true + }, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "gcfsConfig": {}, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "nodeKubeletConfig": {} + } + }, + "nodePools": [ + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 100, + "minNodeCount": 1 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "kubeletConfig": { + "insecureKubeletReadonlyPortEnabled": true + }, + "labels": { + "cluster_name": "CLUSTER_NAME", + "node_pool": "default" + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "n2d-standard-2", + "metadata": { + "cluster_name": "CLUSTER_NAME", + "disable-legacy-endpoints": "true", + "node_pool": "default" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true, + "enableSecureBoot": true + }, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "8c0fb202-bac1-4c08-a4b9-dc3f2a1e71de", + "initialNodeCount": 1, + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gke-confidential-safer-cluste-default-fa922279-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-b/instanceGroupManagers/gke-confidential-safer-cluste-default-3de090d9-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gke-confidential-safer-cluste-default-f802dd9a-grp" + ], + "locations": [ + "us-central1-a", + "us-central1-b", + "us-central1-c" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "default", + "networkConfig": { + "enablePrivateNodes": true, + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0469, + "podRange": "ip-range-pods-4j29" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME/nodePools/default", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.7-gke.1084000" + } + ], + "notificationConfig": { + "pubsub": { + "enabled": true, + "topic": "projects/PROJECT_ID/topics/cluster-updates-4j29" + } + }, + "privateClusterConfig": { + "enablePrivateEndpoint": true, + "enablePrivateNodes": true, + "masterGlobalAccessConfig": { + "enabled": true + }, + "masterIpv4CidrBlock": "172.16.0.0/28", + "privateEndpoint": "172.16.0.2", + "publicEndpoint": "34.59.148.251" + }, + "rbacBindingConfig": { + "enableInsecureBindingSystemAuthenticated": true, + "enableInsecureBindingSystemUnauthenticated": true + }, + "releaseChannel": {}, + "resourceLabels": { + "goog-terraform-provisioned": "true" + }, + "securityPostureConfig": { + "mode": "DISABLED", + "vulnerabilityMode": "VULNERABILITY_MODE_UNSPECIFIED" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "confidential-safer-subnet", + "verticalPodAutoscaling": {}, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "us-central1" +}