Skip to content

Commit f123a0c

Browse files
arthurlapertosaarthurlapertosa
committed
feat: add autopilot confidential nodes example
1 parent e5bd905 commit f123a0c

File tree

8 files changed

+343
-0
lines changed

8 files changed

+343
-0
lines changed

‎build/int.cloudbuild.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,21 @@ steps:
481481
- verify test-confidential-safer-cluster
482482
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
483483
args: ['/bin/bash', '-c', 'cft test run TestConfidentialSaferCluster --stage teardown --verbose']
484+
- id: apply test-confidential-autopilot-private
485+
waitFor:
486+
- init-all
487+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
488+
args: ['/bin/bash', '-c', 'cft test run TestConfidentialAutopilotPrivate --stage apply --verbose']
489+
- id: verify test-confidential-autopilot-private
490+
waitFor:
491+
- apply test-confidential-autopilot-private
492+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
493+
args: ['/bin/bash', '-c', 'cft test run TestConfidentialAutopilotPrivate --stage verify --verbose']
494+
- id: teardown test-confidential-autopilot-private
495+
waitFor:
496+
- verify test-confidential-autopilot-private
497+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
498+
args: ['/bin/bash', '-c', 'cft test run TestConfidentialAutopilotPrivate --stage teardown --verbose']
484499
tags:
485500
- 'ci'
486501
- 'integration'

‎examples/confidential_autopilot_private/README.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Confidential Autopilot Private Cluster
2+
3+
This example illustrates how to create a autopilot cluster with beta features,
4+
using Confidential Nodes and a Customer Managed Encryption Keys (CMEK).
5+
6+
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
7+
## Inputs
8+
9+
| Name | Description | Type | Default | Required |
10+
|------|-------------|------|---------|:--------:|
11+
| project\_id | The project ID to host the cluster in | `any` | n/a | yes |
12+
13+
## Outputs
14+
15+
| Name | Description |
16+
|------|-------------|
17+
| cluster\_name | Cluster name |
18+
| kms\_key | CMEK used for disk and database encryption |
19+
| kubernetes\_endpoint | The cluster endpoint |
20+
| location | n/a |
21+
| master\_kubernetes\_version | Kubernetes version of the master |
22+
| network\_name | The name of the VPC being created |
23+
| region | The region in which the cluster resides |
24+
| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
25+
| subnet\_names | The names of the subnet being created |
26+
| zones | List of zones in which the cluster resides |
27+
28+
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
29+
30+
To provision this example, run the following from within this directory:
31+
- `terraform init` to get the plugins
32+
- `terraform plan` to see the infrastructure plan
33+
- `terraform apply` to apply the infrastructure build
34+
- `terraform destroy` to destroy the built infrastructure

‎examples/confidential_autopilot_private/main.tf‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
locals {
18+
cluster_type = "confidential-autopilot-private"
19+
network_name = "confidential-autopilot-private-network"
20+
subnet_name = "confidential-autopilot-private-subnet"
21+
master_auth_subnetwork = "confidential-autopilot-master-subnet"
22+
pods_range_name = "ip-range-pods-confidential-autopilot"
23+
svc_range_name = "ip-range-svc-confidential-autopilot"
24+
subnet_names = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]]
25+
}
26+
27+
data "google_project" "main" {
28+
project_id = var.project_id
29+
}
30+
31+
resource "random_string" "suffix" {
32+
length = 4
33+
special = false
34+
upper = false
35+
}
36+
37+
module "kms" {
38+
source = "terraform-google-modules/kms/google"
39+
version = "~> 4.0"
40+
41+
project_id = var.project_id
42+
key_protection_level = "HSM"
43+
location = "us-central1"
44+
keyring = "keyring-${random_string.suffix.result}"
45+
keys = ["key"]
46+
prevent_destroy = false
47+
}
48+
49+
resource "google_kms_crypto_key_iam_member" "main" {
50+
crypto_key_id = values(module.kms.keys)[0]
51+
role = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
52+
member = "serviceAccount:service-${data.google_project.main.number}@compute-system.iam.gserviceaccount.com"
53+
}
54+
55+
module "gke" {
56+
source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster"
57+
version = "~> 36.0"
58+
59+
project_id = var.project_id
60+
name = "${local.cluster_type}-cluster"
61+
regional = true
62+
region = "us-central1"
63+
network = module.gcp-network.network_name
64+
subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
65+
ip_range_pods = local.pods_range_name
66+
ip_range_services = local.svc_range_name
67+
release_channel = "REGULAR"
68+
enable_vertical_pod_autoscaling = true
69+
enable_private_endpoint = true
70+
enable_private_nodes = true
71+
network_tags = [local.cluster_type]
72+
deletion_protection = false
73+
boot_disk_kms_key = values(module.kms.keys)[0]
74+
enable_confidential_nodes = true
75+
76+
database_encryption = [
77+
{
78+
"key_name" : values(module.kms.keys)[0],
79+
"state" : "ENCRYPTED"
80+
}
81+
]
82+
83+
depends_on = [google_kms_crypto_key_iam_member.main]
84+
}

‎examples/confidential_autopilot_private/network.tf‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
module "gcp-network" {
18+
source = "terraform-google-modules/network/google"
19+
version = ">= 7.5"
20+
21+
project_id = var.project_id
22+
network_name = local.network_name
23+
24+
subnets = [
25+
{
26+
subnet_name = local.subnet_name
27+
subnet_ip = "10.0.0.0/17"
28+
subnet_region = "us-central1"
29+
subnet_private_access = true
30+
},
31+
{
32+
subnet_name = local.master_auth_subnetwork
33+
subnet_ip = "10.60.0.0/17"
34+
subnet_region = "us-central1"
35+
},
36+
]
37+
38+
secondary_ranges = {
39+
(local.subnet_name) = [
40+
{
41+
range_name = local.pods_range_name
42+
ip_cidr_range = "192.168.0.0/18"
43+
},
44+
{
45+
range_name = local.svc_range_name
46+
ip_cidr_range = "192.168.64.0/18"
47+
},
48+
]
49+
}
50+
}

‎examples/confidential_autopilot_private/outputs.tf‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
output "kubernetes_endpoint" {
18+
description = "The cluster endpoint"
19+
sensitive = true
20+
value = module.gke.endpoint
21+
}
22+
23+
output "cluster_name" {
24+
description = "Cluster name"
25+
value = module.gke.name
26+
}
27+
28+
output "location" {
29+
value = module.gke.location
30+
}
31+
32+
output "master_kubernetes_version" {
33+
description = "Kubernetes version of the master"
34+
value = module.gke.master_version
35+
}
36+
37+
output "service_account" {
38+
description = "The service account to default running nodes as if not overridden in `node_pools`."
39+
value = module.gke.service_account
40+
}
41+
42+
output "network_name" {
43+
description = "The name of the VPC being created"
44+
value = module.gcp-network.network_name
45+
}
46+
47+
output "subnet_names" {
48+
description = "The names of the subnet being created"
49+
value = module.gcp-network.subnets_names
50+
}
51+
52+
output "region" {
53+
description = "The region in which the cluster resides"
54+
value = module.gke.region
55+
}
56+
57+
output "zones" {
58+
description = "List of zones in which the cluster resides"
59+
value = module.gke.zones
60+
}
61+
62+
output "kms_key" {
63+
description = "CMEK used for disk and database encryption"
64+
value = values(module.kms.keys)[0]
65+
}

‎examples/confidential_autopilot_private/variables.tf‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
variable "project_id" {
18+
description = "The project ID to host the cluster in"
19+
}

‎examples/confidential_autopilot_private/versions.tf‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
terraform {
18+
required_providers {
19+
google = {
20+
source = "hashicorp/google"
21+
}
22+
kubernetes = {
23+
source = "hashicorp/kubernetes"
24+
}
25+
}
26+
required_version = ">= 1.3"
27+
}

‎test/integration/confidential_autopilot_private/confidential_autopilot_private_test.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package confidential_autopilot_private
16+
17+
import (
18+
"testing"
19+
"time"
20+
21+
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
22+
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
23+
"github.com/stretchr/testify/assert"
24+
"github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils"
25+
)
26+
27+
func TestConfidentialAutopilotPrivate(t *testing.T) {
28+
projectID := testutils.GetTestProjectFromSetup(t, 1)
29+
bpt := tft.NewTFBlueprintTest(t,
30+
tft.WithVars(map[string]interface{}{"project_id": projectID}),
31+
tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute),
32+
)
33+
34+
bpt.DefineVerify(func(assert *assert.Assertions) {
35+
testutils.TGKEVerify(t, bpt, assert)
36+
37+
location := bpt.GetStringOutput("location")
38+
clusterName := bpt.GetStringOutput("cluster_name")
39+
key := bpt.GetStringOutput("kms_key")
40+
41+
op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectID)
42+
assert.True(op.Get("autopilot.enabled").Bool(), "should be autopilot")
43+
assert.Equal(op.Get("autoscaling.autoprovisioningNodePoolDefaults.bootDiskKmsKey").String(), key, "should have CMEK configured in boot disk")
44+
assert.True(op.Get("confidentialNodes.enabled").Bool(), "should have confidential nodes enabled")
45+
assert.Equal(op.Get("databaseEncryption.state").String(), "ENCRYPTED", "should have database encryption")
46+
assert.Equal(op.Get("databaseEncryption.keyName").String(), key, "should have CMEK configured in database")
47+
})
48+
bpt.Test()
49+
}

0 commit comments

Comments
 (0)