diff --git a/.kitchen.yml b/.kitchen.yml index 62beeda102..7447818116 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -46,31 +46,6 @@ suites: systems: - name: simple_regional_with_ipv6 backend: local - - name: "stub_domains" - transport: - root_module_directory: test/fixtures/stub_domains - verifier: - systems: - - name: stub_domains - backend: local - controls: - - gcloud - - kubectl - # Disabled due to issue #264 - # (https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/264) - # - name: stub_domains_private - # transport: - # root_module_directory: test/fixtures/stub_domains_private - # systems: - # - name: stub_domains_private - # backend: local - - name: "upstream_nameservers" - transport: - root_module_directory: test/fixtures/upstream_nameservers - verifier: - systems: - - name: upstream_nameservers - backend: local - name: "stub_domains_upstream_nameservers" transport: root_module_directory: test/fixtures/stub_domains_upstream_nameservers diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e91b14216d..dafaf09c35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,25 +41,14 @@ Integration tests are used to verify the behaviour of the root module, submodules, and example modules. Additions, changes, and fixes should be accompanied with tests. -The integration tests are run using [Kitchen][kitchen], -[Kitchen-Terraform][kitchen-terraform], and [InSpec][inspec]. These -tools are packaged within a Docker image for convenience. +The integration tests are run using[Cloud Foundation Toolkit CLI (CFT CLI)][https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/tree/main/cli], +and [Blueprint Test][https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/tree/main/infra/blueprint-test]. The CFT CLI +is packaged within a Docker image for convenience. The general strategy for these tests is to verify the behaviour of the [example modules](./examples/), thus ensuring that the root module, submodules, and example modules are all functionally correct. -Six test-kitchen instances are defined: - -- `deploy-service` -- `node-pool` -- `shared-vpc` -- `simple-regional` -- `simple-zonal` -- `stub-domains` - -The test-kitchen instances in `test/fixtures/` wrap identically-named examples in the `examples/` directory.` - ### Test Environment The easiest way to test the module is in an isolated test project. The setup for such a project is defined in [test/setup](./test/setup/) @@ -101,14 +90,14 @@ noninteractively, using the prepared test project. 1. Run `make docker_run` to start the testing Docker container in interactive mode. -1. Run `kitchen_do create ` to initialize the working +1. Run `cft test run --stage init` to initialize the working directory for an example module. -1. Run `kitchen_do converge ` to apply the example module. +1. Run `cft test run --stage apply` to apply the example module. -1. Run `kitchen_do verify ` to test the example module. +1. Run `cft test run --stage verify` to test the example module. -1. Run `kitchen_do destroy ` to destroy the example module +1. Run `cft test run --stage destroy` to destroy the example module state. ## Linting and Formatting diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 6a9d7d0c24..eb4b6f63d6 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -200,36 +200,52 @@ steps: - verify simple-zonal-private-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalPrivate --stage teardown --verbose'] -- id: converge stub-domains-local +- id: apply stub-domains-local waitFor: - create-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge stub-domains-local'] + args: ['/bin/bash', '-c', 'cft test run TestStubDomains --stage apply --verbose --test-dir test/integration'] - id: verify stub-domains-local waitFor: - - converge stub-domains-local + - apply stub-domains-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify stub-domains-local'] + args: ['/bin/bash', '-c', 'cft test run TestStubDomains --stage verify --verbose --test-dir test/integration'] - id: destroy stub-domains-local waitFor: - verify stub-domains-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy stub-domains-local'] -- id: converge upstream-nameservers-local + args: ['/bin/bash', '-c', 'cft test run TestStubDomains --stage destroy --verbose --test-dir test/integration'] +# Disabled: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/264 +# - id: apply stub-domains-private-local +# waitFor: +# - create-all +# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' +# args: ['/bin/bash', '-c', 'cft test run TestStubDomainsPrivate --stage apply --verbose --test-dir test/integration'] +# - id: verify stub-domains-private-local +# waitFor: +# - apply stub-domains-private-local +# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' +# args: ['/bin/bash', '-c', 'cft test run TestStubDomainsPrivate --stage verify --verbose --test-dir test/integration'] +# - id: destroy stub-domains-private-local +# waitFor: +# - verify stub-domains-private-local +# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' +# args: ['/bin/bash', '-c', 'cft test run TestStubDomainsPrivate --stage destroy --verbose --test-dir test/integration'] +- id: apply upstream-nameservers-local waitFor: - create-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge upstream-nameservers-local'] + args: ['/bin/bash', '-c', 'cft test run TestUpstreamNameservers --stage apply --verbose --test-dir test/integration'] - id: verify upstream-nameservers-local waitFor: - - converge upstream-nameservers-local + - apply upstream-nameservers-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify upstream-nameservers-local'] + args: ['/bin/bash', '-c', 'cft test run TestUpstreamNameservers --stage verify --verbose --test-dir test/integration'] - id: destroy upstream-nameservers-local waitFor: - verify upstream-nameservers-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy upstream-nameservers-local'] + args: ['/bin/bash', '-c', 'cft test run TestUpstreamNameservers --stage destroy --verbose --test-dir test/integration'] - id: converge stub-domains-upstream-nameservers-local waitFor: - create-all diff --git a/test/fixtures/stub_domains/example.tf b/test/fixtures/stub_domains/example.tf index df31547535..815d9f75c9 100644 --- a/test/fixtures/stub_domains/example.tf +++ b/test/fixtures/stub_domains/example.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,10 @@ * limitations under the License. */ +locals { + compute_engine_service_account = var.compute_engine_service_accounts[1] +} + module "example" { source = "../../../examples/stub_domains" @@ -24,6 +28,6 @@ module "example" { subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_accounts[1] + compute_engine_service_account = local.compute_engine_service_account } diff --git a/test/fixtures/stub_domains/outputs.tf b/test/fixtures/stub_domains/outputs.tf index 403576809a..7c5d1b6c7e 100644 --- a/test/fixtures/stub_domains/outputs.tf +++ b/test/fixtures/stub_domains/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,3 +56,11 @@ output "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`." value = module.example.service_account } + +output "random_string" { + value = random_string.suffix.result +} + +output "compute_engine_service_account" { + value = local.compute_engine_service_account +} diff --git a/test/fixtures/stub_domains_private/main.tf b/test/fixtures/stub_domains_private/main.tf index a98f8d6ba6..da6e836cec 100644 --- a/test/fixtures/stub_domains_private/main.tf +++ b/test/fixtures/stub_domains_private/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,10 @@ * limitations under the License. */ +locals { + compute_engine_service_account = var.compute_engine_service_accounts[1] +} + resource "random_string" "suffix" { length = 4 special = false @@ -49,7 +53,7 @@ resource "google_compute_subnetwork" "main" { module "example" { source = "../../../examples/stub_domains_private" - compute_engine_service_account = var.compute_engine_service_accounts[1] + compute_engine_service_account = local.compute_engine_service_account ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name network = google_compute_network.main.name diff --git a/test/fixtures/stub_domains_private/outputs.tf b/test/fixtures/stub_domains_private/outputs.tf index 403576809a..7c5d1b6c7e 100644 --- a/test/fixtures/stub_domains_private/outputs.tf +++ b/test/fixtures/stub_domains_private/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,3 +56,11 @@ output "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`." value = module.example.service_account } + +output "random_string" { + value = random_string.suffix.result +} + +output "compute_engine_service_account" { + value = local.compute_engine_service_account +} diff --git a/test/fixtures/upstream_nameservers/example.tf b/test/fixtures/upstream_nameservers/example.tf index 81d60e8559..1488134988 100644 --- a/test/fixtures/upstream_nameservers/example.tf +++ b/test/fixtures/upstream_nameservers/example.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,10 @@ * limitations under the License. */ +locals { + compute_engine_service_account = var.compute_engine_service_accounts[1] +} + module "example" { source = "../../../examples/upstream_nameservers" @@ -24,6 +28,6 @@ module "example" { subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_accounts[1] + compute_engine_service_account = local.compute_engine_service_account } diff --git a/test/fixtures/upstream_nameservers/outputs.tf b/test/fixtures/upstream_nameservers/outputs.tf index 403576809a..7c5d1b6c7e 100644 --- a/test/fixtures/upstream_nameservers/outputs.tf +++ b/test/fixtures/upstream_nameservers/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,3 +56,11 @@ output "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`." value = module.example.service_account } + +output "random_string" { + value = random_string.suffix.result +} + +output "compute_engine_service_account" { + value = local.compute_engine_service_account +} diff --git a/test/integration/node_pool/testdata/TestNodePool.json b/test/integration/node_pool/testdata/TestNodePool.json index 2175d01c60..b99dfd8a88 100644 --- a/test/integration/node_pool/testdata/TestNodePool.json +++ b/test/integration/node_pool/testdata/TestNodePool.json @@ -249,7 +249,6 @@ "name": "default-pool", "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", - "podIpv4RangeUtilization": 0.0624, "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, @@ -302,7 +301,6 @@ "name": "nap-e2-medium-1d469r1p", "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", - "podIpv4RangeUtilization": 0.0624, "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "placementPolicy": {}, @@ -395,7 +393,6 @@ "name": "pool-01", "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", - "podIpv4RangeUtilization": 0.0624, "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, @@ -490,7 +487,6 @@ "name": "pool-02", "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", - "podIpv4RangeUtilization": 0.0624, "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, @@ -583,7 +579,6 @@ "networkConfig": { "enablePrivateNodes": false, "podIpv4CidrBlock": "172.16.0.0/18", - "podIpv4RangeUtilization": 0.0625, "podRange": "test" }, "podIpv4CidrSize": 24, @@ -671,7 +666,6 @@ "name": "pool-04", "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", - "podIpv4RangeUtilization": 0.0624, "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, @@ -759,7 +753,6 @@ "name": "pool-05", "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", - "podIpv4RangeUtilization": 0.0624, "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, diff --git a/test/integration/stub_domains/controls/gcloud.rb b/test/integration/stub_domains/controls/gcloud.rb deleted file mode 100644 index 8131dc371f..0000000000 --- a/test/integration/stub_domains/controls/gcloud.rb +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -project_id = attribute('project_id') -location = attribute('location') -cluster_name = attribute('cluster_name') - -control "gcloud" do - title "Google Compute Engine GKE configuration" - describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq '' } - - let!(:data) do - if subject.exit_status == 0 - JSON.parse(subject.stdout) - else - {} - end - end - - describe "cluster" do - it "is running" do - expect(data['status']).to eq 'RUNNING' - end - - it "has the expected addon settings" do - expect(data['addonsConfig']).to include( - "horizontalPodAutoscaling" => {}, - "httpLoadBalancing" => {}, - "kubernetesDashboard" => { - "disabled" => true, - }, - "networkPolicyConfig" => { - "disabled" => true, - }, - ) - end - end - end -end diff --git a/test/integration/stub_domains/controls/kubectl.rb b/test/integration/stub_domains/controls/kubectl.rb deleted file mode 100644 index 861bedb9d7..0000000000 --- a/test/integration/stub_domains/controls/kubectl.rb +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'kubeclient' -require 'rest-client' - -require 'base64' - -kubernetes_endpoint = attribute('kubernetes_endpoint') -client_token = attribute('client_token') -ca_certificate = attribute('ca_certificate') - -control "kubectl" do - title "Kubernetes configuration" - - describe "kubernetes" do - let(:kubernetes_http_endpoint) { "https://#{kubernetes_endpoint}/api" } - let(:client) do - cert_store = OpenSSL::X509::Store.new - cert_store.add_cert(OpenSSL::X509::Certificate.new(Base64.decode64(ca_certificate))) - Kubeclient::Client.new( - kubernetes_http_endpoint, - "v1", - ssl_options: { - cert_store: cert_store, - verify_ssl: OpenSSL::SSL::VERIFY_PEER, - }, - auth_options: { - bearer_token: Base64.decode64(client_token), - }, - ) - end - - describe "configmap" do - describe "kube-dns" do - let(:kubedns_configmap) { client.get_config_map("kube-dns", "kube-system") } - - it "is managed by Terraform" do - expect(kubedns_configmap.metadata.managedFields[0].manager).to eq "Terraform" - end - - it "reflects the stub_domains configuration" do - expect(JSON.parse(kubedns_configmap.data.stubDomains)).to eq({ - "example.com" => [ - "10.254.154.11", - "10.254.154.12", - ], - "example.net" => [ - "10.254.154.11", - "10.254.154.12", - ], - }) - end - end - - describe "ipmasq" do - let(:ipmasq_configmap) { client.get_config_map("ip-masq-agent", "kube-system") } - - it "is created by Terraform" do - expect(ipmasq_configmap.metadata.labels.maintained_by).to eq "terraform" - end - - it "is configured properly" do - expect(YAML.load(ipmasq_configmap.data.config)).to eq({ - "nonMasqueradeCIDRs" => [ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - ], - "resyncInterval" => "60s", - "masqLinkLocal" => false, - }) - end - end - end - end -end diff --git a/test/integration/stub_domains/inspec.yml b/test/integration/stub_domains/inspec.yml deleted file mode 100644 index b6e3affda3..0000000000 --- a/test/integration/stub_domains/inspec.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: stub_domain -attributes: - - name: project_id - required: true - type: string - - name: location - required: true - type: string - - name: cluster_name - required: true - type: string - - name: kubernetes_endpoint - required: true - type: string - - name: client_token - required: true - type: string - - name: ca_certificate - required: true - type: string diff --git a/test/integration/stub_domains/stub_domains_test.go b/test/integration/stub_domains/stub_domains_test.go new file mode 100644 index 0000000000..47fd0939b7 --- /dev/null +++ b/test/integration/stub_domains/stub_domains_test.go @@ -0,0 +1,103 @@ +// Copyright 2024-2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package node_pool + +import ( + "fmt" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/cai" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" +) + +func TestStubDomains(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + testutils.TGKEVerify(t, bpt, assert) // Verify Resources + + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + randomString := bpt.GetStringOutput("random_string") + kubernetesEndpoint := bpt.GetStringOutput("kubernetes_endpoint") + nodeServiceAccount := bpt.GetStringOutput("compute_engine_service_account") + + // Retrieve Project CAI + projectCAI := cai.GetProjectResources(t, projectId, cai.WithAssetTypes([]string{"container.googleapis.com/Cluster"})) + + // Retrieve Cluster from CAI + // Equivalent gcloud describe command (classic) + // cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName) + cluster := projectCAI.Get("#(name=\"" + clusterResourceName + "\").resource.data") + + // Setup golden image with sanitizers + g := golden.NewOrUpdate(t, cluster.String(), + golden.WithSanitizer(golden.StringSanitizer(nodeServiceAccount, "NODE_SERVICE_ACCOUNT")), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(randomString, "RANDOM_STRING")), + golden.WithSanitizer(golden.StringSanitizer(kubernetesEndpoint, "KUBERNETES_ENDPOINT")), + ) + + // Cluster Assertions + testutils.TGKEAssertGolden(assert, g, &cluster, []string{}, []string{"monitoringConfig.componentConfig.enableComponents"}) // TODO: enableComponents is UL + + // K8s Assertions + // CAI does not include k8s.io/ConfigMap + gcloud.Runf(t, "container clusters get-credentials %s --region %s --project %s", clusterName, location, projectId) + k8sOpts := k8s.NewKubectlOptions(fmt.Sprintf("gke_%s_%s_%s", projectId, location, clusterName), "", "") + + // kube-dns + listKubeDnsConfigMap, err := k8s.RunKubectlAndGetOutputE(t, k8sOpts, "get", "configmap", "kube-dns", "-n", "kube-system", "-o", "json", "--show-managed-fields") + assert.NoError(err) + kubeDnsCM := utils.ParseKubectlJSONResult(t, listKubeDnsConfigMap) + assert.Contains("kube-dns", kubeDnsCM.Get("metadata.name").String(), "kube-dns configmap is present") + assert.Equal("Terraform", kubeDnsCM.Get("metadata.managedFields.0.manager").String(), "kube-dns configmap is managed by Terraform") + //assert.Equal("[\"8.8.8.8\",\"8.8.4.4\"]\n", kubeDnsCM.Get("data.stubDomains").String(), "kube-dns configmap reflects the upstream_nameservers configuration") + + assert.JSONEq(`{ + "example.com": [ + "10.254.154.11", + "10.254.154.12" + ], + "example.net": [ + "10.254.154.11", + "10.254.154.12" + ] + }`, + kubeDnsCM.Get("data.stubDomains").String(), "kube-dns configmap the expected stubdomains") + + // ip-masq-agent + listIpMasqAgentConfigMap, err := k8s.RunKubectlAndGetOutputE(t, k8sOpts, "get", "configmap", "ip-masq-agent", "-n", "kube-system", "-o", "json", "--show-managed-fields") + assert.NoError(err) + ipMasqAgentConfigMap := utils.ParseKubectlJSONResult(t, listIpMasqAgentConfigMap) + assert.Contains("ip-masq-agent", ipMasqAgentConfigMap.Get("metadata.name").String(), "ip-masq-agent configmap is present") + assert.Equal("terraform", ipMasqAgentConfigMap.Get("metadata.labels.maintained_by").String(), "ip-masq-agent configmap is maintained_by Terraform") + assert.Equal("nonMasqueradeCIDRs:\n - 10.0.0.0/8\n - 172.16.0.0/12\n - 192.168.0.0/16\nresyncInterval: 60s\nmasqLinkLocal: false\n", ipMasqAgentConfigMap.Get("data.config").String(), "ip-masq-agent configmap is configured properly") + + }) + bpt.Test() +} diff --git a/test/integration/stub_domains/testdata/TestStubDomains.json b/test/integration/stub_domains/testdata/TestStubDomains.json new file mode 100644 index 0000000000..07794f9ac5 --- /dev/null +++ b/test/integration/stub_domains/testdata/TestStubDomains.json @@ -0,0 +1,197 @@ +{ + "addonsConfig": { + "configConnectorConfig": {}, + "dnsCacheConfig": {}, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + } + }, + "autopilot": {}, + "autoscaling": { + "autoscalingProfile": "BALANCED" + }, + "binaryAuthorization": {}, + "clusterIpv4Cidr": "192.168.0.0/18", + "controlPlaneEndpointsConfig": { + "dnsEndpointConfig": { + "allowExternalTraffic": false + }, + "ipEndpointsConfig": { + "authorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "enablePublicEndpoint": true, + "enabled": true, + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "KUBERNETES_ENDPOINT" + } + }, + "currentNodeCount": 3, + "databaseEncryption": { + "currentState": "CURRENT_STATE_DECRYPTED", + "state": "DECRYPTED" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "KUBERNETES_ENDPOINT", + "enterpriseConfig": { + "clusterTier": "STANDARD" + }, + "identityServiceConfig": {}, + "ipAllocationPolicy": { + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "cft-gke-test-pods-RANDOM_STRING", + "defaultPodIpv4RangeUtilization": 0.0469, + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "cft-gke-test-services-RANDOM_STRING", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "78cdf2f6", + "legacyAbac": {}, + "location": "us-central1", + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clientCertificateConfig": {} + }, + "masterAuthorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "meshCertificates": { + "enableCertificates": false + }, + "monitoringConfig": { + "advancedDatapathObservabilityConfig": {}, + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "STORAGE", + "HPA", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "STATEFULSET", + "CADVISOR", + "KUBELET" + ] + }, + "managedPrometheusConfig": { + "enabled": true + } + }, + "monitoringService": "monitoring.googleapis.com/kubernetes", + "name": "stub-domains-cluster-RANDOM_STRING", + "network": "cft-gke-test-RANDOM_STRING", + "networkConfig": { + "defaultSnatStatus": {}, + "network": "projects/PROJECT_ID/global/networks/cft-gke-test-RANDOM_STRING", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/us-central1/subnetworks/cft-gke-test-RANDOM_STRING" + }, + "nodeConfig": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "gcfsConfig": {}, + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-stub-domains-cluster-RANDOM_STRING", + "gke-stub-domains-cluster-RANDOM_STRING-default-pool" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "gcfsConfig": {}, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "nodeKubeletConfig": {} + } + }, + "notificationConfig": { + "pubsub": {} + }, + "privateClusterConfig": { + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "KUBERNETES_ENDPOINT" + }, + "rbacBindingConfig": { + "enableInsecureBindingSystemAuthenticated": true, + "enableInsecureBindingSystemUnauthenticated": true + }, + "releaseChannel": { + "channel": "REGULAR" + }, + "resourceLabels": { + "goog-terraform-provisioned": "true" + }, + "securityPostureConfig": { + "mode": "DISABLED", + "vulnerabilityMode": "VULNERABILITY_DISABLED" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/stub-domains-cluster-RANDOM_STRING", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "cft-gke-test-RANDOM_STRING", + "verticalPodAutoscaling": {}, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "us-central1" +} diff --git a/test/integration/stub_domains_private/controls/gcloud.rb b/test/integration/stub_domains_private/controls/gcloud.rb deleted file mode 100644 index 2efafdb393..0000000000 --- a/test/integration/stub_domains_private/controls/gcloud.rb +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -control "gcloud" do - title "Google Compute Engine GKE configuration" - describe command( - "gcloud --project=#{attribute("project_id")} container clusters --zone=#{attribute("location")} describe " \ - "#{attribute("cluster_name")} --format=json" - ) do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq "" } - - let!(:data) do - if subject.exit_status == 0 - JSON.parse(subject.stdout) - else - {} - end - end - - describe "cluster" do - it "is running" do - expect(data["status"]).to eq "RUNNING" - end - - it "does not use the private endpoint" do - expect(data["privateClusterConfig"]["enablePrivateEndpoint"]).to eq false - end - - it "uses private nodes" do - expect(data["privateClusterConfig"]["enablePrivateNodes"]).to eq true - end - - it "has the expected addon settings" do - expect(data["addonsConfig"]).to include( - "horizontalPodAutoscaling" => {}, - "httpLoadBalancing" => {}, - "kubernetesDashboard" => { - "disabled" => true, - }, - "networkPolicyConfig" => { - "disabled" => true, - }, - ) - end - end - end -end diff --git a/test/integration/stub_domains_private/controls/kubectl.rb b/test/integration/stub_domains_private/controls/kubectl.rb deleted file mode 100644 index 1c819c209e..0000000000 --- a/test/integration/stub_domains_private/controls/kubectl.rb +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require "base64" -require "kubeclient" -require "rest-client" - -control "kubectl" do - title "Kubernetes configuration" - - describe "kubernetes" do - let(:kubernetes_http_endpoint) { "https://#{attribute("kubernetes_endpoint")}/api" } - - let(:client) do - cert_store = OpenSSL::X509::Store.new - cert_store.add_cert(OpenSSL::X509::Certificate.new(Base64.decode64(attribute("ca_certificate")))) - Kubeclient::Client.new( - kubernetes_http_endpoint, - "v1", - ssl_options: { - cert_store: cert_store, - verify_ssl: OpenSSL::SSL::VERIFY_PEER, - }, - auth_options: { - bearer_token: Base64.decode64(attribute("client_token")), - }, - ) - end - - describe "configmap" do - describe "kube-dns" do - let(:kubedns_configmap) { client.get_config_map("kube-dns", "kube-system") } - - it "is managed by Terraform" do - expect(kubedns_configmap.metadata.managedFields[0].manager).to eq "Terraform" - end - - it "reflects the stub_domains configuration" do - expect(JSON.parse(kubedns_configmap.data.stubDomains)).to eq({ - "example.com" => [ - "10.254.154.11", - "10.254.154.12", - ], - "example.net" => [ - "10.254.154.11", - "10.254.154.12", - ], - }) - end - end - end - end -end diff --git a/test/integration/stub_domains_private/inspec.yml b/test/integration/stub_domains_private/inspec.yml deleted file mode 100644 index 8db53ccb84..0000000000 --- a/test/integration/stub_domains_private/inspec.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: stub_domains_private -attributes: - - name: ca_certificate - required: true - type: string - - name: client_token - required: true - type: string - - name: cluster_name - required: true - type: string - - name: kubernetes_endpoint - required: true - type: string - - name: location - required: true - type: string - - name: project_id - required: true - type: string diff --git a/test/integration/stub_domains_private/stub_domains_private_test.go b/test/integration/stub_domains_private/stub_domains_private_test.go new file mode 100644 index 0000000000..d086f1b2bb --- /dev/null +++ b/test/integration/stub_domains_private/stub_domains_private_test.go @@ -0,0 +1,94 @@ +// Copyright 2024-2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package node_pool + +import ( + "fmt" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/cai" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" +) + +func TestStubDomainsPrivate(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + testutils.TGKEVerify(t, bpt, assert) // Verify Resources + + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + randomString := bpt.GetStringOutput("random_string") + kubernetesEndpoint := bpt.GetStringOutput("kubernetes_endpoint") + nodeServiceAccount := bpt.GetStringOutput("compute_engine_service_account") + + // Retrieve Project CAI + projectCAI := cai.GetProjectResources(t, projectId, cai.WithAssetTypes([]string{"container.googleapis.com/Cluster"})) + + // Retrieve Cluster from CAI + // Equivalent gcloud describe command (classic) + // cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName) + cluster := projectCAI.Get("#(name=\"" + clusterResourceName + "\").resource.data") + + // Setup golden image with sanitizers + g := golden.NewOrUpdate(t, cluster.String(), + golden.WithSanitizer(golden.StringSanitizer(nodeServiceAccount, "NODE_SERVICE_ACCOUNT")), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(randomString, "RANDOM_STRING")), + golden.WithSanitizer(golden.StringSanitizer(kubernetesEndpoint, "KUBERNETES_ENDPOINT")), + ) + + // Cluster Assertions + testutils.TGKEAssertGolden(assert, g, &cluster, []string{}, []string{"monitoringConfig.componentConfig.enableComponents"}) // TODO: enableComponents is UL + + // K8s Assertions + // CAI does not include k8s.io/ConfigMap + gcloud.Runf(t, "container clusters get-credentials %s --region %s --project %s", clusterName, location, projectId) + k8sOpts := k8s.NewKubectlOptions(fmt.Sprintf("gke_%s_%s_%s", projectId, location, clusterName), "", "") + + // kube-dns + listKubeDnsConfigMap, err := k8s.RunKubectlAndGetOutputE(t, k8sOpts, "get", "configmap", "kube-dns", "-n", "kube-system", "-o", "json", "--show-managed-fields") + assert.NoError(err) + kubeDnsCM := utils.ParseKubectlJSONResult(t, listKubeDnsConfigMap) + assert.Contains("kube-dns", kubeDnsCM.Get("metadata.name").String(), "kube-dns configmap is present") + assert.Equal("Terraform", kubeDnsCM.Get("metadata.managedFields.0.manager").String(), "kube-dns configmap is managed by Terraform") + //assert.Equal("[\"8.8.8.8\",\"8.8.4.4\"]\n", kubeDnsCM.Get("data.stubDomains").String(), "kube-dns configmap reflects the upstream_nameservers configuration") + + assert.JSONEq(`{ + "example.com": [ + "10.254.154.11", + "10.254.154.12" + ], + "example.net": [ + "10.254.154.11", + "10.254.154.12" + ] + }`, + kubeDnsCM.Get("data.stubDomains").String(), "kube-dns configmap the expected stubdomains") + }) + bpt.Test() +} diff --git a/test/integration/stub_domains_private/testdata/TestStubDomainsPrivate.json b/test/integration/stub_domains_private/testdata/TestStubDomainsPrivate.json new file mode 100644 index 0000000000..265e4cb2f2 --- /dev/null +++ b/test/integration/stub_domains_private/testdata/TestStubDomainsPrivate.json @@ -0,0 +1,199 @@ +{ + "addonsConfig": { + "configConnectorConfig": {}, + "dnsCacheConfig": {}, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + } + }, + "autopilot": {}, + "autoscaling": { + "autoscalingProfile": "BALANCED" + }, + "binaryAuthorization": {}, + "clusterIpv4Cidr": "192.168.0.0/18", + "controlPlaneEndpointsConfig": { + "dnsEndpointConfig": { + "allowExternalTraffic": false + }, + "ipEndpointsConfig": { + "authorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "enablePublicEndpoint": true, + "enabled": true, + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "KUBERNETES_ENDPOINT" + } + }, + "currentNodeCount": 3, + "databaseEncryption": { + "currentState": "CURRENT_STATE_DECRYPTED", + "state": "DECRYPTED" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "KUBERNETES_ENDPOINT", + "enterpriseConfig": { + "clusterTier": "STANDARD" + }, + "identityServiceConfig": {}, + "ipAllocationPolicy": { + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "cft-gke-test-pods-RANDOM_STRING", + "defaultPodIpv4RangeUtilization": 0.0469, + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "cft-gke-test-services-RANDOM_STRING", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "78cdf2f6", + "legacyAbac": {}, + "location": "us-central1", + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clientCertificateConfig": {} + }, + "masterAuthorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "meshCertificates": { + "enableCertificates": false + }, + "monitoringConfig": { + "advancedDatapathObservabilityConfig": {}, + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "STORAGE", + "HPA", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "STATEFULSET", + "CADVISOR", + "KUBELET" + ] + }, + "managedPrometheusConfig": { + "enabled": true + } + }, + "monitoringService": "monitoring.googleapis.com/kubernetes", + "name": "upstream-nameservers-cluster-RANDOM_STRING", + "network": "cft-gke-test-RANDOM_STRING", + "networkConfig": { + "defaultSnatStatus": {}, + "network": "projects/PROJECT_ID/global/networks/cft-gke-test-RANDOM_STRING", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/us-central1/subnetworks/cft-gke-test-RANDOM_STRING" + }, + "nodeConfig": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "gcfsConfig": {}, + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-upstream-nameservers-cluster-RANDOM_STRING", + "gke-upstream-nameservers-cluster-RANDOM_STRING-default-pool" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "gcfsConfig": {}, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "nodeKubeletConfig": {} + } + }, + "notificationConfig": { + "pubsub": {} + }, + "privateClusterConfig": { + "enablePrivateEndpoint": false, + "enablePrivateNodes": true, + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "KUBERNETES_ENDPOINT" + }, + "rbacBindingConfig": { + "enableInsecureBindingSystemAuthenticated": true, + "enableInsecureBindingSystemUnauthenticated": true + }, + "releaseChannel": { + "channel": "REGULAR" + }, + "resourceLabels": { + "goog-terraform-provisioned": "true" + }, + "securityPostureConfig": { + "mode": "DISABLED", + "vulnerabilityMode": "VULNERABILITY_DISABLED" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/upstream-nameservers-cluster-RANDOM_STRING", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "cft-gke-test-RANDOM_STRING", + "verticalPodAutoscaling": {}, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "us-central1" +} diff --git a/test/integration/upstream_nameservers/controls/gcloud.rb b/test/integration/upstream_nameservers/controls/gcloud.rb deleted file mode 100644 index 8131dc371f..0000000000 --- a/test/integration/upstream_nameservers/controls/gcloud.rb +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -project_id = attribute('project_id') -location = attribute('location') -cluster_name = attribute('cluster_name') - -control "gcloud" do - title "Google Compute Engine GKE configuration" - describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq '' } - - let!(:data) do - if subject.exit_status == 0 - JSON.parse(subject.stdout) - else - {} - end - end - - describe "cluster" do - it "is running" do - expect(data['status']).to eq 'RUNNING' - end - - it "has the expected addon settings" do - expect(data['addonsConfig']).to include( - "horizontalPodAutoscaling" => {}, - "httpLoadBalancing" => {}, - "kubernetesDashboard" => { - "disabled" => true, - }, - "networkPolicyConfig" => { - "disabled" => true, - }, - ) - end - end - end -end diff --git a/test/integration/upstream_nameservers/controls/kubectl.rb b/test/integration/upstream_nameservers/controls/kubectl.rb deleted file mode 100644 index 788c9f11d0..0000000000 --- a/test/integration/upstream_nameservers/controls/kubectl.rb +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'kubeclient' -require 'rest-client' - -require 'base64' - -kubernetes_endpoint = attribute('kubernetes_endpoint') -client_token = attribute('client_token') -ca_certificate = attribute('ca_certificate') - -control "kubectl" do - title "Kubernetes configuration" - - describe "kubernetes" do - let(:kubernetes_http_endpoint) { "https://#{kubernetes_endpoint}/api" } - let(:client) do - cert_store = OpenSSL::X509::Store.new - cert_store.add_cert(OpenSSL::X509::Certificate.new(Base64.decode64(ca_certificate))) - Kubeclient::Client.new( - kubernetes_http_endpoint, - "v1", - ssl_options: { - cert_store: cert_store, - verify_ssl: OpenSSL::SSL::VERIFY_PEER, - }, - auth_options: { - bearer_token: Base64.decode64(client_token), - }, - ) - end - - describe "configmap" do - describe "kube-dns" do - let(:kubedns_configmap) { client.get_config_map("kube-dns", "kube-system") } - - it "is managed by Terraform" do - expect(kubedns_configmap.metadata.managedFields[0].manager).to eq "Terraform" - end - - it "reflects the upstream_nameservers configuration" do - expect(JSON.parse(kubedns_configmap.data.upstreamNameservers)).to eq(["8.8.8.8", "8.8.4.4"]) - end - end - - describe "ipmasq" do - let(:ipmasq_configmap) { client.get_config_map("ip-masq-agent", "kube-system") } - - it "is created by Terraform" do - expect(ipmasq_configmap.metadata.labels.maintained_by).to eq "terraform" - end - - it "is configured properly" do - expect(YAML.load(ipmasq_configmap.data.config)).to eq({ - "nonMasqueradeCIDRs" => [ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - ], - "resyncInterval" => "60s", - "masqLinkLocal" => false, - }) - end - end - end - end -end diff --git a/test/integration/upstream_nameservers/inspec.yml b/test/integration/upstream_nameservers/inspec.yml deleted file mode 100644 index 5376d0c447..0000000000 --- a/test/integration/upstream_nameservers/inspec.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: upstream_nameservers -attributes: - - name: project_id - required: true - type: string - - name: location - required: true - type: string - - name: cluster_name - required: true - type: string - - name: kubernetes_endpoint - required: true - type: string - - name: client_token - required: true - type: string - - name: ca_certificate - required: true - type: string diff --git a/test/integration/upstream_nameservers/testdata/TestUpstreamNameservers.json b/test/integration/upstream_nameservers/testdata/TestUpstreamNameservers.json new file mode 100644 index 0000000000..f650e4898c --- /dev/null +++ b/test/integration/upstream_nameservers/testdata/TestUpstreamNameservers.json @@ -0,0 +1,197 @@ +{ + "addonsConfig": { + "configConnectorConfig": {}, + "dnsCacheConfig": {}, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + } + }, + "autopilot": {}, + "autoscaling": { + "autoscalingProfile": "BALANCED" + }, + "binaryAuthorization": {}, + "clusterIpv4Cidr": "192.168.0.0/18", + "controlPlaneEndpointsConfig": { + "dnsEndpointConfig": { + "allowExternalTraffic": false + }, + "ipEndpointsConfig": { + "authorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "enablePublicEndpoint": true, + "enabled": true, + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "KUBERNETES_ENDPOINT" + } + }, + "currentNodeCount": 3, + "databaseEncryption": { + "currentState": "CURRENT_STATE_DECRYPTED", + "state": "DECRYPTED" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "KUBERNETES_ENDPOINT", + "enterpriseConfig": { + "clusterTier": "STANDARD" + }, + "identityServiceConfig": {}, + "ipAllocationPolicy": { + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "cft-gke-test-pods-RANDOM_STRING", + "defaultPodIpv4RangeUtilization": 0.0469, + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "cft-gke-test-services-RANDOM_STRING", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "78cdf2f6", + "legacyAbac": {}, + "location": "us-central1", + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clientCertificateConfig": {} + }, + "masterAuthorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "meshCertificates": { + "enableCertificates": false + }, + "monitoringConfig": { + "advancedDatapathObservabilityConfig": {}, + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "STORAGE", + "HPA", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "STATEFULSET", + "CADVISOR", + "KUBELET" + ] + }, + "managedPrometheusConfig": { + "enabled": true + } + }, + "monitoringService": "monitoring.googleapis.com/kubernetes", + "name": "upstream-nameservers-cluster-RANDOM_STRING", + "network": "cft-gke-test-RANDOM_STRING", + "networkConfig": { + "defaultSnatStatus": {}, + "network": "projects/PROJECT_ID/global/networks/cft-gke-test-RANDOM_STRING", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/us-central1/subnetworks/cft-gke-test-RANDOM_STRING" + }, + "nodeConfig": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "gcfsConfig": {}, + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-upstream-nameservers-cluster-RANDOM_STRING", + "gke-upstream-nameservers-cluster-RANDOM_STRING-default-pool" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "gcfsConfig": {}, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "nodeKubeletConfig": {} + } + }, + "notificationConfig": { + "pubsub": {} + }, + "privateClusterConfig": { + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "KUBERNETES_ENDPOINT" + }, + "rbacBindingConfig": { + "enableInsecureBindingSystemAuthenticated": true, + "enableInsecureBindingSystemUnauthenticated": true + }, + "releaseChannel": { + "channel": "REGULAR" + }, + "resourceLabels": { + "goog-terraform-provisioned": "true" + }, + "securityPostureConfig": { + "mode": "DISABLED", + "vulnerabilityMode": "VULNERABILITY_DISABLED" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/upstream-nameservers-cluster-RANDOM_STRING", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "cft-gke-test-RANDOM_STRING", + "verticalPodAutoscaling": {}, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "us-central1" +} diff --git a/test/integration/upstream_nameservers/upstream_nameservers_test.go b/test/integration/upstream_nameservers/upstream_nameservers_test.go new file mode 100644 index 0000000000..4349d732d9 --- /dev/null +++ b/test/integration/upstream_nameservers/upstream_nameservers_test.go @@ -0,0 +1,90 @@ +// Copyright 2024-2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package node_pool + +import ( + "fmt" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/cai" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" +) + +func TestUpstreamNameservers(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + testutils.TGKEVerify(t, bpt, assert) // Verify Resources + + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + randomString := bpt.GetStringOutput("random_string") + kubernetesEndpoint := bpt.GetStringOutput("kubernetes_endpoint") + nodeServiceAccount := bpt.GetStringOutput("compute_engine_service_account") + + // Retrieve Project CAI + projectCAI := cai.GetProjectResources(t, projectId, cai.WithAssetTypes([]string{"container.googleapis.com/Cluster"})) + + // Retrieve Cluster from CAI + // Equivalent gcloud describe command (classic) + // cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName) + cluster := projectCAI.Get("#(name=\"" + clusterResourceName + "\").resource.data") + + // Setup golden image with sanitizers + g := golden.NewOrUpdate(t, cluster.String(), + golden.WithSanitizer(golden.StringSanitizer(nodeServiceAccount, "NODE_SERVICE_ACCOUNT")), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(randomString, "RANDOM_STRING")), + golden.WithSanitizer(golden.StringSanitizer(kubernetesEndpoint, "KUBERNETES_ENDPOINT")), + ) + + // Cluster Assertions + testutils.TGKEAssertGolden(assert, g, &cluster, []string{}, []string{"monitoringConfig.componentConfig.enableComponents"}) // TODO: enableComponents is UL + + // K8s Assertions + // CAI does not include k8s.io/ConfigMap + gcloud.Runf(t, "container clusters get-credentials %s --region %s --project %s", clusterName, location, projectId) + k8sOpts := k8s.NewKubectlOptions(fmt.Sprintf("gke_%s_%s_%s", projectId, location, clusterName), "", "") + + // kube-dns + listKubeDnsConfigMap, err := k8s.RunKubectlAndGetOutputE(t, k8sOpts, "get", "configmap", "kube-dns", "-n", "kube-system", "-o", "json", "--show-managed-fields") + assert.NoError(err) + kubeDnsCM := utils.ParseKubectlJSONResult(t, listKubeDnsConfigMap) + assert.Contains("kube-dns", kubeDnsCM.Get("metadata.name").String(), "kube-dns configmap is present") + assert.Equal("Terraform", kubeDnsCM.Get("metadata.managedFields.0.manager").String(), "kube-dns configmap is managed by Terraform") + assert.Equal("[\"8.8.8.8\",\"8.8.4.4\"]\n", kubeDnsCM.Get("data.upstreamNameservers").String(), "kube-dns configmap reflects the upstream_nameservers configuration") + + // ip-masq-agent + listIpMasqAgentConfigMap, err := k8s.RunKubectlAndGetOutputE(t, k8sOpts, "get", "configmap", "ip-masq-agent", "-n", "kube-system", "-o", "json", "--show-managed-fields") + assert.NoError(err) + ipMasqAgentConfigMap := utils.ParseKubectlJSONResult(t, listIpMasqAgentConfigMap) + assert.Contains("ip-masq-agent", ipMasqAgentConfigMap.Get("metadata.name").String(), "ip-masq-agent configmap is present") + assert.Equal("terraform", ipMasqAgentConfigMap.Get("metadata.labels.maintained_by").String(), "ip-masq-agent configmap is maintained_by Terraform") + assert.Equal("nonMasqueradeCIDRs:\n - 10.0.0.0/8\n - 172.16.0.0/12\n - 192.168.0.0/16\nresyncInterval: 60s\nmasqLinkLocal: false\n", ipMasqAgentConfigMap.Get("data.config").String(), "ip-masq-agent configmap is configured properly") + }) + bpt.Test() +}