Merge pull request #8 from alexkonkin/feature/vm_module_add_ci_pipeline

aaron-lane · web-flow · commit e1da025822b8 · 2019-05-30T12:17:21.000-04:00
changes for concourse ci integration
diff --git a/Makefile b/Makefile
@@ -12,50 +12,54 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Please note that this file was generated from [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template).
+# Please make sure to contribute relevant changes upstream!
+
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
 # Docker build config variables
-CREDENTIALS_PATH ?= /cft/workdir/credentials.json
-DOCKER_ORG := gcr.io/cloud-foundation-cicd
-DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 0.11.10_216.0.0_1.19.1_0.1.10
-DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCKER_TAG_BASE_KITCHEN_TERRAFORM}
-DOCKER_TAG_KITCHEN_TERRAFORM ?= ${DOCKER_TAG_BASE_KITCHEN_TERRAFORM}
-DOCKER_IMAGE_KITCHEN_TERRAFORM := cft/kitchen_terraform
+CREDENTIALS_PATH 			?= /cft/workdir/credentials.json
+DOCKER_ORG 				:= gcr.io/cloud-foundation-cicd
+DOCKER_TAG_BASE_KITCHEN_TERRAFORM 	?= 1.2.0
+DOCKER_REPO_BASE_KITCHEN_TERRAFORM 	:= ${DOCKER_ORG}/cft/kitchen-terraform:${DOCKER_TAG_BASE_KITCHEN_TERRAFORM}
+
+
+# All is the first target in the file so it will get picked up when you just run 'make' on its own
+.PHONY: all
+all: check generate_docs
 
-all: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace generate_docs ## Run all linters and update documentation
+# Run all available linters
+.PHONY: check
+check: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace
 
 # The .PHONY directive tells make that this isn't a real target and so
 # the presence of a file named 'check_shell' won't cause this target to stop
 # working
 .PHONY: check_shell
-check_shell: ## Lint shell scripts
+check_shell:
 	@source test/make.sh && check_shell
 
 .PHONY: check_python
-check_python: ## Lint Python source files
+check_python:
 	@source test/make.sh && check_python
 
 .PHONY: check_golang
-check_golang: ## Lint Go source files
+check_golang:
 	@source test/make.sh && golang
 
 .PHONY: check_terraform
 check_terraform:
-	@source ## Lint Terraform source files
+	@source test/make.sh && check_terraform
 
 .PHONY: check_docker
-check_docker: ## Lint Dockerfiles
+check_docker:
 	@source test/make.sh && docker
 
 .PHONY: check_base_files
 check_base_files:
 	@source test/make.sh && basefiles
 
-.PHONY: check_shebangs
-check_shebangs: ## Check that scripts have correct shebangs
-	@source test/make.sh && check_bash
-
 .PHONY: check_trailing_whitespace
 check_trailing_whitespace:
 	@source test/make.sh && check_trailing_whitespace
@@ -65,91 +69,86 @@ test_check_headers:
 	@echo "Testing the validity of the header check"
 	@python test/test_verify_boilerplate.py
 
+#.PHONY: check_headers
+#check_headers:
+#	@source test/make.sh && check_headers
 .PHONY: check_headers
 check_headers: ## Check that source files have appropriate boilerplate
 	@echo "Checking file headers"
 	@python test/verify_boilerplate.py
 
+
 # Integration tests
 .PHONY: test_integration
-test_integration: ## Run integration tests
-	bundle install
-	bundle exec kitchen create
-	bundle exec kitchen converge
-	bundle exec kitchen converge
-	bundle exec kitchen verify
-	bundle exec kitchen destroy
+test_integration:
+	test/ci_integration.sh
 
 .PHONY: generate_docs
-generate_docs: ## Update README documentation for Terraform variables and outputs
+generate_docs:
 	@source test/make.sh && generate_docs
 
-.PHONY: release-new-version
-release-new-version:
-	@source helpers/release-new-version.sh
-
-# Build Docker
-.PHONY: docker_build_kitchen_terraform
-docker_build_kitchen_terraform:
-	docker build -f build/docker/kitchen_terraform/Dockerfile \
-		--build-arg BASE_IMAGE=${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
-		-t ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} .
-
-# Push Docker image
-.PHONY: docker_push_kitchen_terraform
-docker_push_kitchen_terraform:
-	docker tag ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} ${DOCKER_ORG}/${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM}
-	docker push ${DOCKER_ORG}/${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM}
+# Versioning
+.PHONY: version
+version:
+	@source helpers/version-repo.sh
 
 # Run docker
 .PHONY: docker_run
-docker_run: ## Launch a shell within the Docker test environment
+docker_run:
 	docker run --rm -it \
-		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e PROJECT_ID \
+		-e SERVICE_ACCOUNT_JSON \
 		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
 		-v $(CURDIR):/cft/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
-		/bin/bash
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
+		/bin/bash -c "source test/ci_integration.sh && setup_environment && exec /bin/bash"
 
 .PHONY: docker_create
-docker_create: docker_build_kitchen_terraform
+docker_create:
 	docker run --rm -it \
-		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e PROJECT_ID \
+		-e SERVICE_ACCOUNT_JSON \
 		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
 		-v $(CURDIR):/cft/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
-		/bin/bash -c "kitchen create"
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
+		/bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen create"
 
 .PHONY: docker_converge
-docker_converge: ## Run `kitchen converge` within the Docker test environment
+docker_converge:
 	docker run --rm -it \
-		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e PROJECT_ID \
+		-e SERVICE_ACCOUNT_JSON \
 		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
 		-v $(CURDIR):/cft/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
-		/bin/bash -c "kitchen converge && kitchen converge"
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
+		/bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen converge"
 
 .PHONY: docker_verify
-docker_verify: ## Run `kitchen verify` within the Docker test environment
+docker_verify:
 	docker run --rm -it \
-		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e PROJECT_ID \
+		-e SERVICE_ACCOUNT_JSON \
 		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
 		-v $(CURDIR):/cft/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
-		/bin/bash -c "kitchen verify"
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
+		/bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen verify"
 
 .PHONY: docker_destroy
-docker_destroy: ## Run `kitchen destroy` within the Docker test environment
+docker_destroy:
 	docker run --rm -it \
-		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e PROJECT_ID \
+		-e SERVICE_ACCOUNT_JSON \
 		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
 		-v $(CURDIR):/cft/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
-		/bin/bash -c "kitchen destroy"
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
+		/bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen destroy"
 
 .PHONY: test_integration_docker
-test_integration_docker: docker_create docker_converge docker_verify docker_destroy ## Run a full integration test cycle
-	@echo "Running test-kitchen tests in docker"
-
-help: ## Prints help for targets with comments
-	@grep -E '^[a-zA-Z._-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+test_integration_docker:
+	docker run --rm -it \
+		-e PROJECT_ID \
+		-e SERVICE_ACCOUNT_JSON \
+		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
+		-v $(CURDIR):/cft/workdir \
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
+		make test_integration
diff --git a/examples/mig/full/variables.tf b/examples/mig/full/variables.tf
@@ -37,6 +37,16 @@ variable "subnetwork" {
   default     = ""
 }
 
+variable "project_id" {
+  description = "The GCP project to use for integration tests"
+  default     = ""
+}
+
+variable "credentials_path" {
+  description = "The path to the GCP credentials JSON file"
+  default     = ""
+}
+
 variable "subnetwork_project" {
   description = "The project that subnetwork belongs to"
   default     = ""
@@ -241,3 +251,7 @@ variable "autoscaling_lb" {
   type        = "list"
   default     = []
 }
+
+variable "autoscaling_enabled" {
+  description = "Creates an autoscaler for the managed instance group"
+}
diff --git a/examples/umig/full/outputs.tf b/examples/umig/full/outputs.tf
@@ -23,8 +23,3 @@ output "umig_self_links" {
   description = "List of self-links for unmanaged instance groups"
   value       = "${module.umig.self_links}"
 }
-
-output "mig_self_link" {
-  description = "Self-link for managed instance group"
-  value       = "${module.mig.self_link}"
-}
diff --git a/examples/umig/full/variables.tf b/examples/umig/full/variables.tf
@@ -18,6 +18,15 @@
 # Common
 #########
 
+variable "project_id" {
+  description = "The GCP project to use for integration tests"
+}
+
+variable "credentials_path" {
+  description = "The path to the GCP credentials JSON file"
+  default     = ""
+}
+
 variable "hostname" {
   description = "Hostname prefix for instances."
   default     = "default"
diff --git a/modules/compute_instance/README.md b/modules/compute_instance/README.md
@@ -1,6 +1,6 @@
 # Compute Instance
 
-This module is used to create compute instances (and only compute instances) using 
+This module is used to create compute instances (and only compute instances) using
 [google_compute_instance_from_template](https://www.terraform.io/docs/providers/google/r/compute_instance_from_template.html), with no instance groups.
 
 ## Usage
diff --git a/modules/instance_template/README.md b/modules/instance_template/README.md
@@ -1,7 +1,7 @@
 # instance_template
 
 This submodule allows you to create an `google_compute_instance_template`
-resource, which is used as the basis for the other instance, managed, and 
+resource, which is used as the basis for the other instance, managed, and
 unmanaged instance groups submodules.
 
 ## Usage
diff --git a/modules/umig/README.md b/modules/umig/README.md
@@ -1,6 +1,6 @@
 # Unmanaged Instance Group (UMIG)
 
-This module is used to create compute instances using 
+This module is used to create compute instances using
 [google_compute_instance_from_template](https://www.terraform.io/docs/providers/google/r/compute_instance_from_template.html), and [google_compute_instance_groups](https://www.terraform.io/docs/providers/google/r/compute_instance_group.html).
 
 ## Usage
diff --git a/test/ci_integration.sh b/test/ci_integration.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Always clean up.
+DELETE_AT_EXIT="$(mktemp -d)"
+finish() {
+  echo 'BEGIN: finish() trap handler' >&2
+  kitchen destroy "$SUITE"
+  [[ -d "${DELETE_AT_EXIT}" ]] && rm -rf "${DELETE_AT_EXIT}"
+  echo 'END: finish() trap handler' >&2
+}
+
+# Map the input parameters provided by Concourse CI, or whatever mechanism is
+# running the tests to Terraform input variables.  Also setup credentials for
+# use with kitchen-terraform, inspec, and gcloud.
+setup_environment() {
+  local tmpfile
+  tmpfile="$(mktemp)"
+  echo "${SERVICE_ACCOUNT_JSON}" > "${tmpfile}"
+
+  cat "${tmpfile}" > test/fixtures/shared/credentials.json
+
+  client_email="$(jq -r .client_email "${tmpfile}")"
+  # gcloud variables
+  export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${tmpfile}"
+  # Application default credentials (Terraform google provider and inspec-gcp)
+  export GOOGLE_APPLICATION_CREDENTIALS="${tmpfile}"
+
+  # Terraform variables
+  export TF_VAR_project_id="$PROJECT_ID"
+  export TF_VAR_region="${REGION:-us-central1}"
+  export TF_VAR_service_account='{email="'$client_email'", scopes = ["cloud-platform"]}'
+  export TF_VAR_credentials_path_relative="../../shared/credentials.json"
+}
+
+main() {
+  export SUITE="${SUITE:-}"
+
+  set -eu
+  # Setup trap handler to auto-cleanup
+  export TMPDIR="${DELETE_AT_EXIT}"
+  trap finish EXIT
+
+  # Setup environment variables
+  setup_environment
+  set -x
+
+  # Execute the test lifecycle
+  kitchen create "$SUITE"
+  kitchen converge "$SUITE"
+  kitchen verify "$SUITE"
+}
+
+# if script is being executed and not sourced.
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+  main "$@"
+fi
+
diff --git a/test/fixtures/compute_instance/simple/terraform.tfvars b/test/fixtures/compute_instance/simple/terraform.tfvars
@@ -0,0 +1 @@
+../../shared/terraform.tfvars
diff --git a/test/fixtures/mig/autoscaler/terraform.tfvars b/test/fixtures/mig/autoscaler/terraform.tfvars
@@ -0,0 +1 @@
+../../shared/terraform.tfvars
diff --git a/test/integration/instance_simple/controls/instance_simple.rb b/test/integration/instance_simple/controls/instance_simple.rb
@@ -46,7 +46,7 @@
       let(:instance) do
         data.find { |i| i['name'] == "instance-simple-001" }
       end
-      
+
       it "should be in zone us-central1-a}" do
         expect(instance['zone']).to match(/.*us-central1-a$/)
       end
@@ -56,7 +56,7 @@
       let(:instance) do
         data.find { |i| i['name'] == "instance-simple-002" }
       end
-      
+
       it "should be in zone us-central1-b}" do
         expect(instance['zone']).to match(/.*us-central1-b$/)
       end
@@ -66,7 +66,7 @@
       let(:instance) do
         data.find { |i| i['name'] == "instance-simple-003" }
       end
-      
+
       it "should be in zone us-central1-c}" do
         expect(instance['zone']).to match(/.*us-central1-c$/)
       end
@@ -76,7 +76,7 @@
       let(:instance) do
         data.find { |i| i['name'] == "instance-simple-004" }
       end
-      
+
       it "should be in zone us-central1-f}" do
         expect(instance['zone']).to match(/.*us-central1-f$/)
       end
diff --git a/test/integration/umig_named_ports/controls/umig_named_ports.rb b/test/integration/umig_named_ports/controls/umig_named_ports.rb
diff --git a/test/integration/umig_simple/controls/umig_simple.rb b/test/integration/umig_simple/controls/umig_simple.rb
diff --git a/test/integration/umig_static_ips/controls/umig_static_ips.rb b/test/integration/umig_static_ips/controls/umig_static_ips.rb

Original file line number	Diff line number	Diff line change
`@@ -23,8 +23,3 @@ output "umig_self_links" {`
`23`	`23`	`description = "List of self-links for unmanaged instance groups"`
`24`	`24`	`value = "${module.umig.self_links}"`
`25`	`25`	`}`
`26`		`-`
`27`		`-output "mig_self_link" {`
`28`		`- description = "Self-link for managed instance group"`
`29`		`- value = "${module.mig.self_link}"`
`30`		`-}`