Use S3 bucket for CI e2e state (#436)

* Use S3 bucket for CI e2e state

* Use terraform output for ip address

* Move terraform into PATH

* Fix bash loop

* Update bucket name

* Hopefully fix ssh key

* Fix ssh key import

* Hopefully final fix for s3-based state

Author: alvaroaleman

hack/ci-e2e-test.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
 function cleanup {
     set +e
@@ -9,39 +9,56 @@ function cleanup {
     echo "Cleaning up machines."
     ./test/tools/integration/cleanup_machines.sh
 
+    cd test/tools/integration
     for try in {1..20}; do
       # Clean up master
       echo "Cleaning up controller, attempt ${try}"
-      make -C test/tools/integration destroy
+      # Clean up only the server, we want to keep the key as only one key may exist
+      # for a given fingerprint
+      terraform destroy -target=hcloud_server.machine-controller-test -force
       if [[ $? == 0 ]]; then break; fi
       echo "Sleeping for $try seconds"
       sleep ${try}s
     done
 }
 trap cleanup EXIT
 
-export BUILD_ID="${BUILD_ID}"
-
 # Install dependencies
 echo "Installing dependencies."
 apt update && apt install -y jq rsync unzip &&
-curl --retry 5  -LO https://storage.googleapis.com/kubernetes-release/release/v1.10.0/bin/linux/amd64/kubectl &&
+curl --retry 5  -LO \
+  https://storage.googleapis.com/kubernetes-release/release/v1.12.4/bin/linux/amd64/kubectl &&
 chmod +x kubectl &&
 mv kubectl /usr/local/bin
 
 # Generate ssh keypair
-echo "Generating ssh keypairs."
-ssh-keygen -f $HOME/.ssh/id_rsa -P ''
+echo "Set permissions for ssh key"
+chmod 0700 $HOME/.ssh
+
+# Initialize terraform
+echo "Initalizing terraform"
+cd test/tools/integration
+make terraform
+cp provider.tf{.disabled,}
+terraform init --input=false --backend-config=key=$BUILD_ID
+export TF_VAR_hcloud_token="${HZ_E2E_TOKEN}"
+export TF_VAR_hcloud_sshkey_content="$(cat ~/.ssh/id_rsa.pub)"
+export TF_VAR_hcloud_test_server_name="machine-controller-test-${BUILD_ID}"
 
 for try in {1..20}; do
+  set +e
   # Create environment at cloud provider
   echo "Creating environment at cloud provider."
-  make -C test/tools/integration apply
+  terraform import hcloud_ssh_key.default 265119
+  terraform apply -auto-approve
   if [[ $? == 0 ]]; then break; fi
   echo "Sleeping for $try seconds"
   sleep ${try}s
 done
 
+set -e
+cd -
+
 # Build binaries
 echo "Building machine-controller and webhook"
 make machine-controller webhook

test/tools/integration/Makefile

@@ -1,3 +1,5 @@
+SHELL := /bin/bash
+
 BUILD_ID ?= $(USER)-local
 
 USER ?= prow
@@ -10,14 +12,20 @@ else ifeq ($(MAKECMDGOALS),destroy)
 	EXTRA_ARG = -force
 endif
 
+.PHONY: terraform
 terraform:
 	@if ! which terraform; then \
-		curl https://releases.hashicorp.com/terraform/0.11.7/terraform_0.11.7_linux_amd64.zip > /tmp/terraform.zip && \
-		unzip -n /tmp/terraform.zip terraform; \
+		curl https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \
+			--retry 5 \
+			-o	/tmp/terraform.zip && \
+		unzip -n /tmp/terraform.zip terraform && \
+		mv terraform /usr/local/bin; \
 	fi
 
 .terraform: terraform
-	terraform init >/dev/null 2>&1
+	@if ! ls .terraform &>/dev/null; then \
+		terraform init &>/dev/null; \
+	fi
 
 .PHONY: plan apply destroy
 plan apply destroy: .terraform

test/tools/integration/cleanup_machines.sh

@@ -5,7 +5,7 @@ set -x
 
 cd $(dirname $0)
 
-export ADDR=$(cat terraform.tfstate |jq -r '.modules[0].resources["hcloud_server.machine-controller-test"].primary.attributes.ipv4_address')
+export ADDR=$(terraform output -json|jq '.ip.value' -r)
 
 
 ssh_exec() { ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@$ADDR $@; }

test/tools/integration/output.tf

@@ -0,0 +1,3 @@
+output "ip" {
+  value = "${hcloud_server.machine-controller-test.ipv4_address}"
+}

test/tools/integration/provider.tf.disabled

@@ -0,0 +1,14 @@
+terraform {
+  backend "s3" {
+    bucket                      = "terraform-machine-controller"
+    endpoint                    = "http://minio.minio:9000"
+    access_key                  = "PMIC1HMXNB2R67RNPIX8"
+    secret_key                  = "NemiWx+uY79rcJ0hXrktzHk1dm9c0k85WepbuSlK"
+    region                      = "myregion"
+    skip_region_validation      = "true"
+    skip_metadata_api_check     = "true"
+    skip_requesting_account_id  = "true"
+    skip_credentials_validation = "true"
+    force_path_style            = "true"
+  }
+}

test/tools/integration/provision_master.sh

@@ -5,7 +5,7 @@ set -x
 
 cd $(dirname $0)
 
-export ADDR=$(cat terraform.tfstate |jq -r '.modules[0].resources["hcloud_server.machine-controller-test"].primary.attributes.ipv4_address')
+export ADDR=$(terraform output -json|jq '.ip.value' -r)
 
 
 ssh_exec() { ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@$ADDR $@; }

test/tools/integration/variables.tf

@@ -1,4 +1,8 @@
 variable "hcloud_token" {}
 variable "hcloud_sshkey_content" {}
-variable "hcloud_sshkey_name" {}
+
+variable "hcloud_sshkey_name" {
+  default = "machine-controller-e2e"
+}
+
 variable "hcloud_test_server_name" {}