tests: initial implementation of smoke tests

A simple implementation of smoke tests based on Vagrant.
It initializes a simple K8s cluster with 2 nodes, deploys an nginx
pod, a service for the deployment, and checks if it's running.

Fixes #56

Author: Dongsu Park

Gopkg.lock

@@ -0,0 +1,16 @@
+apiVersion: apps/v1beta1 # for versions before 1.6.0 use extensions/v1beta1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80

tests/fixtures/nginx-deployment.yaml

@@ -0,0 +1,222 @@
+/*
+Copyright 2017 Kinvolk GmbH
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +build integration
+
+package tests
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/kinvolk/kube-spawn/pkg/utils"
+)
+
+const (
+	k8sStableVersion    string = "1.7.5"
+	defaultKubeSpawnDir string = "/var/lib/kube-spawn"
+	deploymentName      string = "nginx-deployment"
+)
+
+var (
+	numNodes   int = 2
+	numDeploys int = 2
+
+	kubeSpawnDir     string = defaultKubeSpawnDir
+	kubeSpawnK8sPath string = filepath.Join(kubeSpawnDir, "k8s")
+	kubeSpawnPath    string
+	kubeCtlPath      string
+
+	machineCtlPath string
+)
+
+func checkRequirements(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Fatal("smoke test requires root privileges")
+	}
+}
+
+func initPath(t *testing.T) {
+	var err error
+
+	// go one dir upper, from "tests" to the top source directory
+	if err := os.Chdir(".."); err != nil {
+		t.Fatal(err)
+	}
+
+	kubeSpawnPath = "./kube-spawn"
+	if err := utils.CheckValidFile(kubeSpawnPath); err != nil {
+		if kubeSpawnPath, err = exec.LookPath("kube-spawn"); err != nil {
+			// fall back to an ordinary abspath to kube-spawn
+			kubeSpawnPath = "/usr/bin/kube-spawn"
+		}
+	}
+
+	_ = os.MkdirAll(kubeSpawnK8sPath, os.FileMode(0755))
+
+	kubeCtlPath = filepath.Join(kubeSpawnK8sPath, "kubectl")
+	if err := utils.CheckValidFile(kubeCtlPath); err != nil {
+		if kubeCtlPath, err = exec.LookPath(kubeCtlPath); err != nil {
+			// fall back to an ordinary abspath to kubectl
+			kubeCtlPath = "/usr/bin/kubectl"
+		}
+	}
+
+	machineCtlPath, err = exec.LookPath("machinectl")
+	if err != nil {
+		// fall back to an ordinary abspath to machinectl
+		machineCtlPath = "/usr/bin/machinectl"
+	}
+}
+
+func initNode(t *testing.T) {
+	// If no coreos image exists, just download it
+	if _, _, err := runCommand(fmt.Sprintf("%s show-image coreos", machineCtlPath)); err != nil {
+		if stdout, stderr, err := runCommand(fmt.Sprintf("%s pull-raw --verify=no %s %s",
+			machineCtlPath,
+			"https://alpha.release.core-os.net/amd64-usr/current/coreos_developer_container.bin.bz2",
+			"coreos",
+		)); err != nil {
+			t.Fatalf("error running machinectl pull-raw: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
+		}
+	}
+}
+
+func getRunningNodes() ([]string, error) {
+	var nodeNames []string
+
+	stdout, stderr, err := runCommand(fmt.Sprintf("%s list --no-legend", machineCtlPath))
+	if err != nil {
+		return nil, fmt.Errorf("error running machinectl list: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
+	}
+
+	s := bufio.NewScanner(strings.NewReader(strings.TrimSpace(stdout)))
+	for s.Scan() {
+		line := strings.Fields(s.Text())
+		if len(line) <= 2 {
+			continue
+		}
+
+		// an example line:
+		//  kubespawn0 container systemd-nspawn coreos 1478.0.0 10.22.0.130...
+		nodeName := strings.TrimSpace(line[0])
+		if !strings.HasPrefix(nodeName, "kubespawn") {
+			continue
+		}
+
+		nodeNames = append(nodeNames, nodeName)
+	}
+
+	return nodeNames, nil
+}
+
+func checkK8sNodes(t *testing.T) {
+	nodeStates, err := waitForNReadyNodes(numNodes)
+	if err != nil {
+		t.Fatalf("error waiting on %d ready nodes, result %v: %v\n", numNodes, nodeStates, err)
+	}
+}
+
+func testSetupK8sStable(t *testing.T) {
+	if stdout, stderr, err := runCommand(fmt.Sprintf("%s --kubernetes-version=%s setup --nodes=%d",
+		kubeSpawnPath, k8sStableVersion, numNodes),
+	); err != nil {
+		t.Fatalf("error running kube-spawn setup: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
+	}
+
+	nodes, err := getRunningNodes()
+	if err != nil {
+		t.Fatalf("error getting list of running nodes: %v\n", err)
+	}
+	if len(nodes) != numNodes {
+		t.Fatalf("got %d nodes, expected %d nodes.\n", len(nodes), numNodes)
+	}
+}
+
+func testInitK8sStable(t *testing.T) {
+	if stdout, stderr, err := runCommand(fmt.Sprintf("%s --kubernetes-version=%s init",
+		kubeSpawnPath, k8sStableVersion),
+	); err != nil {
+		t.Fatalf("error running kube-spawn init: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
+	}
+
+	// set env variable KUBECONFIG to /var/lib/kube-spawn/default/kubeconfig
+	if err := os.Setenv("KUBECONFIG", utils.GetValidKubeConfig()); err != nil {
+		t.Fatalf("error running setenv: %v\n", err)
+	}
+	checkK8sNodes(t)
+}
+
+func testCreateDeploy(t *testing.T) {
+	if stdout, stderr, err := runCommand(fmt.Sprintf("%s create -f %s",
+		kubeCtlPath, "./tests/fixtures/nginx-deployment.yaml"),
+	); err != nil {
+		t.Fatalf("error creating deployment: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
+	}
+
+	deploys, err := waitForNDeployments(numDeploys)
+	if err != nil {
+		t.Fatalf("error waiting on %d deployments, result %v: %v\n", numDeploys, deploys, err)
+	}
+}
+
+func testServices(t *testing.T) {
+	stdout, stderr, err := runCommand(fmt.Sprintf("%s get services --no-headers=true", kubeCtlPath))
+	if err != nil {
+		t.Fatalf("error getting services: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
+	}
+
+	outStr := strings.TrimSpace(string(stdout))
+	scanner := bufio.NewScanner(strings.NewReader(outStr))
+	var svcNames []string
+	for scanner.Scan() {
+		if len(strings.TrimSpace(scanner.Text())) == 0 {
+			continue
+		}
+		svcNames = append(svcNames, strings.Fields(scanner.Text())[0])
+	}
+
+	if len(svcNames) == 0 {
+		t.Fatalf("cannot find any services\n")
+	}
+
+	if !existInSlice("kubernetes", svcNames) {
+		t.Fatalf("cannot find a service\n")
+	}
+}
+
+func testDeleteDeploy(t *testing.T) {
+	stdout, stderr, err := runCommand(fmt.Sprintf("%s delete deployment %s", kubeCtlPath, deploymentName))
+	if err != nil {
+		t.Fatalf("error deleting deployments %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
+	}
+}
+
+func TestMainK8sStable(t *testing.T) {
+	checkRequirements(t)
+	initPath(t)
+	initNode(t)
+
+	testSetupK8sStable(t)
+	testInitK8sStable(t)
+	testCreateDeploy(t)
+	testServices(t)
+}

tests/init_test.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2017 Kinvolk GmbH
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +build integration
+
+package tests
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"log"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func runCommand(command string) (string, string, error) {
+	log.Printf(command)
+
+	var stdoutBytes, stderrBytes bytes.Buffer
+	args := strings.Split(command, " ")
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Stdout = &stdoutBytes
+	cmd.Stderr = &stderrBytes
+	err := cmd.Run()
+	return stdoutBytes.String(), stderrBytes.String(), err
+}
+
+func existInSlice(key string, inSlice []string) bool {
+	for _, n := range inSlice {
+		if n == key {
+			return true
+		}
+	}
+	return false
+}
+
+func waitForNReadyNodes(expectedNodes int) (map[string]string, error) {
+	nodeStates := make(map[string]string, 0)
+	timeout := 180 * time.Second
+	alarm := time.After(timeout)
+
+	getReadyNodes := func(nodeStates map[string]string) int {
+		nReadyNodes := 0
+
+		for _, s := range nodeStates {
+			if s != "Ready" {
+				continue
+			}
+			nReadyNodes += 1
+		}
+
+		return nReadyNodes
+	}
+
+	ticker := time.Tick(500 * time.Millisecond)
+loop:
+	for {
+		select {
+		case <-alarm:
+			return nodeStates, fmt.Errorf("failed to find %d ready nodes within %v", expectedNodes, timeout)
+		case <-ticker:
+			stdout, _, err := runCommand(fmt.Sprintf("%s get nodes --no-headers=true", kubeCtlPath))
+			if err != nil {
+				continue
+			}
+
+			outStr := strings.TrimSpace(stdout)
+			scanner := bufio.NewScanner(strings.NewReader(outStr))
+			nodeStates := make(map[string]string, 0)
+			for scanner.Scan() {
+				if len(strings.TrimSpace(scanner.Text())) == 0 {
+					continue
+				}
+				name := strings.Fields(scanner.Text())[0]
+				state := strings.Fields(scanner.Text())[1]
+				nodeStates[name] = state
+			}
+
+			if getReadyNodes(nodeStates) != expectedNodes {
+				continue
+			}
+
+			break loop
+		}
+	}
+
+	return nodeStates, nil
+}
+
+func waitForNDeployments(expectedDeps int) (map[string]string, error) {
+	deploys := make(map[string]string, 0)
+	timeout := 10 * time.Second
+	alarm := time.After(timeout)
+
+	ticker := time.Tick(250 * time.Millisecond)
+loop:
+	for {
+		select {
+		case <-alarm:
+			return deploys, fmt.Errorf("failed to find %d deployments within %v", expectedDeps, timeout)
+		case <-ticker:
+			stdout, _, err := runCommand(fmt.Sprintf("%s get deployments --no-headers=true", kubeCtlPath))
+			if err != nil {
+				continue
+			}
+
+			outStr := strings.TrimSpace(stdout)
+			scanner := bufio.NewScanner(strings.NewReader(outStr))
+			deploys := make(map[string]string, 0)
+			for scanner.Scan() {
+				if len(strings.TrimSpace(scanner.Text())) == 0 {
+					continue
+				}
+				// example of output:
+				//
+				// NAME               DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
+				// nginx-deployment   2         2         2            2           3m
+				depName := strings.Fields(scanner.Text())[0]
+				depAvailable := strings.Fields(scanner.Text())[4]
+				deploys[depName] = depAvailable
+			}
+
+			actualNodes, _ := strconv.Atoi(deploys[deploymentName])
+			if actualNodes != expectedDeps {
+				continue
+			}
+
+			break loop
+		}
+	}
+
+	return deploys, nil
+}