Update tests to be compatible with AL2

This adds the necessary cleanup functions to TestMultipleVMs as well as
adding a default runc config path to our tests.

Signed-off-by: xibz <[email protected]>

Author: xibz

.buildkite/al2_cleanup.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+source .buildkite/al2env.sh
+
+sudo rm -rf $dir
+./tools/thinpool.sh remove $unique_id

.buildkite/al2_pipeline.yml

@@ -24,67 +24,28 @@ steps:
     command:
       - ./.buildkite/setup_al2.sh
       - docker run --rm -v $PWD:/mnt debian:stretch-slim rm -rf /mnt/tools/image-builder/rootfs
-      - sudo install -d -o root -g buildkite-agent -m 775 "/local/artifacts/$BUILDKITE_BUILD_NUMBER"
-      - cp tools/image-builder/rootfs.img "/local/artifacts/$BUILDKITE_BUILD_NUMBER/"
 
   - wait
 
-    # Global concurrency has been set to one in the website configuration.
-    # With that we can now clean up loop devices before starting jobs to ensure
-    # we don't hit the loop device limit
-  - label: ":lint-roller: loop device cleanup"
-    agents:
-      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
-      distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
-      hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
-    command: 'sudo losetup -D'
-    concurrency: 1
-    concurrency_group: 'loop-device test'
-
-  - wait
-
-  # Git history validation happens after the 'wait' step so it happens
-  # in parallel with the subsequent tests and does not prevent them
-  # from running in the event of a validation failure.
-  - label: 'git log validation'
-    command: './.buildkite/logcheck.sh'
-    # This should run in the same queue, but we don't care whether it runs on
-    # the same host.
-    agents:
-      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
-
-  - label: ":protractor: verify proto"
+  - label: ":onion: al2 tests"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
       distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
       hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
     env:
-      DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
+      NUMBER_OF_VMS: "100"
     command:
-      - make proto
-      - test -z "$(git status --porcelain)" && exit 0 || git status && echo -e '\nGenerated protobuf code differs from committed, please run "make proto" and commit the updated files.\n' && exit 1
-    timeout_in_minutes: 1
+      - ./.buildkite/al2_test.sh
+    timeout_in_minutes: 10
 
-  - label: ":golang: go mod tidy"
-    agents:
-      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
-      distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
-      hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
-    env:
-      DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
-    command:
-      - sudo make tidy-in-docker
-      - test -z "$(git status --porcelain)" && exit 0 || git status && echo -e '\ngo.mod and/or go.sum differ from committed, please run "go mod tidy" and commit the updated files.\n' && exit 1
-    timeout_in_minutes: 1
-    
-  - label: ":onion: al2 tests"
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":onion: cleanup"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
       distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
       hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
-    env:
-      NUMBER_OF_VMS: "5"
     command:
-      - ./.buildkite/al2_test.sh
+      - ./.buildkite/al2_cleanup.sh
     timeout_in_minutes: 10
-

.buildkite/al2_test.sh

@@ -1,15 +1,17 @@
 #!/bin/bash
+set -eu
 
 source ./.buildkite/al2env.sh
 
 export PATH=$bin_path:$PATH
+export FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH=$runtime_config_path
+export ENABLE_ISOLATED_TESTS=true
+export CONTAINERD_SOCKET=$dir/containerd.sock 
+export SHIM_BASE_DIR=$dir
 
-sudo -E "FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH=$runtime_config_path" $bin_path/firecracker-containerd --config $dir/config.toml &
+sudo -E PATH=$PATH $bin_path/firecracker-containerd --config $dir/config.toml &
 containerd_pid=$!
 sudo $bin_path/firecracker-ctr --address $dir/containerd.sock content fetch docker.io/library/alpine:3.10.1
-sudo -E env "PATH=$PATH" /usr/local/bin/go test -run TestMultipleVMs_Isolated ./...
+sudo -E PATH=$bin_path:$PATH /usr/local/bin/go test -count=1 -run TestMultipleVMs_Isolated ./... -v
 
-# cleanup
-sudo kill -9 $containerd_pid
-sudo rm -rf $dir
-./tools/thinpool.sh remove $uuid
+sudo kill $containerd_pid

.buildkite/al2env.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 shim_base=/tmp/shim-base
-uuid=$BUILDKITE_BUILD_NUMBER
-dir=$shim_base/$uuid
+unique_id=$BUILDKITE_BUILD_NUMBER
+dir=$shim_base/$unique_id
 bin_path=$dir/bin
 devmapper_path=$dir/devmapper
 state_path=$dir/state

.buildkite/hooks/pre-exit

@@ -1,8 +1,6 @@
 #!/bin/bash
-set -eu
-
-sudo make -C "$BUILDKITE_BUILD_CHECKOUT_PATH" clean
+sudo -E PATH=$PATH make -C "$BUILDKITE_BUILD_CHECKOUT_PATH" clean
 
 # clean up ephemeral files since this will cause an error to build kite due to
 # these files being owned by root
-sudo make -C "$BUILDKITE_BUILD_CHECKOUT_PATH/tools/image-builder" distclean
+sudo -E PATH=$PATH make -C "$BUILDKITE_BUILD_CHECKOUT_PATH/tools/image-builder" distclean

.buildkite/setup_al2.sh

@@ -1,31 +1,25 @@
 #!/bin/bash
 
-source ./.buildkite/al2env.sh
+set -eux
 
-echo "Creating necessary directories"
+source ./.buildkite/al2env.sh
 
 mkdir -p $dir
+mkdir -p $dir/rootfs
 mkdir -p $bin_path
 mkdir -p $devmapper_path
 mkdir -p $state_path
 
-echo "Resetting thinpool $uuid"
-./tools/thinpool.sh reset $uuid
+./tools/thinpool.sh reset $unique_id
 
 export INSTALLROOT=$dir
 export FIRECRACKER_CONTAINERD_RUNTIME_DIR=$dir
-echo "Running make"
-sudo make
-echo "Running make install"
-sudo -E "INSTALLROOT=$INSTALLROOT" make install
-echo "Running make install-default-vmlinux"
-sudo make install-default-vmlinux
-echo "Running make image"
-sudo make image
-echo "Running make install-default-rootfs"
-sudo make install-default-rootfs
+make
+sudo -E INSTALLROOT=$INSTALLROOT PATH=$PATH make install
+cp /var/lib/fc-ci/vmlinux.bin $dir/default-vmlinux.bin
+make image
+sudo -E PATH=$PATH make install-default-rootfs
 
-echo "Creating $dir/config.toml"
 cat << EOF > $dir/config.toml
 disabled_plugins = ["cri"]
 root = "$dir"
@@ -34,14 +28,13 @@ state = "$state_path"
   address = "$dir/containerd.sock"
 [plugins]
   [plugins.devmapper]
-    pool_name = "fcci--vg-$uuid"
+    pool_name = "fcci--vg-$unique_id"
     base_image_size = "10GB"
     root_path = "$devmapper_path"
 [debug]
   level = "debug"
 EOF
 
-echo "Creating $runtime_config_path"
 cat << EOF > $runtime_config_path
 {
 	"cpu_template": "T2",
@@ -59,7 +52,5 @@ cat << EOF > $runtime_config_path
 }
 EOF
 
-echo "Copying firecracker-runc-config.json"
 cp ./runtime/firecracker-runc-config.json.example $dir/config.json
-echo "Copying runc"
 cp ./_submodules/runc/runc $bin_path/runc

runtime/integ_test.go

@@ -29,7 +29,8 @@ import (
 )
 
 const runtimeConfigPath = "/etc/containerd/firecracker-runtime.json"
-const shimBaseDir = "/srv/firecracker_containerd_tests"
+const shimBaseDirEnvVar = "SHIM_BASE_DIR"
+const defaultShimBaseDir = "/srv/firecracker_containerd_tests"
 
 var defaultRuntimeConfig = config.Config{
 	FirecrackerBinaryPath: "/usr/local/bin/firecracker",
@@ -39,12 +40,21 @@ var defaultRuntimeConfig = config.Config{
 	CPUTemplate:           "T2",
 	LogLevel:              "Debug",
 	Debug:                 true,
-	ShimBaseDir:           shimBaseDir,
+	ShimBaseDir:           shimBaseDir(),
 	JailerConfig: config.JailerConfig{
 		RuncBinaryPath: "/usr/local/bin/runc",
+		RuncConfigPath: "/etc/containerd/firecracker-runc-config.json",
 	},
 }
 
+func shimBaseDir() string {
+	if v := os.Getenv(shimBaseDirEnvVar); v != "" {
+		return v
+	}
+
+	return defaultShimBaseDir
+}
+
 func defaultSnapshotterName() string {
 	if name := os.Getenv("FICD_SNAPSHOTTER"); name != "" {
 		return name

runtime/service_integ_test.go

@@ -17,6 +17,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -60,23 +61,30 @@ import (
 const (
 	defaultNamespace = namespaces.Default
 
-	containerdSockPath = "/run/containerd/containerd.sock"
-
 	firecrackerRuntime     = "aws.firecracker"
 	shimProcessName        = "containerd-shim-aws-firecracker"
 	firecrackerProcessName = "firecracker"
 
-	defaultVMRootfsPath = "/var/lib/firecracker-containerd/runtime/default-rootfs.img"
 	defaultVMNetDevName = "eth0"
 	numberOfVmsEnvName  = "NUMBER_OF_VMS"
 	defaultNumberOfVms  = 5
+
+	containerdSockPathEnvVar = "CONTAINERD_SOCKET"
 )
 
 var (
 	findShim        = findProcWithName(shimProcessName)
 	findFirecracker = findProcWithName(firecrackerProcessName)
+
+	containerdSockPath = "/run/containerd/containerd.sock"
 )
 
+func init() {
+	if v := os.Getenv(containerdSockPathEnvVar); v != "" {
+		containerdSockPath = v
+	}
+}
+
 // Images are presumed by the isolated tests to have already been pulled
 // into the content store. This will just unpack the layers into an
 // image with the provided snapshotter.
@@ -211,6 +219,18 @@ func vmIDtoMacAddr(vmID uint) string {
 	return strings.Join(addrParts, ":")
 }
 
+func deleteTapDevice(ctx context.Context, tapName string) error {
+	if err := exec.CommandContext(ctx, "ip", "link", "delete", tapName).Run(); err != nil {
+		return err
+	}
+
+	if err := exec.CommandContext(ctx, "ip", "tuntap", "del", tapName, "mode", "tap").Run(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func createTapDevice(ctx context.Context, tapName string) error {
 	err := exec.CommandContext(ctx, "ip", "tuntap", "add", tapName, "mode", "tap").Run()
 	if err != nil {
@@ -225,7 +245,39 @@ func createTapDevice(ctx context.Context, tapName string) error {
 	return nil
 }
 
+type testMultipleVMsRunner struct {
+	containers []containerd.Container
+}
+
+func (runner *testMultipleVMsRunner) ImportPath() string { return "" }
+func (runner *testMultipleVMsRunner) MatchString(pat, str string) (bool, error) {
+	return pat == str, nil
+}
+func (runner *testMultipleVMsRunner) StartCPUProfile(w io.Writer) error                   { return nil }
+func (runner *testMultipleVMsRunner) StopCPUProfile()                                     {}
+func (runner *testMultipleVMsRunner) StartTestLog(w io.Writer)                            {}
+func (runner *testMultipleVMsRunner) StopTestLog() error                                  { return nil }
+func (runner *testMultipleVMsRunner) WriteProfileTo(str string, w io.Writer, n int) error { return nil }
+
 func TestMultipleVMs_Isolated(t *testing.T) {
+	runner := &testMultipleVMsRunner{}
+	tests := []testing.InternalTest{
+		{
+			Name: "TestMultipleVMs_Isolated",
+			F:    runner.TestMultipleVMs,
+		},
+	}
+	m := testing.MainStart(runner, tests, nil, nil)
+	code := m.Run()
+	for _, container := range runner.containers {
+		err := container.Delete(context.Background())
+		t.Log(err)
+	}
+
+	os.Exit(code)
+}
+
+func (runner *testMultipleVMsRunner) TestMultipleVMs(t *testing.T) {
 	prepareIntegTest(t)
 
 	netns, err := ns.GetCurrentNS()
@@ -270,6 +322,9 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 	pluginClient, err := ttrpcutil.NewClient(containerdSockPath + ".ttrpc")
 	require.NoError(t, err, "failed to create ttrpc client")
 
+	cfg, err := config.LoadConfig("")
+	require.NoError(t, err, "failed to load config")
+
 	// This test spawns separate VMs in parallel and ensures containers are spawned within each expected VM. It asserts each
 	// container ends up in the right VM by assigning each VM a network device with a unique mac address and having each container
 	// print the mac address it sees inside its VM.
@@ -283,10 +338,11 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 
 			tapName := fmt.Sprintf("tap%d", vmID)
 			err := createTapDevice(ctx, tapName)
+			defer deleteTapDevice(ctx, tapName)
 
 			require.NoError(t, err, "failed to create tap device for vm %d", vmID)
 
-			rootfsPath := defaultVMRootfsPath
+			rootfsPath := cfg.RootDrive
 
 			vmIDStr := strconv.Itoa(vmID)
 			fcClient := fccontrol.NewFirecrackerClient(pluginClient.Client())
@@ -321,7 +377,7 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 				containerWg.Add(1)
 				go func(containerID int) {
 					defer containerWg.Done()
-					testMultipleExecs(
+					runner.testMultipleExecs(
 						ctx,
 						t,
 						vmID,
@@ -366,7 +422,7 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 	vmWg.Wait()
 }
 
-func testMultipleExecs(
+func (runner *testMultipleVMsRunner) testMultipleExecs(
 	ctx context.Context,
 	t *testing.T,
 	vmID int,
@@ -399,6 +455,7 @@ func testMultipleExecs(
 		),
 	)
 	require.NoError(t, err, "failed to create container %s", containerName)
+	runner.containers = append(runner.containers, newContainer)
 
 	var taskStdout bytes.Buffer
 	var taskStderr bytes.Buffer
@@ -438,7 +495,7 @@ func testMultipleExecs(
 	if jailerConfig != nil {
 		jailer := &runcJailer{
 			Config: runcJailerConfig{
-				OCIBundlePath: filepath.Join(shimBaseDir, vmIDStr),
+				OCIBundlePath: filepath.Join(shimBaseDir(), vmIDStr),
 			},
 			vmID: vmIDStr,
 		}