Merge pull request #381 from xibz/instance-test

Adding AL2 pipeline.yml

Author: xibz

.buildkite/al2_cleanup.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+source .buildkite/al2env.sh
+
+sudo rm -rf $dir
+./tools/thinpool.sh remove $unique_id

.buildkite/al2_pipeline.yml

@@ -0,0 +1,51 @@
+# Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+# 	http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+steps:
+
+  - label: ":docker: Build"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
+      distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
+      hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
+    env:
+      DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
+      EXTRAGOARGS: "-race"
+    command:
+      - ./.buildkite/setup_al2.sh
+      - docker run --rm -v $PWD:/mnt debian:stretch-slim rm -rf /mnt/tools/image-builder/rootfs
+
+  - wait
+
+  - label: ":onion: al2 tests"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
+      distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
+      hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
+    env:
+      NUMBER_OF_VMS: "100"
+    command:
+      - ./.buildkite/al2_test.sh
+    timeout_in_minutes: 10
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":onion: cleanup"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
+      distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
+      hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
+    command:
+      - ./.buildkite/al2_cleanup.sh
+    timeout_in_minutes: 10

.buildkite/al2_test.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -eu
+
+source ./.buildkite/al2env.sh
+
+export PATH=$bin_path:$PATH
+export FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH=$runtime_config_path
+export ENABLE_ISOLATED_TESTS=true
+export CONTAINERD_SOCKET=$dir/containerd.sock 
+export SHIM_BASE_DIR=$dir
+
+sudo -E PATH=$PATH $bin_path/firecracker-containerd --config $dir/config.toml &
+containerd_pid=$!
+sudo $bin_path/firecracker-ctr --address $dir/containerd.sock content fetch docker.io/library/alpine:3.10.1
+sudo -E PATH=$bin_path:$PATH /usr/local/bin/go test -count=1 -run TestMultipleVMs_Isolated ./... -v
+
+sudo kill $containerd_pid

.buildkite/al2env.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+shim_base=/tmp/shim-base
+unique_id=$BUILDKITE_BUILD_NUMBER
+dir=$shim_base/$unique_id
+bin_path=$dir/bin
+devmapper_path=$dir/devmapper
+state_path=$dir/state
+runtime_config_path=$dir/firecracker-runtime.json
+firecracker_bin=firecracker-v0.19.0

.buildkite/hooks/pre-exit

@@ -1,4 +1,6 @@
 #!/bin/bash
-set -eu
+sudo -E PATH=$PATH make -C "$BUILDKITE_BUILD_CHECKOUT_PATH" clean
 
-make -C "$BUILDKITE_BUILD_CHECKOUT_PATH" clean
+# clean up ephemeral files since this will cause an error to build kite due to
+# these files being owned by root
+sudo -E PATH=$PATH make -C "$BUILDKITE_BUILD_CHECKOUT_PATH/tools/image-builder" distclean

.buildkite/setup_al2.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -eux
+
+source ./.buildkite/al2env.sh
+
+mkdir -p $dir
+mkdir -p $dir/rootfs
+mkdir -p $bin_path
+mkdir -p $devmapper_path
+mkdir -p $state_path
+
+./tools/thinpool.sh reset $unique_id
+
+export INSTALLROOT=$dir
+export FIRECRACKER_CONTAINERD_RUNTIME_DIR=$dir
+make
+sudo -E INSTALLROOT=$INSTALLROOT PATH=$PATH make install
+cp /var/lib/fc-ci/vmlinux.bin $dir/default-vmlinux.bin
+make image
+sudo -E PATH=$PATH make install-default-rootfs
+
+cat << EOF > $dir/config.toml
+disabled_plugins = ["cri"]
+root = "$dir"
+state = "$state_path"
+[grpc]
+  address = "$dir/containerd.sock"
+[plugins]
+  [plugins.devmapper]
+    pool_name = "fcci--vg-$unique_id"
+    base_image_size = "10GB"
+    root_path = "$devmapper_path"
+[debug]
+  level = "debug"
+EOF
+
+cat << EOF > $runtime_config_path
+{
+	"cpu_template": "T2",
+	"debug": true,
+	"firecracker_binary_path": "/usr/local/bin/$firecracker_bin",
+	"shim_base_dir": "$dir",
+	"kernel_image_path": "$dir/default-vmlinux.bin",
+	"kernel_args": "ro console=ttyS0 noapic reboot=k panic=1 pci=off nomodules systemd.journald.forward_to_console systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init",
+	"log_level": "DEBUG",
+	"root_drive": "$dir/default-rootfs.img",
+	"jailer": {
+		"runc_binary_path": "$bin_path/runc",
+		"runc_config_path": "$dir/config.json"
+	}
+}
+EOF
+
+cp ./runtime/firecracker-runc-config.json.example $dir/config.json
+cp ./_submodules/runc/runc $bin_path/runc

Makefile

@@ -54,7 +54,7 @@ $(SUBDIRS):
 	$(MAKE) -C $@ EXTRAGOARGS=$(EXTRAGOARGS)
 
 %-in-docker:
-	docker run --rm -it \
+	docker run --rm \
 		--user $(UID):$(GID) \
 		--volume $(CURDIR):/src \
 		--volume $(GO_CACHE_VOLUME_NAME):/go \
@@ -325,7 +325,7 @@ tools/runc-builder-stamp: tools/docker/Dockerfile.runc-builder
 	touch $@
 
 $(RUNC_BIN): $(RUNC_DIR)/VERSION tools/runc-builder-stamp
-	docker run --rm -it --user $(UID) \
+	docker run --rm --user $(UID) \
 		--volume $(CURDIR)/$(RUNC_DIR):/gopath/src/github.com/opencontainers/runc \
 		--volume $(CURDIR)/deps:/target \
 		-e HOME=/tmp \

config/config.go

@@ -25,14 +25,17 @@ import (
 )
 
 const (
-	configPathEnvName  = "FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH"
+	// ConfigPathEnvName is the name of the environment variable used to
+	// overwrite the default runtime config path
+	ConfigPathEnvName  = "FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH"
 	defaultConfigPath  = "/etc/containerd/firecracker-runtime.json"
 	defaultKernelArgs  = "console=ttyS0 noapic reboot=k panic=1 pci=off nomodules rw"
 	defaultFilesPath   = "/var/lib/firecracker-containerd/runtime/"
 	defaultKernelPath  = defaultFilesPath + "default-vmlinux.bin"
 	defaultRootfsPath  = defaultFilesPath + "default-rootfs.img"
 	defaultCPUTemplate = models.CPUTemplateT2
 	defaultShimBaseDir = "/var/lib/firecracker-containerd/shim-base"
+	runcConfigPath     = "/etc/containerd/firecracker-runc-config.json"
 )
 
 // Config represents runtime configuration parameters
@@ -60,12 +63,13 @@ type Config struct {
 // TODO: Add netns field
 type JailerConfig struct {
 	RuncBinaryPath string `json:"runc_binary_path"`
+	RuncConfigPath string `json:"runc_config_path"`
 }
 
 // LoadConfig loads configuration from JSON file at 'path'
 func LoadConfig(path string) (*Config, error) {
 	if path == "" {
-		path = os.Getenv(configPathEnvName)
+		path = os.Getenv(ConfigPathEnvName)
 	}
 
 	if path == "" {
@@ -83,6 +87,9 @@ func LoadConfig(path string) (*Config, error) {
 		RootDrive:       defaultRootfsPath,
 		CPUTemplate:     string(defaultCPUTemplate),
 		ShimBaseDir:     defaultShimBaseDir,
+		JailerConfig: JailerConfig{
+			RuncConfigPath: runcConfigPath,
+		},
 	}
 
 	if err := json.Unmarshal(data, cfg); err != nil {

runtime/integ_test.go

@@ -29,7 +29,8 @@ import (
 )
 
 const runtimeConfigPath = "/etc/containerd/firecracker-runtime.json"
-const shimBaseDir = "/srv/firecracker_containerd_tests"
+const shimBaseDirEnvVar = "SHIM_BASE_DIR"
+const defaultShimBaseDir = "/srv/firecracker_containerd_tests"
 
 var defaultRuntimeConfig = config.Config{
 	FirecrackerBinaryPath: "/usr/local/bin/firecracker",
@@ -39,12 +40,21 @@ var defaultRuntimeConfig = config.Config{
 	CPUTemplate:           "T2",
 	LogLevel:              "Debug",
 	Debug:                 true,
-	ShimBaseDir:           shimBaseDir,
+	ShimBaseDir:           shimBaseDir(),
 	JailerConfig: config.JailerConfig{
 		RuncBinaryPath: "/usr/local/bin/runc",
+		RuncConfigPath: "/etc/containerd/firecracker-runc-config.json",
 	},
 }
 
+func shimBaseDir() string {
+	if v := os.Getenv(shimBaseDirEnvVar); v != "" {
+		return v
+	}
+
+	return defaultShimBaseDir
+}
+
 func defaultSnapshotterName() string {
 	if name := os.Getenv("FICD_SNAPSHOTTER"); name != "" {
 		return name

runtime/jailer.go

@@ -34,10 +34,6 @@ const (
 	rootfsFolder          = "rootfs"
 )
 
-var (
-	runcConfigPath = "/etc/containerd/firecracker-runc-config.json"
-)
-
 // jailer will allow modification and provide options to the the Firecracker VM
 // to allow for jailing. In addition, this will allow for given files to be exposed
 // to the jailed filesystem.
@@ -99,12 +95,13 @@ func newJailer(
 
 	l := logger.WithField("jailer", "runc")
 	config := runcJailerConfig{
-		OCIBundlePath: ociBundlePath,
-		RuncBinPath:   service.config.JailerConfig.RuncBinaryPath,
-		UID:           request.JailerConfig.UID,
-		GID:           request.JailerConfig.GID,
-		CPUs:          request.JailerConfig.CPUs,
-		Mems:          request.JailerConfig.Mems,
+		OCIBundlePath:  ociBundlePath,
+		RuncBinPath:    service.config.JailerConfig.RuncBinaryPath,
+		RuncConfigPath: service.config.JailerConfig.RuncConfigPath,
+		UID:            request.JailerConfig.UID,
+		GID:            request.JailerConfig.GID,
+		CPUs:           request.JailerConfig.CPUs,
+		Mems:           request.JailerConfig.Mems,
 	}
 	return newRuncJailer(ctx, l, service.vmID, config)
 }