Merge pull request #629 from austinvazquez/remote-snapshotter-integ-test

add: integration test for remote snapshotter

Author: austinvazquez

.buildkite/pipeline.yml

@@ -73,6 +73,20 @@ steps:
       - make test-in-docker
     timeout_in_minutes: 10
 
+  - label: ":running: snapshotter isolated tests"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
+      distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
+      hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
+    env:
+      DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
+      NUMBER_OF_VMS: 10
+      EXTRAGOARGS: "-v -count=1 -race"
+    artifact_paths:
+      - "snapshotter/logs/*"
+    command:
+      - make -C snapshotter integ-test
+
   - label: ":running: runtime isolated tests"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"

.gitignore

@@ -1,5 +1,6 @@
 .idea/
 bin/
+tmp/
 runtime/logs
 *stamp
 default-vmlinux.bin

Makefile

@@ -187,7 +187,7 @@ $(INTEG_TEST_SUBDIRS): test-images
 
 test-images: test-images-stamp
 
-test-images-stamp: | image firecracker-containerd-test-image
+test-images-stamp: | image image-stargz firecracker-containerd-test-image
 	touch $@
 
 firecracker-containerd-test-image: all-in-docker firecracker runc test-cni-bins cni-bins default-vmlinux kernel

internal/integtest/runtime.go

@@ -15,22 +15,31 @@ package integtest
 
 import (
 	"os"
+	"strconv"
 )
 
 const (
 	// FirecrackerRuntime is the Firecracker-containerd runtime
 	FirecrackerRuntime = "aws.firecracker"
 
 	containerdSockPathEnvVar = "CONTAINERD_SOCKET"
+	numberOfVmsEnvVar        = "NUMBER_OF_VMS"
 )
 
 var (
 	// ContainerdSockPath is the default Firecracker-containerd socket path
 	ContainerdSockPath = "/run/firecracker-containerd/containerd.sock"
+
+	// NumberOfVms is the number of VMs used in integration testing set
+	// by either environment variable read or defaults to 5 VMs.
+	NumberOfVms = 5
 )
 
 func init() {
 	if v := os.Getenv(containerdSockPathEnvVar); v != "" {
 		ContainerdSockPath = v
 	}
+	if v := os.Getenv(numberOfVmsEnvVar); v != "" {
+		NumberOfVms, _ = strconv.Atoi(v)
+	}
 }

snapshotter/.gitignore

@@ -1,2 +1,3 @@
 demux-snapshotter
 http-address-resolver
+logs/

snapshotter/Makefile

@@ -12,33 +12,72 @@
 # permissions and limitations under the License.
 
 EXTRAGOARGS?=
+NUMBER_OF_VMS?=
 
 SOURCES := $(shell find . -name '*.go')
 GOMOD := $(shell go env GOMOD)
 GOSUM := $(GOMOD:.mod=.sum)
+DOCKER_IMAGE_TAG?=latest
+GO_CACHE_VOLUME_NAME?=gocache
+FIRECRACKER_CONTAINERD_TEST_IMAGE?=localhost/firecracker-containerd-test
 
 REVISION := $(shell git rev-parse HEAD)
 
-all: demux-snapshotter
+INTEG_TEST_SUFFIX := _Isolated
+INTEG_TESTNAMES=$(shell docker run --rm \
+	--network=none \
+	--volume $(CURDIR)/..:/src \
+	--volume $(GO_CACHE_VOLUME_NAME):/go \
+	--entrypoint=/bin/bash \
+	--workdir=/src/snapshotter \
+	$(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \
+	-c "go test -list . | sed '$$d' | grep $(INTEG_TEST_SUFFIX)")
+
+all: demux-snapshotter http-address-resolver
 
 demux-snapshotter: $(SOURCES) $(GOMOD) $(GOSUM)
 	go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@
 
 http-address-resolver: $(SOURCES) $(GOMOD) $(GOSUM)
 	go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@ internal/http_address_resolver.go
 
+install: demux-snapshotter http-address-resolver
+	install -D -o root -g root -m755 -t $(INSTALLROOT)/bin $^
+
 test:
 	go test ./... $(EXTRAGOARGS)
 
 integ-test:
+	$(MAKE) $(addprefix integ-test-,$(INTEG_TESTNAMES))
+
+integ-test-%: logs
+	docker run --rm -it \
+		--privileged \
+		--ipc=host \
+		--network=bridge \
+		--volume /dev:/dev \
+		--volume /run/udev/control:/run/udev/control \
+		--volume $(CURDIR)/logs:/var/log/firecracker-containerd-test \
+		--volume $(CURDIR)/..:/src \
+		--volume $(GO_CACHE_VOLUME_NAME):/go \
+		--env ENABLE_ISOLATED_TESTS=1 \
+		--env GOPROXY=direct \
+		--env GOSUMDB=off \
+		--env GO111MODULE=on \
+		--env NUMBER_OF_VMS=$(NUMBER_OF_VMS) \
+		--workdir="/src/snapshotter" \
+		--init \
+		$(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \
+		"go test $(EXTRAGOARGS) -run '^$(subst integ-test-,,$@)$$'"
+
+logs:
+	mkdir logs
 
 clean:
 	- rm -f demux-snapshotter
-	- rm -f http-resolver
+	- rm -f http-address-resolver
 
 distclean: clean
+	- rm -rf logs
 
-# Install is a noop here, for now.
-install:
-
-.PHONY: all install test clean distclean
+.PHONY: all clean distclean install test integ-test

snapshotter/internal/integtest/stargz/fs/config/config.go

@@ -0,0 +1,52 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// Copied from https://github.com/containerd/stargz-snapshotter/blob/v0.11.4/fs/config/config.go
+//
+// Taking Stargz as a code dependency required dropping support for Go 1.16.
+// Used to add Stargz annotations to pull requests.
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright 2019 The Go Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style
+   license that can be found in the NOTICE.md file.
+*/
+
+package config
+
+const (
+	// TargetSkipVerifyLabel is a snapshot label key that indicates to skip content
+	// verification for the layer.
+	TargetSkipVerifyLabel = "containerd.io/snapshot/remote/stargz.skipverify"
+
+	// TargetPrefetchSizeLabel is a snapshot label key that indicates size to prefetch
+	// the layer. If the layer is eStargz and contains prefetch landmarks, these config
+	// will be respeced.
+	TargetPrefetchSizeLabel = "containerd.io/snapshot/remote/stargz.prefetch"
+)

snapshotter/internal/integtest/stargz/fs/source/source.go

@@ -0,0 +1,128 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// Copied from https://github.com/containerd/stargz-snapshotter/blob/v0.11.4/fs/source/source.go
+//
+// Taking Stargz as a code dependency required dropping support for Go 1.16.
+// Used to add Stargz annotations to pull requests.
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package source
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/labels"
+	"github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/integtest/stargz/fs/config"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+const (
+	// targetRefLabel is a label which contains image reference.
+	targetRefLabel = "containerd.io/snapshot/remote/stargz.reference"
+
+	// targetDigestLabel is a label which contains layer digest.
+	targetDigestLabel = "containerd.io/snapshot/remote/stargz.digest"
+
+	// targetImageLayersLabel is a label which contains layer digests contained in
+	// the target image.
+	targetImageLayersLabel = "containerd.io/snapshot/remote/stargz.layers"
+
+	// targetImageURLsLabelPrefix is a label prefix which constructs a map from the layer index to
+	// urls of the layer descriptor.
+	targetImageURLsLabelPrefix = "containerd.io/snapshot/remote/urls."
+
+	// targetURsLLabel is a label which contains layer URL. This is only used to pass URL from containerd
+	// to snapshotter.
+	targetURLsLabel = "containerd.io/snapshot/remote/urls"
+)
+
+// AppendDefaultLabelsHandlerWrapper makes a handler which appends image's basic
+// information to each layer descriptor as annotations during unpack. These
+// annotations will be passed to this remote snapshotter as labels and used to
+// construct source information.
+func AppendDefaultLabelsHandlerWrapper(ref string, prefetchSize int64) func(f images.Handler) images.Handler {
+	return func(f images.Handler) images.Handler {
+		return images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+			children, err := f.Handle(ctx, desc)
+			if err != nil {
+				return nil, err
+			}
+			switch desc.MediaType {
+			case ocispec.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest:
+				for i := range children {
+					c := &children[i]
+					if images.IsLayerType(c.MediaType) {
+						if c.Annotations == nil {
+							c.Annotations = make(map[string]string)
+						}
+						c.Annotations[targetRefLabel] = ref
+						c.Annotations[targetDigestLabel] = c.Digest.String()
+						var layers string
+						for i, l := range children[i:] {
+							if images.IsLayerType(l.MediaType) {
+								ls := fmt.Sprintf("%s,", l.Digest.String())
+								// This avoids the label hits the size limitation.
+								// Skipping layers is allowed here and only affects performance.
+								if err := labels.Validate(targetImageLayersLabel, layers+ls); err != nil {
+									break
+								}
+								layers += ls
+
+								// Store URLs of the neighbouring layer as well.
+								urlsKey := targetImageURLsLabelPrefix + fmt.Sprintf("%d", i)
+								c.Annotations[urlsKey] = appendWithValidation(urlsKey, l.URLs)
+							}
+						}
+						c.Annotations[targetImageLayersLabel] = strings.TrimSuffix(layers, ",")
+						c.Annotations[config.TargetPrefetchSizeLabel] = fmt.Sprintf("%d", prefetchSize)
+
+						// store URL in annotation to let containerd to pass it to the snapshotter
+						c.Annotations[targetURLsLabel] = appendWithValidation(targetURLsLabel, c.URLs)
+					}
+				}
+			}
+			return children, nil
+		})
+	}
+}
+
+func appendWithValidation(key string, values []string) string {
+	var v string
+	for _, u := range values {
+		s := fmt.Sprintf("%s,", u)
+		if err := labels.Validate(key, v+s); err != nil {
+			break
+		}
+		v += s
+	}
+	return strings.TrimSuffix(v, ",")
+}