Add tests

Signed-off-by: Amory Hoste <[email protected]>

Author: amohoste

ctriface/image/Makefile

@@ -0,0 +1,33 @@
+# MIT License
+#
+# Copyright (c) 2020 Dmitrii Ustiugov, Plamen Petrov and EASE lab
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+EXTRAGOARGS:=-v -race -cover
+
+test:
+	# Need to pass GOROOT because GitHub-hosted runners may have several
+	# go versions installed so that calling go from root may fail
+	sudo env "PATH=$(PATH)" "GOROOT=$(GOROOT)" go test ./ $(EXTRAGOARGS)
+
+test-man:
+	echo "Nothing to test manually"
+
+.PHONY: test test-man

ctrimages/imageManager.go renamed to ctriface/image/manager.go

@@ -21,7 +21,7 @@
 // SOFTWARE.
 
 // Package ctrimages provides an image manager that manages and caches container images.
-package ctrimages
+package image
 
 import (
 	"context"
@@ -35,29 +35,29 @@ import (
 	"sync"
 )
 
-// ImageState is used to synchronize image pulling to avoid pulling the same image multiple times concurrently.
+// ImageState is used for synchronization to avoid pulling the same image multiple times concurrently.
 type ImageState struct {
 	sync.Mutex
-	pulled bool
+	isCached bool
 }
 
-// NewImageState creates a new ImageState object that can be used to synchronize image pulling.
+// NewImageState creates a new ImageState object that can be used to synchronize pulling a single image
 func NewImageState() *ImageState {
 	state := new(ImageState)
-	state.pulled = false
+	state.isCached = false
 	return state
 }
 
 // ImageManager manages the images that have been pulled to the node.
 type ImageManager struct {
 	sync.Mutex
-	snapshotter  string						 // image snapshotter
+	snapshotter  string // image snapshotter
 	cachedImages map[string]containerd.Image // Cached container images
 	imageStates  map[string]*ImageState
 	client       *containerd.Client
 }
 
-// NewImageManager creates a new imagemanager that can be used to fetch container images.
+// NewImageManager creates a new image manager that can be used to fetch container images.
 func NewImageManager(client *containerd.Client, snapshotter string) *ImageManager {
 	log.Info("Creating image manager")
 	manager := new(ImageManager)
@@ -104,8 +104,10 @@ func (mgr *ImageManager) pullImage(ctx context.Context, imageName string) error
 	return nil
 }
 
-// GetImage fetches an image that can be used to create a container using containerd
+// GetImage fetches an image that can be used to create a container using containerd. Synchronization is implemented
+// on a per image level to keep waiting to a minimum.
 func (mgr *ImageManager) GetImage(ctx context.Context, imageName string) (*containerd.Image, error) {
+	// Get reference to synchronization object for image
 	mgr.Lock()
 	imgState, found := mgr.imageStates[imageName]
 	if !found {
@@ -114,14 +116,14 @@ func (mgr *ImageManager) GetImage(ctx context.Context, imageName string) (*conta
 	}
 	mgr.Unlock()
 
-	// Pull image if necessary
+	// Pull image if necessary. The image will only be pulled by the first thread to take the lock.
 	imgState.Lock()
-	if !imgState.pulled {
+	if !imgState.isCached {
 		if err := mgr.pullImage(ctx, imageName); err != nil {
 			imgState.Unlock()
 			return nil, err
 		}
-		imgState.pulled = true
+		imgState.isCached = true
 	}
 	imgState.Unlock()
 

ctriface/image/manager_test.go

@@ -0,0 +1,135 @@
+// MIT License
+//
+// Copyright (c) 2020 Plamen Petrov, Amory Hoste and EASE lab
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package image
+
+import (
+	"context"
+	"fmt"
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/namespaces"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	ctrdlog "github.com/containerd/containerd/log"
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	TestImageName = "ghcr.io/ease-lab/helloworld:var_workload"
+	containerdAddress      = "/run/firecracker-containerd/containerd.sock"
+	NamespaceName          = "firecracker-containerd"
+)
+
+func getAllImages() map[string]string {
+	return map[string]string{
+		"helloworld":          "ghcr.io/ease-lab/helloworld:var_workload",
+		"chameleon":           "ghcr.io/ease-lab/chameleon:var_workload",
+		"pyaes":               "ghcr.io/ease-lab/pyaes:var_workload",
+		"image_rotate":        "ghcr.io/ease-lab/image_rotate:var_workload",
+		"lr_training":         "ghcr.io/ease-lab/lr_training:var_workload",
+	}
+}
+
+func TestMain(m *testing.M) {
+	// call flag.Parse() here if TestMain uses flags
+
+	log.SetFormatter(&log.TextFormatter{
+		TimestampFormat: ctrdlog.RFC3339NanoFixed,
+		FullTimestamp:   true,
+	})
+	//log.SetReportCaller(true) // FIXME: make sure it's false unless debugging
+
+	log.SetOutput(os.Stdout)
+
+	log.SetLevel(log.InfoLevel)
+
+	os.Exit(m.Run())
+}
+
+func TestSingleConcurrent(t *testing.T) {
+	// Create client
+	client, err := containerd.New(containerdAddress)
+	defer client.Close()
+	require.NoError(t, err, "Containerd client creation returned error")
+
+	// Create image manager
+	mgr := NewImageManager(client, "devmapper")
+
+	testTimeout := 120 * time.Second
+	ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), NamespaceName), testTimeout)
+	defer cancel()
+
+	// Pull image
+	var wg sync.WaitGroup
+	concurrentPulls := 100
+	wg.Add(concurrentPulls)
+
+	for i := 0; i < concurrentPulls; i++ {
+		go func(i int) {
+			defer wg.Done()
+			_, err := mgr.GetImage(ctx, TestImageName)
+			require.NoError(t, err, fmt.Sprintf("Failed to pull image %s", TestImageName))
+		}(i)
+	}
+	wg.Wait()
+}
+
+func TestMultipleConcurrent(t *testing.T) {
+	// Create client
+	client, err := containerd.New(containerdAddress)
+	defer client.Close()
+	require.NoError(t, err, "Containerd client creation returned error")
+
+	// Create image manager
+	mgr := NewImageManager(client, "devmapper")
+
+	testTimeout := 120 * time.Second
+	ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), NamespaceName), testTimeout)
+	defer cancel()
+
+	// Pull image
+	var wg sync.WaitGroup
+	concurrentPulls := 100
+	wg.Add(len(getAllImages()))
+
+	for _, imgName := range getAllImages() {
+		go func(imgName string) {
+			var imgWg sync.WaitGroup
+			imgWg.Add(concurrentPulls)
+			for i := 0; i < concurrentPulls; i++ {
+				go func(i int) {
+					defer wg.Done()
+					_, err := mgr.GetImage(ctx, imgName)
+					require.NoError(t, err, fmt.Sprintf("Failed to pull image %s", imgName))
+				}(i)
+			}
+			imgWg.Wait()
+			wg.Done()
+		}(imgName)
+	}
+
+	wg.Wait()
+}

ctriface/orch.go

@@ -23,7 +23,7 @@
 package ctriface
 
 import (
-	"github.com/ease-lab/vhive/ctrimages"
+	"github.com/ease-lab/vhive/ctriface/image"
 	"github.com/ease-lab/vhive/devmapper"
 	"github.com/ease-lab/vhive/memory/manager"
 	"os"
@@ -81,7 +81,7 @@ type Orchestrator struct {
 	client       *containerd.Client
 	fcClient     *fcclient.Client
 	devMapper    *devmapper.DeviceMapper
-	imageManager *ctrimages.ImageManager
+	imageManager *image.ImageManager
 	// store *skv.KVStore
 	snapshotsEnabled bool
 	isUPFEnabled     bool
@@ -140,7 +140,7 @@ func NewOrchestrator(snapshotter, hostIface, poolName, metadataDev string, netPo
 
 	o.devMapper = devmapper.NewDeviceMapper(o.client, poolName, metadataDev)
 
-	o.imageManager = ctrimages.NewImageManager(o.client, o.snapshotter)
+	o.imageManager = image.NewImageManager(o.client, o.snapshotter)
 
 	return o
 }

networking/Makefile

@@ -0,0 +1,33 @@
+# MIT License
+#
+# Copyright (c) 2020 Dmitrii Ustiugov, Plamen Petrov and EASE lab
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+EXTRAGOARGS:=-v -race -cover
+
+test:
+	# Need to pass GOROOT because GitHub-hosted runners may have several
+	# go versions installed so that calling go from root may fail
+	sudo env "PATH=$(PATH)" "GOROOT=$(GOROOT)" go test ./ $(EXTRAGOARGS)
+
+test-man:
+	echo "Nothing to test manually"
+
+.PHONY: test test-man

networking/networkManager.go

@@ -42,6 +42,9 @@ type NetworkManager struct {
 
 	// Mapping of function instance IDs to their network config
 	netConfigs      map[string]*NetworkConfig
+
+	// Network configs that are being created
+	inCreation      sync.WaitGroup
 }
 
 // NewNetworkManager creates and returns a new network manager that connects function instances to the network
@@ -98,6 +101,7 @@ func (mgr *NetworkManager) addNetConfig() {
 	mgr.Lock()
 	id := mgr.nextID
 	mgr.nextID += 1
+	mgr.inCreation.Add(1)
 	mgr.Unlock()
 
 	netCfg := NewNetworkConfig(id, mgr.hostIfaceName)
@@ -110,6 +114,7 @@ func (mgr *NetworkManager) addNetConfig() {
 	// Signal in case someone is waiting for a new config to become available in the pool
 	mgr.poolCond.Signal()
 	mgr.poolCond.L.Unlock()
+	mgr.inCreation.Done()
 }
 
 // allocNetConfig allocates a new network config from the pool to a function instance identified by funcID
@@ -171,16 +176,30 @@ func (mgr *NetworkManager) RemoveNetwork(funcID string) error {
 	return nil
 }
 
-// Cleanup removes and deallocates all network configurations that are in use or in the network pool.
+// Cleanup removes and deallocates all network configurations that are in use or in the network pool. Make sure to first
+// clean up all running functions before removing their network configs.
 func (mgr *NetworkManager) Cleanup() error {
 	log.Info("Cleaning up network manager")
 	mgr.Lock()
 	defer mgr.Unlock()
 
+	// Wait till all network configs still in creation are added
+	mgr.inCreation.Wait()
+
 	// Release network configs still in use
+	var wgu sync.WaitGroup
+	wgu.Add(len(mgr.netConfigs))
 	for funcID := range mgr.netConfigs {
-		mgr.releaseNetConfig(funcID)
+		config := mgr.netConfigs[funcID]
+		go func(config *NetworkConfig) {
+			if err := config.RemoveNetwork(); err != nil {
+				log.Errorf("failed to remove network %s:", err)
+			}
+			wgu.Done()
+		}(config)
 	}
+	wgu.Wait()
+	mgr.netConfigs = make(map[string]*NetworkConfig)
 
 	// Cleanup network pool
 	mgr.poolCond.L.Lock()

networking/networkconfig.go

@@ -39,12 +39,12 @@ const (
 )
 
 // NetworkConfig represents the network devices, IPs, namespaces, routes and filter rules to connect a uVM
-// to the network. Note that due to the current allocation of IPs at most 2^14 VMs can be simultaneously be
-// available on a single host.
+// to the network. The network config ID is deterministically mapped to IP addresses to be used for the uVM.
+// Note that due to the current allocation of IPs at most 2^14 VMs can be simultaneously be available on a single host.
 type NetworkConfig struct {
 	id             int
 	containerCIDR  string // Container IP address (CIDR notation)
-	gatewayCIDR    string // Container gateway IP address
+	gatewayCIDR    string // Container gateway IP address (CIDR notation)
 	containerTap   string // Container tap name
 	containerMac   string // Container Mac address
 	hostIfaceName  string // Host network interface name