added support for docker volumes (#198)

* added support for docker volumes

* fixed linting

* added test cases for docker volumes

* minor fix

* fixed linting

Co-authored-by: Saurabh Prakash <[email protected]>

Author: nevilm-lt

.vscode/settings.json

@@ -2,5 +2,6 @@
     "go.lintTool": "golangci-lint",
     "go.lintFlags": [
         "--fast"
-    ]
+    ],
+    "go.testTimeout": "90s"
 }

cmd/synapse/bin.go

@@ -98,7 +98,7 @@ func run(cmd *cobra.Command, args []string) {
 
 	// setting up cron handler
 	wg.Add(1)
-	go cron.Setup(ctx, &wg, logger)
+	go cron.Setup(ctx, &wg, logger, runner)
 
 	// All attempts to connect to lambdatest server failed
 	connectionFailed := make(chan struct{})

config/parse.go

@@ -90,7 +90,6 @@ func recursivelySet(val reflect.Value, prefix string) error {
 				return fmt.Errorf("unexpected type detected ~ aborting: %s", thisField.Kind())
 			}
 		}
-
 	}
 
 	return nil

pkg/core/runner.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	"context"
+	"time"
 
 	"github.com/LambdaTest/test-at-scale/config"
 	"github.com/LambdaTest/test-at-scale/pkg/errs"
@@ -45,7 +46,7 @@ type DockerRunner interface {
 	// Run runs the execution engine
 	Run(context.Context, *RunnerOptions) ContainerStatus
 
-	//WaitForRunning waits for runner to get completed
+	// WaitForRunning waits for runner to get completed
 	WaitForCompletion(ctx context.Context, r *RunnerOptions) error
 
 	// Destroy the execution engine
@@ -62,6 +63,32 @@ type DockerRunner interface {
 
 	// KillRunningDocker kills  container spawn by synapse
 	KillRunningDocker(ctx context.Context)
+
+	CreateVolume(ctx context.Context, r *RunnerOptions) error
+
+	// RemoveOldVolumes removes volumes that are older than X hours
+	RemoveOldVolumes(ctx context.Context)
+
+	// CopyFileToContainer copies content to container in file
+	CopyFileToContainer(ctx context.Context, path, fileName, containerID string, content []byte) error
+
+	// FindVolumes checks if docker volume is available
+	FindVolumes(volumeName string) (bool, error)
+
+	// RemoveVolume removes volume
+	RemoveVolume(ctx context.Context, volumeName string) error
+}
+
+// VolumeDetails docker volume options
+type VolumeDetails struct {
+	CreatedAt  time.Time              `json:"CreatedAt,omitempty"`
+	Driver     string                 `json:"Driver"`
+	Labels     map[string]string      `json:"Labels"`
+	Mountpoint string                 `json:"Mountpoint"`
+	Name       string                 `json:"Name"`
+	Options    map[string]string      `json:"Options"`
+	Scope      string                 `json:"Scope"`
+	Status     map[string]interface{} `json:"Status,omitempty"`
 }
 
 // RunnerOptions provides the the required instructions for execution engine.

pkg/core/secrets.go

@@ -15,14 +15,15 @@ type SecretsManager interface {
 	// GetLambdatestSecrets returns lambdatest config
 	GetLambdatestSecrets() *config.LambdatestConfig
 
-	// WriteGitSecrets writes git secrets to file
-	WriteGitSecrets(path string) error
-
-	// WriteRepoSecrets writes repo secrets to file
-	WriteRepoSecrets(repo string, path string) error
-
 	// GetDockerSecrets returns Mode , RegistryAuth, and URL for pulling remote docker image
 	GetDockerSecrets(r *RunnerOptions) (ContainerImageConfig, error)
+
 	// GetSynapseName returns synapse name mentioned in config
 	GetSynapseName() string
+
+	// GetGitSecretBytes get git secrets in bytes
+	GetGitSecretBytes() ([]byte, error)
+
+	// GetRepoSecretBytes get repo secrets in bytes
+	GetRepoSecretBytes(repo string) ([]byte, error)
 }

pkg/cron/setup.go

@@ -2,27 +2,19 @@ package cron
 
 import (
 	"context"
-	"fmt"
-	"os"
 	"sync"
-	"time"
 
+	"github.com/LambdaTest/test-at-scale/pkg/core"
 	"github.com/LambdaTest/test-at-scale/pkg/lumber"
-	"github.com/LambdaTest/test-at-scale/pkg/utils"
 	"github.com/robfig/cron/v3"
 )
 
-const (
-	buildCacheExpiry time.Duration = 4 * time.Hour
-	buildCacheDir    string        = "/tmp/synapse"
-)
-
 // Setup initializes all crons on service startup
-func Setup(ctx context.Context, wg *sync.WaitGroup, logger lumber.Logger) {
+func Setup(ctx context.Context, wg *sync.WaitGroup, logger lumber.Logger, runner core.DockerRunner) {
 	defer wg.Done()
 
 	c := cron.New()
-	if _, err := c.AddFunc("@every 5m", func() { cleanupBuildCache(logger) }); err != nil {
+	if _, err := c.AddFunc("@every 5m", func() { cleanupBuildCache(runner) }); err != nil {
 		logger.Errorf("error setting up cron")
 		return
 	}
@@ -33,23 +25,6 @@ func Setup(ctx context.Context, wg *sync.WaitGroup, logger lumber.Logger) {
 	logger.Infof("Caller has requested graceful shutdown. Returning.....")
 }
 
-func cleanupBuildCache(logger lumber.Logger) {
-	files, err := os.ReadDir(buildCacheDir)
-	if err != nil {
-		logger.Errorf("error in reading directory: %s", err)
-		return
-	}
-	for _, file := range files {
-		now := time.Now()
-		info, err := file.Info()
-		if err != nil {
-			logger.Errorf("error retrieving file info")
-		}
-		if diff := now.Sub(info.ModTime()); diff > buildCacheExpiry {
-			filePath := fmt.Sprintf("%s/%s", buildCacheDir, file.Name())
-			if err := utils.DeleteDirectory(filePath); err != nil {
-				logger.Errorf("error deleting directory: %s", err.Error())
-			}
-		}
-	}
+func cleanupBuildCache(runner core.DockerRunner) {
+	runner.RemoveOldVolumes(context.Background())
 }

pkg/errs/synapse.go

@@ -180,7 +180,7 @@ func ERR_FIL_CRT(err string) Err {
 		Message: fmt.Sprintf("Unable to create file :  \n%s", err)}
 }
 
-//ERR_API_WEB_HOK function returns error with code ERR::API::WEB::HOK
+// ERR_API_WEB_HOK function returns error with code ERR::API::WEB::HOK
 func ERR_API_WEB_HOK(err string) Err {
 	return Err{
 		Code:    "ERR::API::WEB::HOK",
@@ -208,6 +208,27 @@ func ERR_DOCKER_STRT(err string) Err {
 		Message: fmt.Sprintf("Docker start failed with error:  \n%s", err)}
 }
 
+// ErrDockerVolCrt function returns error with code "ERR::DOCKER::VOL::CRT"
+func ErrDockerVolCrt(err string) Err {
+	return Err{
+		Code:    "ERR::DOCKER::VOL::CRT",
+		Message: fmt.Sprintf("Docker volume create failed with error:  \n%s", err)}
+}
+
+// ErrDockerCP function returns error with code "ERR::DOCKER::CP"
+func ErrDockerCP(err string) Err {
+	return Err{
+		Code:    "ERR::DOCKER::CP",
+		Message: fmt.Sprintf("Error copying file to docker:  \n%s", err)}
+}
+
+// ErrSecretLoad function returns error with code "ERR::SECRET::LOAD"
+func ErrSecretLoad(err string) Err {
+	return Err{
+		Code:    "ERR::SECRET::LOAD",
+		Message: fmt.Sprintf("Error in loading secrets:  \n%s", err)}
+}
+
 // ERR_JSON_MAR function returns error with code "ERR::JSON::MAR"
 func ERR_JSON_MAR(err string) Err {
 	return Err{

pkg/global/synapseconstants.go

@@ -10,6 +10,7 @@ const (
 	ProxyServerPort       = "8000"
 	DirectoryPermissions  = 0755
 	FilePermissions       = 0755
+	VaultSecretDir        = "/vault/secrets"
 	GitConfigFileName     = "oauth"
 	RepoSecretsFileName   = "reposecrets"
 	SynapseContainerURL   = "http://synapse:8000"

pkg/proxyserver/setup.go

@@ -45,5 +45,4 @@ func ListenAndServe(ctx context.Context, proxyHandler *ProxyHandler, config *con
 	case <-done:
 		return nil
 	}
-
 }

pkg/runner/docker/config.go

@@ -15,15 +15,29 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/go-units"
 )
 
 const (
 	defaultVaultPath = "/vault/secrets"
 	repoSourcePath   = "/tmp/synapse/%s/nucleus"
 	nanoCPUUnit      = 1e9
+	volumePrefix     = "tas-build"
 )
 
+func (d *docker) getVolumeName(r *core.RunnerOptions) string {
+	return fmt.Sprintf("%s-%s", volumePrefix, r.Label[synapse.BuildID])
+}
+
+func (d *docker) getVolumeConfiguration(r *core.RunnerOptions) *volume.VolumeCreateBody {
+	return &volume.VolumeCreateBody{
+		Driver: "local",
+		Name:   d.getVolumeName(r),
+		Labels: map[string]string{synapse.BuildID: r.Label[synapse.BuildID]},
+	}
+}
+
 func (d *docker) getContainerConfiguration(r *core.RunnerOptions) *container.Config {
 	return &container.Config{
 		Image:   r.DockerImage,
@@ -44,8 +58,8 @@ func (d *docker) getContainerHostConfiguration(r *core.RunnerOptions) *container
 	d.logger.Infof("Specs %+v", specs)
 	mounts := []mount.Mount{
 		{
-			Type:   mount.TypeBind,
-			Source: r.HostVolumePath,
+			Type:   mount.TypeVolume,
+			Source: d.getVolumeName(r),
 			Target: defaultVaultPath,
 		},
 	}
@@ -55,8 +69,8 @@ func (d *docker) getContainerHostConfiguration(r *core.RunnerOptions) *container
 			d.logger.Errorf("error creating directory: %v", err)
 		}
 		mounts = append(mounts, mount.Mount{
-			Type:   mount.TypeBind,
-			Source: repoBuildSourcePath,
+			Type:   mount.TypeVolume,
+			Source: d.getVolumeName(r),
 			Target: global.WorkspaceCacheDir,
 		})
 	}