Implement an artifact cache in the depot.

We cache up to 500MB of unused artifacts by default, but this size is configurable.

Author: mitchell-as

internal/constants/constants.go

@@ -493,3 +493,6 @@ const IgnoreEnvEnvVarName = "ACTIVESTATE_CLI_IGNORE_ENV"
 
 // ProgressUrlPathName is the trailing path for a project's build progress.
 const BuildProgressUrlPathName = "distributions"
+
+// RuntimeCacheSizeConfigKey is the config key for the runtime cache size.
+const RuntimeCacheSizeConfigKey = "runtime.cache_size"

internal/fileutils/fileutils.go

@@ -1299,3 +1299,22 @@ func windowsPathToPosixPath(path string) string {
 func posixPathToWindowsPath(path string) string {
 	return strings.ReplaceAll(path, "/", "\\")
 }
+
+// GetDirSize returns the on-disk size of the given directory and all of its contents.
+// Does not follow symlinks.
+func GetDirSize(dir string) (int64, error) {
+	var size int64
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if !info.IsDir() {
+			size += info.Size()
+		}
+		return nil
+	})
+	if err != nil {
+		return size, errs.Wrap(err, "Unable to compute size")
+	}
+	return size, nil
+}

internal/runbits/runtime/runtime.go

@@ -281,6 +281,7 @@ func Update(
 	if proj.IsPortable() {
 		rtOpts = append(rtOpts, runtime.WithPortable())
 	}
+	rtOpts = append(rtOpts, runtime.WithCacheSize(prime.Config().GetInt(constants.RuntimeCacheSizeConfigKey)))
 
 	if isArmPlatform(buildPlan) {
 		prime.Output().Notice(locale.Tl("warning_arm_unstable", "[WARNING]Warning:[/RESET] You are using an ARM64 architecture, which is currently unstable. While it may work, you might encounter issues."))

pkg/runtime/depot.go

@@ -7,14 +7,19 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"sync"
+	"time"
 
 	"github.com/go-openapi/strfmt"
 
+	"github.com/ActiveState/cli/internal/constants"
 	"github.com/ActiveState/cli/internal/errs"
 	"github.com/ActiveState/cli/internal/fileutils"
 	"github.com/ActiveState/cli/internal/installation/storage"
 	"github.com/ActiveState/cli/internal/logging"
+	configMediator "github.com/ActiveState/cli/internal/mediators/config"
+	"github.com/ActiveState/cli/internal/multilog"
 	"github.com/ActiveState/cli/internal/sliceutils"
 	"github.com/ActiveState/cli/internal/smartlink"
 )
@@ -24,7 +29,8 @@ const (
 )
 
 type depotConfig struct {
-	Deployments map[strfmt.UUID][]deployment `json:"deployments"`
+	Deployments map[strfmt.UUID][]deployment  `json:"deployments"`
+	Cache       map[strfmt.UUID]*artifactInfo `json:"cache"`
 }
 
 type deployment struct {
@@ -41,6 +47,12 @@ const (
 	deploymentTypeCopy                = "copy"
 )
 
+type artifactInfo struct {
+	InUse          bool  `json:"inUse"`
+	Size           int64 `json:"size"`
+	LastAccessTime int64 `json:"lastAccessTime"`
+}
+
 type ErrVolumeMismatch struct {
 	DepotVolume string
 	PathVolume  string
@@ -55,8 +67,15 @@ type depot struct {
 	depotPath string
 	artifacts map[strfmt.UUID]struct{}
 	fsMutex   *sync.Mutex
+	cacheSize int64
+}
+
+func init() {
+	configMediator.RegisterOption(constants.RuntimeCacheSizeConfigKey, configMediator.Int, 500)
 }
 
+const MB int64 = 1024 * 1024
+
 func newDepot(runtimePath string) (*depot, error) {
 	depotPath := filepath.Join(storage.CachePath(), depotName)
 
@@ -73,6 +92,7 @@ func newDepot(runtimePath string) (*depot, error) {
 	result := &depot{
 		config: depotConfig{
 			Deployments: map[strfmt.UUID][]deployment{},
+			Cache:       map[strfmt.UUID]*artifactInfo{},
 		},
 		depotPath: depotPath,
 		artifacts: map[strfmt.UUID]struct{}{},
@@ -122,6 +142,10 @@ func newDepot(runtimePath string) (*depot, error) {
 	return result, nil
 }
 
+func (d *depot) SetCacheSize(mb int) {
+	d.cacheSize = int64(mb) * MB
+}
+
 func (d *depot) Exists(id strfmt.UUID) bool {
 	_, ok := d.artifacts[id]
 	return ok
@@ -196,6 +220,7 @@ func (d *depot) DeployViaLink(id strfmt.UUID, relativeSrc, absoluteDest string)
 		Files:       files.RelativePaths(),
 		RelativeSrc: relativeSrc,
 	})
+	d.recordUse(id)
 
 	return nil
 }
@@ -253,10 +278,28 @@ func (d *depot) DeployViaCopy(id strfmt.UUID, relativeSrc, absoluteDest string)
 		Files:       files.RelativePaths(),
 		RelativeSrc: relativeSrc,
 	})
+	d.recordUse(id)
 
 	return nil
 }
 
+func (d *depot) recordUse(id strfmt.UUID) {
+	// Ensure a cache entry for this artifact exists and then update its last access time.
+	if _, exists := d.config.Cache[id]; !exists {
+		size, err := fileutils.GetDirSize(d.Path(id))
+		if err != nil {
+			multilog.Error("Could not get artifact size on disk: %v", err)
+			size = 0
+		}
+		logging.Debug("Recording artifact '%s' with size %.1f MB", id.String(), float64(size)/float64(MB))
+		d.config.Cache[id] = &artifactInfo{Size: size}
+	} else {
+		logging.Debug("Recording use of artifact '%s'", id.String())
+	}
+	d.config.Cache[id].InUse = true
+	d.config.Cache[id].LastAccessTime = time.Now().Unix()
+}
+
 func (d *depot) Undeploy(id strfmt.UUID, relativeSrc, path string) error {
 	d.fsMutex.Lock()
 	defer d.fsMutex.Unlock()
@@ -372,14 +415,14 @@ func (d *depot) getSharedFilesToRedeploy(id strfmt.UUID, deploy deployment, path
 // Save will write config changes to disk (ie. links between depot artifacts and runtimes that use it).
 // It will also delete any stale artifacts which are not used by any runtime.
 func (d *depot) Save() error {
-	// Delete artifacts that are no longer used
+	// Mark artifacts that are no longer used and remove the old ones.
 	for id := range d.artifacts {
 		if deployments, ok := d.config.Deployments[id]; !ok || len(deployments) == 0 {
-			if err := os.RemoveAll(d.Path(id)); err != nil {
-				return errs.Wrap(err, "failed to remove stale artifact")
-			}
+			d.config.Cache[id].InUse = false
+			logging.Debug("Artifact '%s' is no longer in use", id.String())
 		}
 	}
+	d.removeStaleArtifacts()
 
 	// Write config file changes to disk
 	configFile := filepath.Join(d.depotPath, depotFile)
@@ -422,3 +465,39 @@ func someFilesExist(filePaths []string, basePath string) bool {
 	}
 	return false
 }
+
+// removeStaleArtifacts iterates over all unused artifacts in the depot, sorts them by last access
+// time, and removes them until the size of cached artifacts is under the limit.
+func (d *depot) removeStaleArtifacts() {
+	type artifact struct {
+		id   strfmt.UUID
+		info *artifactInfo
+	}
+	var totalSize int64
+	unusedArtifacts := make([]*artifact, 0)
+
+	for id, info := range d.config.Cache {
+		if !info.InUse {
+			totalSize += info.Size
+			unusedArtifacts = append(unusedArtifacts, &artifact{id: id, info: info})
+		}
+	}
+	logging.Debug("There are %d unused artifacts totaling %.1f MB in size", len(unusedArtifacts), float64(totalSize)/float64(MB))
+
+	sort.Slice(unusedArtifacts, func(i, j int) bool {
+		return unusedArtifacts[i].info.LastAccessTime < unusedArtifacts[j].info.LastAccessTime
+	})
+
+	for _, artifact := range unusedArtifacts {
+		if totalSize <= d.cacheSize {
+			break // done
+		}
+		logging.Debug("Removing cached artifact '%s', last accessed on %s", artifact.id.String(), time.Unix(artifact.info.LastAccessTime, 0).Format(time.UnixDate))
+		if err := os.RemoveAll(d.Path(artifact.id)); err == nil {
+			totalSize -= artifact.info.Size
+		} else {
+			multilog.Error("Could not delete old artifact: %v", err)
+		}
+		delete(d.config.Cache, artifact.id)
+	}
+}

pkg/runtime/options.go

@@ -25,6 +25,10 @@ func WithPortable() SetOpt {
 	return func(opts *Opts) { opts.Portable = true }
 }
 
+func WithCacheSize(mb int) SetOpt {
+	return func(opts *Opts) { opts.CacheSize = mb }
+}
+
 func WithArchive(dir string, platformID strfmt.UUID, ext string) SetOpt {
 	return func(opts *Opts) {
 		opts.FromArchive = &fromArchive{dir, platformID, ext}

pkg/runtime/setup.go

@@ -48,6 +48,7 @@ type Opts struct {
 	BuildlogFilePath     string
 	BuildProgressUrl     string
 	Portable             bool
+	CacheSize            int
 
 	FromArchive *fromArchive
 
@@ -93,6 +94,7 @@ type setup struct {
 }
 
 func newSetup(path string, bp *buildplan.BuildPlan, env *envdef.Collection, depot *depot, opts *Opts) (*setup, error) {
+	depot.SetCacheSize(opts.CacheSize)
 	installedArtifacts := depot.List(path)
 
 	var platformID strfmt.UUID

test/integration/runtime_int_test.go

@@ -212,6 +212,35 @@ func (suite *RuntimeIntegrationTestSuite) TestIgnoreEnvironmentVars() {
 	cp.ExpectExitCode(0)
 }
 
+func (suite *RuntimeIntegrationTestSuite) TestRuntimeCache() {
+	suite.OnlyRunForTags(tagsuite.Critical)
+	ts := e2e.New(suite.T(), false)
+	defer ts.Close()
+
+	ts.PrepareProject("ActiveState-CLI/Empty", "b55d0e63-db48-43c4-8341-e2b7a1cc134c")
+
+	cp := ts.Spawn("install", "shared/zlib")
+	cp.Expect("Downloading")
+	cp.ExpectExitCode(0)
+
+	cp = ts.Spawn("reset", "-n") // should not remove cached shared/zlib artifact
+	cp.ExpectExitCode(0)
+
+	cp = ts.Spawn("install", "shared/zlib")
+	cp.ExpectExitCode(0)
+	suite.Assert().NotContains(cp.Snapshot(), "Downloading", "shared/zlib should have been cached")
+
+	cp = ts.Spawn("config", "set", constants.RuntimeCacheSizeConfigKey, "0")
+	cp.ExpectExitCode(0)
+
+	cp = ts.Spawn("reset", "-n") // should remove cached shared/zlib artifact
+	cp.ExpectExitCode(0)
+
+	cp = ts.Spawn("install", "shared/zlib")
+	cp.Expect("Downloading")
+	cp.ExpectExitCode(0)
+}
+
 func TestRuntimeIntegrationTestSuite(t *testing.T) {
 	suite.Run(t, new(RuntimeIntegrationTestSuite))
 }