add HTTP cache (~/Library/Caches/lima)

The `~/Library/Caches/lima/download/by-url-sha256/<SHA256_OF_URL>`
directory contains the following files:

- `url`: raw url text, without "\n"
- `data`: data

Signed-off-by: Akihiro Suda <[email protected]>

Author: AkihiroSuda

cmd/limactl/main.go

@@ -46,6 +46,7 @@ func newApp() *cli.App {
 		shellCommand,
 		lsCommand,
 		deleteCommand,
+		pruneCommand,
 		completionCommand,
 	}
 	return app

cmd/limactl/prune.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var pruneCommand = &cli.Command{
+	Name:   "prune",
+	Usage:  "Prune garbage objects",
+	Action: pruneAction,
+}
+
+func pruneAction(clicontext *cli.Context) error {
+	ucd, err := os.UserCacheDir()
+	if err != nil {
+		return err
+	}
+	cacheDir := filepath.Join(ucd, "lima")
+	logrus.Infof("Pruning %q", cacheDir)
+	return os.RemoveAll(cacheDir)
+}

docs/internal.md

@@ -13,3 +13,11 @@ An instance directory contains the following files:
 - `ga.sock`: Forwarded to `/run/user/$UID/lima-guestagent.sock`
 - `serial.log`: QEMU serial log, for debugging
 - `serial.sock`: QEMU serial socket, for debugging (Usage: `socat -,echo=0,icanon=0 unix-connect:serial.sock`)
+
+
+## Cache directory (`~/Library/Caches/lima/download/by-url-sha256/<SHA256_OF_URL>`)
+
+The directory contains the following files:
+
+- `url`: raw url text, without "\n"
+- `data`: data

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/AkihiroSuda/sshocker v0.1.1-0.20210510144941-56aa3c7472b0
 	github.com/alessio/shellescape v1.4.1
 	github.com/containerd/containerd v1.5.0
+	github.com/containerd/continuity v0.1.0
 	github.com/diskfs/go-diskfs v1.1.2-0.20210512141858-8a6b8b88d14a
 	github.com/docker/go-units v0.4.0
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -164,6 +164,7 @@ github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL
 github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
 github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
+github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
@@ -524,6 +525,7 @@ github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
@@ -812,6 +814,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

pkg/downloader/downloader.go

@@ -0,0 +1,191 @@
+package downloader
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/AkihiroSuda/lima/pkg/localpathutil"
+	"github.com/containerd/continuity/fs"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type Status = string
+
+const (
+	StatusUnknown    Status = ""
+	StatusDownloaded Status = "downloaded"
+	StatusSkipped    Status = "skipped"
+	StatusUsedCache  Status = "used-cache"
+)
+
+type Result struct {
+	Status    Status
+	CachePath string // "/Users/foo/Library/Caches/lima/download/by-url-sha256/<SHA256_OF_URL>/data"
+}
+
+type options struct {
+	cacheDir string // default: empty (disables caching)
+}
+
+type Opt func(*options) error
+
+// WithCache enables caching using filepath.Join(os.UserCacheDir(), "lima") as the cache dir.
+func WithCache() Opt {
+	return func(o *options) error {
+		ucd, err := os.UserCacheDir()
+		if err != nil {
+			return err
+		}
+		cacheDir := filepath.Join(ucd, "lima")
+		return WithCacheDir(cacheDir)(o)
+	}
+}
+
+// WithCacheDir enables caching using the specified dir.
+// Empty value disables caching.
+func WithCacheDir(cacheDir string) Opt {
+	return func(o *options) error {
+		o.cacheDir = cacheDir
+		return nil
+	}
+}
+
+func Download(local, remote string, opts ...Opt) (*Result, error) {
+	var o options
+	for _, f := range opts {
+		if err := f(&o); err != nil {
+			return nil, err
+		}
+	}
+	localPath, err := localPath(local)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := os.Stat(localPath); err == nil {
+		logrus.Debugf("file %q already exists, skipping downloading from %q", localPath, remote)
+		res := &Result{
+			Status: StatusSkipped,
+		}
+		return res, nil
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+
+	localPathDir := filepath.Dir(localPath)
+	if err := os.MkdirAll(localPathDir, 0755); err != nil {
+		return nil, err
+	}
+
+	if isLocal(remote) {
+		if err := copyLocal(localPath, remote); err != nil {
+			return nil, err
+		}
+		res := &Result{
+			Status: StatusDownloaded,
+		}
+		return res, nil
+	}
+
+	if o.cacheDir == "" {
+		if err := downloadHTTP(localPath, remote); err != nil {
+			return nil, err
+		}
+		res := &Result{
+			Status: StatusDownloaded,
+		}
+		return res, nil
+	}
+
+	shad := filepath.Join(o.cacheDir, "download", "by-url-sha256", fmt.Sprintf("%x", sha256.Sum256([]byte(remote))))
+	shadData := filepath.Join(shad, "data")
+	if _, err := os.Stat(shadData); err == nil {
+		logrus.Debugf("file %q is cached as %q", localPath, shadData)
+		if err := copyLocal(localPath, shadData); err != nil {
+			return nil, err
+		}
+		res := &Result{
+			Status:    StatusUsedCache,
+			CachePath: shadData,
+		}
+		return res, nil
+	}
+	if err := os.RemoveAll(shad); err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(shad, 0700); err != nil {
+		return nil, err
+	}
+	shadURL := filepath.Join(shad, "url")
+	if err := os.WriteFile(shadURL, []byte(remote), 0644); err != nil {
+		return nil, err
+	}
+	if err := downloadHTTP(shadData, remote); err != nil {
+		return nil, err
+	}
+	if err := copyLocal(localPath, shadData); err != nil {
+		return nil, err
+	}
+
+	res := &Result{
+		Status:    StatusDownloaded,
+		CachePath: shadData,
+	}
+	return res, nil
+}
+
+func isLocal(s string) bool {
+	return !strings.Contains(s, "://") || strings.HasPrefix(s, "file://")
+}
+
+func localPath(s string) (string, error) {
+	if !isLocal(s) {
+		return "", errors.Errorf("got non-local path: %q", s)
+	}
+	if strings.HasPrefix(s, "file://") {
+		res := strings.TrimPrefix(s, "file://")
+		if !filepath.IsAbs(res) {
+			return "", errors.Errorf("got non-absolute path %q", res)
+		}
+		return res, nil
+	}
+	return localpathutil.Expand(s)
+}
+
+func copyLocal(dst, src string) error {
+	srcPath, err := localPath(src)
+	if err != nil {
+		return err
+	}
+	dstPath, err := localPath(dst)
+	if err != nil {
+		return err
+	}
+	return fs.CopyFile(dstPath, srcPath)
+}
+
+func downloadHTTP(localPath, url string) error {
+	logrus.Debugf("downloading %q into %q", url, localPath)
+	localPathTmp := localPath + ".tmp"
+	if err := os.RemoveAll(localPathTmp); err != nil {
+		return err
+	}
+	// use curl for printing progress
+	cmd := exec.Command("curl", "-fSL", "-o", localPathTmp, url)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "failed to run %v", cmd.Args)
+	}
+	if err := os.RemoveAll(localPath); err != nil {
+		return err
+	}
+	if err := os.Rename(localPathTmp, localPath); err != nil {
+		return err
+	}
+	return nil
+}

pkg/qemu/qemu.go

@@ -7,10 +7,9 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
-	"strings"
 
+	"github.com/AkihiroSuda/lima/pkg/downloader"
 	"github.com/AkihiroSuda/lima/pkg/limayaml"
-	"github.com/AkihiroSuda/lima/pkg/localpathutil"
 	"github.com/docker/go-units"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -31,35 +30,26 @@ func EnsureDisk(cfg Config) error {
 
 	baseDisk := filepath.Join(cfg.InstanceDir, "basedisk")
 	if _, err := os.Stat(baseDisk); errors.Is(err, os.ErrNotExist) {
-		baseDiskTmp := filepath.Join(cfg.InstanceDir, "basedisk.tmp")
-		if err := os.RemoveAll(baseDiskTmp); err != nil {
-			return err
-		}
 		var ensuredBaseDisk bool
 		errs := make([]error, len(cfg.LimaYAML.Images))
 		for i, f := range cfg.LimaYAML.Images {
 			if f.Arch != cfg.LimaYAML.Arch {
 				errs[i] = fmt.Errorf("unsupported arch: %q", f.Arch)
 				continue
 			}
-			url := f.Location
-			if !strings.Contains(url, "://") {
-				expanded, err := localpathutil.Expand(url)
-				if err != nil {
-					return err
-				}
-				url = "file://" + expanded
-			}
-			cmd := exec.Command("curl", "-fSL", "-o", baseDiskTmp, url)
-			cmd.Stdout = os.Stdout
-			cmd.Stderr = os.Stderr
-			logrus.Infof("Attempting to download the image from %q", url)
-			if err := cmd.Run(); err != nil {
-				errs[i] = errors.Wrapf(err, "failed to run %v", cmd.Args)
+			logrus.Infof("Attempting to download the image from %q", f.Location)
+			res, err := downloader.Download(baseDisk, f.Location, downloader.WithCache())
+			if err != nil {
+				errs[i] = errors.Wrapf(err, "failed to download %q", f.Location)
 				continue
 			}
-			if err := os.Rename(baseDiskTmp, baseDisk); err != nil {
-				return err
+			switch res.Status {
+			case downloader.StatusDownloaded:
+				logrus.Infof("Downloaded image from %q", f.Location)
+			case downloader.StatusUsedCache:
+				logrus.Infof("Using cache %q", res.CachePath)
+			default:
+				logrus.Warnf("Unexpected result from downloader.Download(): %+v", res)
 			}
 			ensuredBaseDisk = true
 			break