Improve cache handling and provence generation (#202)

* Improve cache handling and provence generation

* Update dependencies

* Update to dependencies

* Improve CI jobs

* Fix remote cache configuration

* Cleanup

Author: aledbf

.devcontainer/devcontainer.json

@@ -1,6 +1,13 @@
 {
 	"name": "leeway",
-	"image": "mcr.microsoft.com/devcontainers/go:1.22",
-	// Features to add to the dev container. More info: https://containers.dev/features.
-	"features": {}
+	"build": {
+		"context": "..",
+		"dockerfile": "../.gitpod.Dockerfile"
+	},
+	"runArgs": [
+		"--privileged",
+		"--security-opt=seccomp=unconfined",
+		"--network=host"
+	],
+	"containerUser": "root"
 }

.github/workflows/ci-build.yaml

@@ -9,7 +9,7 @@ on:
       - 'main'
 
 env:
-  GO_VERSION: '1.23'
+  GO_VERSION: '1.24'
 
 jobs:
   check-go:

.github/workflows/release.yaml

@@ -19,7 +19,7 @@ jobs:
       - name: Setup Golang
         uses: actions/setup-go@v5
         with:
-          go-version: "1.23"
+          go-version: "1.24"
       - name: Docker Login
         uses: docker/login-action@v1
         with:
@@ -31,7 +31,7 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
         with:
           distribution: goreleaser
-          version: v2.3.2
+          version: "~> v2"
           args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitpod.Dockerfile

@@ -1,3 +1,52 @@
-FROM gitpod/workspace-full
+FROM ubuntu:jammy
 
-USER root
+ADD https://raw.githubusercontent.com/gitpod-io/workspace-images/main/base/install-packages /usr/bin/install-packages
+RUN chmod +x /usr/bin/install-packages
+
+RUN install-packages \
+    zip unzip \
+    jq \
+    curl \
+    ca-certificates \
+    file \
+    git \
+    node.js
+
+ENV GO_VERSION=1.24.0
+RUN echo "TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64}" && \
+    case "${TARGETPLATFORM:-linux/amd64}" in \
+    "linux/arm64") \
+      echo "GO_PLATFORM=linux-arm64" > /tmp/go_platform.env \
+      ;; \
+    *) \
+      echo "GO_PLATFORM=linux-amd64" > /tmp/go_platform.env \
+      ;; \
+    esac
+
+# Install Go and add it to PATH
+RUN . /tmp/go_platform.env && \
+    curl -fsSL https://dl.google.com/go/go$GO_VERSION.$GO_PLATFORM.tar.gz | tar -C /usr/local -xzs
+
+# Set Go environment variables
+ENV GOROOT=/usr/local/go
+ENV PATH=$GOROOT/bin:$PATH
+ENV GOPATH=/root/go
+ENV PATH=$GOPATH/bin:$PATH
+
+# install VS Code Go tools for use with gopls as per https://github.com/golang/vscode-go/blob/master/docs/tools.md
+# also https://github.com/golang/vscode-go/blob/27bbf42a1523cadb19fad21e0f9d7c316b625684/src/goTools.ts#L139
+RUN go install -v github.com/uudashr/gopkgs/cmd/gopkgs@v2 \
+    && go install -v github.com/ramya-rao-a/go-outline@latest \
+    && go install -v github.com/cweill/gotests/gotests@latest \
+    && go install -v github.com/fatih/gomodifytags@latest \
+    && go install -v github.com/josharian/impl@latest \
+    && go install -v github.com/haya14busa/goplay/cmd/goplay@latest \
+    && go install -v github.com/go-delve/delve/cmd/dlv@latest \
+    && go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest \
+    && go install -v golang.org/x/tools/gopls@latest \
+    && go install -v honnef.co/go/tools/cmd/staticcheck@latest \
+    && rm -rf /root/.cache
+
+ENV SHFMT_VERSION=3.10.0
+RUN curl -sSL -o /usr/local/bin/shfmt "https://github.com/mvdan/sh/releases/download/v${SHFMT_VERSION}/shfmt_v${SHFMT_VERSION}_linux_amd64" && \
+    chmod 755 /usr/local/bin/shfmt

cmd/build.go

@@ -12,6 +12,9 @@ import (
 	"time"
 
 	"github.com/gitpod-io/leeway/pkg/leeway"
+	"github.com/gitpod-io/leeway/pkg/leeway/cache"
+	"github.com/gitpod-io/leeway/pkg/leeway/cache/local"
+	"github.com/gitpod-io/leeway/pkg/leeway/cache/remote"
 	"github.com/gookit/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -84,7 +87,7 @@ var buildCmd = &cobra.Command{
 	},
 }
 
-func serveBuildResult(ctx context.Context, addr string, localCache *leeway.FilesystemCache, pkg *leeway.Package) {
+func serveBuildResult(ctx context.Context, addr string, localCache cache.LocalCache, pkg *leeway.Package) {
 	br, exists := localCache.Location(pkg)
 	if !exists {
 		log.Fatal("build result is not in local cache despite just being built. Something's wrong with the cache.")
@@ -121,7 +124,7 @@ func serveBuildResult(ctx context.Context, addr string, localCache *leeway.Files
 	}
 }
 
-func saveBuildResult(ctx context.Context, loc string, localCache *leeway.FilesystemCache, pkg *leeway.Package) {
+func saveBuildResult(ctx context.Context, loc string, localCache cache.LocalCache, pkg *leeway.Package) {
 	br, exists := localCache.Location(pkg)
 	if !exists {
 		log.Fatal("build result is not in local cache despite just being built. Something's wrong with the cache.")
@@ -178,15 +181,15 @@ func addBuildFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("report-github", os.Getenv("GITHUB_OUTPUT") != "", "Report package build success/failure to GitHub Actions using the GITHUB_OUTPUT environment variable")
 }
 
-func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemCache) {
+func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) {
 	cm, _ := cmd.Flags().GetString("cache")
 	log.WithField("cacheMode", cm).Debug("configuring caches")
 	cacheLevel := leeway.CacheLevel(cm)
 
 	remoteCache := getRemoteCache()
 	switch cacheLevel {
 	case leeway.CacheNone, leeway.CacheLocal:
-		remoteCache = leeway.NoRemoteCache{}
+		remoteCache = remote.NewNoRemoteCache()
 	case leeway.CacheRemotePull:
 		remoteCache = &pullOnlyRemoteCache{C: remoteCache}
 	case leeway.CacheRemotePush:
@@ -212,7 +215,7 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemC
 		}
 	}
 	log.WithField("location", localCacheLoc).Debug("set up local cache")
-	localCache, err := leeway.NewFilesystemCache(localCacheLoc)
+	localCache, err := local.NewFilesystemCache(localCacheLoc)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -310,33 +313,67 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemC
 }
 
 type pushOnlyRemoteCache struct {
-	C leeway.RemoteCache
+	C cache.RemoteCache
 }
 
-func (c *pushOnlyRemoteCache) ExistingPackages(pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
-	return c.C.ExistingPackages(pkgs)
+func (c *pushOnlyRemoteCache) ExistingPackages(ctx context.Context, pkgs []cache.Package) (map[cache.Package]struct{}, error) {
+	return c.C.ExistingPackages(ctx, pkgs)
 }
 
-func (c *pushOnlyRemoteCache) Download(dst leeway.Cache, pkgs []*leeway.Package) error {
+func (c *pushOnlyRemoteCache) Download(ctx context.Context, dst cache.LocalCache, pkgs []cache.Package) error {
 	return nil
 }
 
-func (c *pushOnlyRemoteCache) Upload(src leeway.Cache, pkgs []*leeway.Package) error {
-	return c.C.Upload(src, pkgs)
+func (c *pushOnlyRemoteCache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache.Package) error {
+	return c.C.Upload(ctx, src, pkgs)
 }
 
 type pullOnlyRemoteCache struct {
-	C leeway.RemoteCache
+	C cache.RemoteCache
 }
 
-func (c *pullOnlyRemoteCache) ExistingPackages(pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
-	return c.C.ExistingPackages(pkgs)
+func (c *pullOnlyRemoteCache) ExistingPackages(ctx context.Context, pkgs []cache.Package) (map[cache.Package]struct{}, error) {
+	return c.C.ExistingPackages(ctx, pkgs)
 }
 
-func (c *pullOnlyRemoteCache) Download(dst leeway.Cache, pkgs []*leeway.Package) error {
-	return c.C.Download(dst, pkgs)
+func (c *pullOnlyRemoteCache) Download(ctx context.Context, dst cache.LocalCache, pkgs []cache.Package) error {
+	return c.C.Download(ctx, dst, pkgs)
 }
 
-func (c *pullOnlyRemoteCache) Upload(src leeway.Cache, pkgs []*leeway.Package) error {
+func (c *pullOnlyRemoteCache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache.Package) error {
 	return nil
 }
+
+func getRemoteCache() cache.RemoteCache {
+	remoteCacheBucket := os.Getenv(EnvvarRemoteCacheBucket)
+	remoteStorage := os.Getenv(EnvvarRemoteCacheStorage)
+	if remoteCacheBucket != "" {
+		switch remoteStorage {
+		case "GCP":
+			return remote.NewGSUtilCache(
+				&cache.RemoteConfig{
+					BucketName: remoteCacheBucket,
+				},
+			)
+		case "AWS":
+			rc, err := remote.NewS3Cache(
+				&cache.RemoteConfig{
+					BucketName: remoteCacheBucket,
+				},
+			)
+			if err != nil {
+				log.Fatalf("cannot access remote S3 cache: %v", err)
+			}
+
+			return rc
+		default:
+			return remote.NewGSUtilCache(
+				&cache.RemoteConfig{
+					BucketName: remoteCacheBucket,
+				},
+			)
+		}
+	}
+
+	return remote.NewNoRemoteCache()
+}

cmd/provenance-assert.go

@@ -73,7 +73,7 @@ var provenanceAssertCmd = &cobra.Command{
 				return nil
 			}
 
-			failures = append(assertions.AssertEnvelope(env), failures...)
+			failures = append(assertions.AssertBundle(env), failures...)
 
 			raw, err := base64.StdEncoding.DecodeString(env.Payload)
 			if err != nil {

cmd/root.go

@@ -173,33 +173,6 @@ func getBuildArgs() (leeway.Arguments, error) {
 	return res, nil
 }
 
-func getRemoteCache() leeway.RemoteCache {
-	remoteCacheBucket := os.Getenv(EnvvarRemoteCacheBucket)
-	remoteStorage := os.Getenv(EnvvarRemoteCacheStorage)
-	if remoteCacheBucket != "" {
-		switch remoteStorage {
-		case "GCP":
-			return leeway.GSUtilRemoteCache{
-				BucketName: remoteCacheBucket,
-			}
-		case "AWS":
-			rc, err := leeway.NewS3RemoteCache(remoteCacheBucket, nil)
-			if err != nil {
-				log.Fatalf("cannot access remote S3 cache: %v", err)
-			}
-
-			return rc
-		default:
-			return leeway.GSUtilRemoteCache{
-				BucketName: remoteCacheBucket,
-			}
-		}
-
-	}
-
-	return leeway.NoRemoteCache{}
-}
-
 func addExperimentalCommand(parent, child *cobra.Command) {
 	if os.Getenv("LEEWAY_EXPERIMENTAL") != "true" {
 		return

go.mod

@@ -1,13 +1,12 @@
 module github.com/gitpod-io/leeway
 
-go 1.23
+go 1.24
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.32.3
-	github.com/aws/aws-sdk-go-v2/config v1.28.1
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2
-	github.com/aws/aws-sdk-go-v2/service/sts v1.32.3
+	github.com/aws/aws-sdk-go-v2 v1.36.1
+	github.com/aws/aws-sdk-go-v2/config v1.29.6
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.59
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.75.4
 	github.com/creack/pty v1.1.23
 	github.com/disiqueira/gotree v1.0.0
 	github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd
@@ -16,7 +15,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/gookit/color v1.5.4
 	github.com/imdario/mergo v0.3.13
-	github.com/in-toto/in-toto-golang v0.3.3
+	github.com/in-toto/in-toto-golang v0.9.0
 	github.com/karrick/godirwalk v1.17.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/opencontainers/runc v1.1.10
@@ -26,50 +25,50 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
-	golang.org/x/mod v0.21.0
-	golang.org/x/sync v0.8.0
+	golang.org/x/mod v0.23.0
+	golang.org/x/sync v0.11.0
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da
 	gopkg.in/yaml.v3 v3.0.1
-	sigs.k8s.io/bom v0.1.0
+	sigs.k8s.io/bom v0.6.0
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.42 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
-	github.com/aws/smithy-go v1.22.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.59 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect
+	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/cyphar/filepath-securejoin v0.3.4 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.11.4 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
-	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/godbus/dbus/v5 v5.0.6 // indirect
 	github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
-	github.com/moby/sys/mountinfo v0.7.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/moby/sys/mountinfo v0.5.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
-	github.com/seccomp/libseccomp-golang v0.10.0 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect
 	github.com/segmentio/backo-go v1.0.0 // indirect
-	github.com/shibumi/go-pathspec v1.2.0 // indirect
+	github.com/shibumi/go-pathspec v1.3.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	sigs.k8s.io/release-utils v0.7.7 // indirect
 )