Merge pull request moby#3398 from crazy-max/test-s3-cache

integration: basic s3 cache test

Author: crazy-max

.github/workflows/build.yml

@@ -184,29 +184,6 @@ jobs:
           SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }}
           CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}
 
-  test-s3:
-    runs-on: ubuntu-20.04
-    needs:
-      - base
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Expose GitHub Runtime
-        uses: crazy-max/ghaction-github-runtime@v2
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          version: ${{ env.BUILDX_VERSION }}
-          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-          buildkitd-flags: --debug
-      -
-        name: Test
-        run: |
-          hack/s3_test/run_test.sh
-
   test-azblob:
     runs-on: ubuntu-20.04
     needs:

Dockerfile

@@ -11,9 +11,15 @@ ARG STARGZ_SNAPSHOTTER_VERSION=v0.13.0
 ARG NERDCTL_VERSION=v1.0.0
 ARG DNSNAME_VERSION=v1.3.1
 ARG NYDUS_VERSION=v2.1.0
+ARG MINIO_VERSION=RELEASE.2022-05-03T20-36-08Z
+ARG MINIO_MC_VERSION=RELEASE.2022-05-04T06-07-55Z
 
 ARG ALPINE_VERSION=3.17
 
+# minio for s3 integration tests
+FROM minio/minio:${MINIO_VERSION} AS minio
+FROM minio/mc:${MINIO_MC_VERSION} AS minio-mc
+
 # alpine base for buildkit image
 # TODO: remove this when alpine image supports riscv64
 FROM alpine:${ALPINE_VERSION} AS alpine-amd64
@@ -240,6 +246,8 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 ENV BUILDKIT_INTEGRATION_CONTAINERD_EXTRA="containerd-1.6=/opt/containerd-alt-16/bin"
 ENV BUILDKIT_INTEGRATION_SNAPSHOTTER=stargz
 ENV CGO_ENABLED=0
+COPY --link --from=minio /opt/bin/minio /usr/bin/
+COPY --link --from=minio-mc /usr/bin/mc /usr/bin/
 COPY --link --from=nydus /out/nydus-static/* /usr/bin/
 COPY --link --from=stargz-snapshotter /out/* /usr/bin/
 COPY --link --from=rootlesskit /rootlesskit /usr/bin/

client/client_test.go

@@ -111,6 +111,7 @@ func TestIntegration(t *testing.T) {
 		testReadonlyRootFS,
 		testBasicRegistryCacheImportExport,
 		testBasicLocalCacheImportExport,
+		testBasicS3CacheImportExport,
 		testCachedMounts,
 		testCopyFromEmptyImage,
 		testProxyEnv,
@@ -4672,6 +4673,44 @@ func testBasicLocalCacheImportExport(t *testing.T, sb integration.Sandbox) {
 	testBasicCacheImportExport(t, sb, []CacheOptionsEntry{im}, []CacheOptionsEntry{ex})
 }
 
+func testBasicS3CacheImportExport(t *testing.T, sb integration.Sandbox) {
+	integration.CheckFeatureCompat(t, sb, integration.FeatureCacheExport)
+
+	opts := integration.MinioOpts{
+		Region:          "us-east-1",
+		AccessKeyID:     "minioadmin",
+		SecretAccessKey: "minioadmin",
+	}
+
+	s3Addr, s3Bucket, cleanup, err := integration.NewMinioServer(t, sb, opts)
+	require.NoError(t, err)
+	defer cleanup()
+
+	im := CacheOptionsEntry{
+		Type: "s3",
+		Attrs: map[string]string{
+			"region":            opts.Region,
+			"access_key_id":     opts.AccessKeyID,
+			"secret_access_key": opts.SecretAccessKey,
+			"bucket":            s3Bucket,
+			"endpoint_url":      s3Addr,
+			"use_path_style":    "true",
+		},
+	}
+	ex := CacheOptionsEntry{
+		Type: "s3",
+		Attrs: map[string]string{
+			"region":            opts.Region,
+			"access_key_id":     opts.AccessKeyID,
+			"secret_access_key": opts.SecretAccessKey,
+			"bucket":            s3Bucket,
+			"endpoint_url":      s3Addr,
+			"use_path_style":    "true",
+		},
+	}
+	testBasicCacheImportExport(t, sb, []CacheOptionsEntry{im}, []CacheOptionsEntry{ex})
+}
+
 func testBasicInlineCacheImportExport(t *testing.T, sb integration.Sandbox) {
 	integration.CheckFeatureCompat(t, sb, integration.FeatureDirectPush, integration.FeatureCacheImport)
 	requiresLinux(t)

hack/s3_test/Dockerfile

@@ -0,0 +1,116 @@
+package integration
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	minioBin = "minio"
+	mcBin    = "mc"
+)
+
+type MinioOpts struct {
+	Region          string
+	AccessKeyID     string
+	SecretAccessKey string
+}
+
+func NewMinioServer(t *testing.T, sb Sandbox, opts MinioOpts) (address string, bucket string, cl func() error, err error) {
+	t.Helper()
+	bucket = randomString(10)
+
+	if _, err := exec.LookPath(minioBin); err != nil {
+		return "", "", nil, errors.Wrapf(err, "failed to lookup %s binary", minioBin)
+	}
+	if _, err := exec.LookPath(mcBin); err != nil {
+		return "", "", nil, errors.Wrapf(err, "failed to lookup %s binary", mcBin)
+	}
+
+	deferF := &multiCloser{}
+	cl = deferF.F()
+
+	defer func() {
+		if err != nil {
+			deferF.F()()
+			cl = nil
+		}
+	}()
+
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		return "", "", nil, err
+	}
+
+	addr := l.Addr().String()
+	if err = l.Close(); err != nil {
+		return "", "", nil, err
+	}
+	address = "http://" + addr
+
+	// start server
+	cmd := exec.Command(minioBin, "server", "--json", "--address", addr, t.TempDir())
+	cmd.Env = append(os.Environ(), []string{
+		"MINIO_ROOT_USER=" + opts.AccessKeyID,
+		"MINIO_ROOT_PASSWORD=" + opts.SecretAccessKey,
+	}...)
+	minioStop, err := startCmd(cmd, sb.Logs())
+	if err != nil {
+		return "", "", nil, err
+	}
+	if err = waitMinio(address, 15*time.Second); err != nil {
+		minioStop()
+		return "", "", nil, errors.Wrapf(err, "minio did not start up: %s", formatLogs(sb.Logs()))
+	}
+	deferF.append(minioStop)
+
+	// create alias config
+	alias := randomString(10)
+	cmd = exec.Command(mcBin, "alias", "set", alias, address, opts.AccessKeyID, opts.SecretAccessKey)
+	if err := runCmd(cmd, sb.Logs()); err != nil {
+		return "", "", nil, err
+	}
+	deferF.append(func() error {
+		return exec.Command(mcBin, "alias", "rm", alias).Run()
+	})
+
+	// create bucket
+	cmd = exec.Command(mcBin, "mb", "--region", opts.Region, fmt.Sprintf("%s/%s", alias, bucket)) // #nosec G204
+	if err := runCmd(cmd, sb.Logs()); err != nil {
+		return "", "", nil, err
+	}
+
+	// trace
+	cmd = exec.Command(mcBin, "admin", "trace", "--json", alias)
+	traceStop, err := startCmd(cmd, sb.Logs())
+	if err != nil {
+		return "", "", nil, err
+	}
+	deferF.append(traceStop)
+
+	return
+}
+
+func waitMinio(address string, d time.Duration) error {
+	step := 1 * time.Second
+	i := 0
+	for {
+		if resp, err := http.Get(fmt.Sprintf("%s/minio/health/live", address)); err == nil {
+			resp.Body.Close()
+			break
+		}
+		i++
+		if time.Duration(i)*step > d {
+			return errors.Errorf("failed dialing: %s", address)
+		}
+		time.Sleep(step)
+	}
+	return nil
+}