[content-service] Remove docker dependency

by moving archive creation back to ws-daemon

Author: Christian Weichel

components/content-service/go.mod

@@ -2,20 +2,8 @@ module github.com/gitpod-io/gitpod/content-service
 
 go 1.14
 
-// containerd, see https://github.com/containerd/containerd/issues/3031
-replace github.com/docker/distribution v2.7.1+incompatible => github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible // leeway ignore
-
-replace github.com/docker/docker v1.13.1 => github.com/docker/engine v0.0.0-20190822205725-ed20165a37b4 // leeway ignore
-
-replace github.com/Sirupsen/logrus v1.6.0 => github.com/sirupsen/logrus v1.6.0 // leeway ignore
-
 require (
 	cloud.google.com/go/storage v1.10.0
-	github.com/Microsoft/go-winio v0.4.16 // indirect
-	github.com/Microsoft/hcsshim v0.8.14 // indirect
-	github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7 // indirect
-	github.com/docker/docker v1.13.1
-	github.com/docker/go-units v0.4.0 // indirect
 	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/content-service/api v0.0.0-00010101000000-000000000000
 	github.com/go-ozzo/ozzo-validation v3.5.0+incompatible
@@ -27,7 +15,6 @@ require (
 	github.com/minio/minio-go/v7 v7.0.7
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
-	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/prometheus/client_golang v1.1.0
 	github.com/spf13/cobra v1.1.1
@@ -38,9 +25,7 @@ require (
 	golang.org/x/text v0.3.5 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	google.golang.org/api v0.32.0
-	google.golang.org/appengine v1.6.6
 	google.golang.org/grpc v1.34.0
-	google.golang.org/protobuf v1.25.0
 	gopkg.in/ini.v1 v1.62.0 // indirect
 )
 

components/content-service/go.sum

@@ -6,36 +6,33 @@ package archive
 
 import (
 	"archive/tar"
-	"bufio"
 	"context"
-	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"path"
 	"sort"
 	"time"
 
-	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/idtools"
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/tracing"
 	"github.com/opentracing/opentracing-go"
 	"golang.org/x/xerrors"
 )
 
-type tarConfig struct {
+// TarConfig configures tarbal creation/extraction
+type TarConfig struct {
 	MaxSizeBytes int64
-	UIDMaps      []idtools.IDMap
-	GIDMaps      []idtools.IDMap
+	UIDMaps      []IDMapping
+	GIDMaps      []IDMapping
 }
 
 // BuildTarbalOption configures the tarbal creation
-type TarOption func(o *tarConfig)
+type TarOption func(o *TarConfig)
 
 // TarbalMaxSize limits the size of a tarbal
 func TarbalMaxSize(n int64) TarOption {
-	return func(o *tarConfig) {
+	return func(o *TarConfig) {
 		o.MaxSizeBytes = n
 	}
 }
@@ -49,35 +46,21 @@ type IDMapping struct {
 
 // WithUIDMapping reverses the given user ID mapping during archive creation
 func WithUIDMapping(mappings []IDMapping) TarOption {
-	return func(o *tarConfig) {
-		o.UIDMaps = make([]idtools.IDMap, len(mappings))
-		for i, m := range mappings {
-			o.UIDMaps[i] = idtools.IDMap{
-				ContainerID: m.ContainerID,
-				HostID:      m.HostID,
-				Size:        m.Size,
-			}
-		}
+	return func(o *TarConfig) {
+		o.UIDMaps = mappings
 	}
 }
 
 // WithGIDMapping reverses the given user ID mapping during archive creation
 func WithGIDMapping(mappings []IDMapping) TarOption {
-	return func(o *tarConfig) {
-		o.GIDMaps = make([]idtools.IDMap, len(mappings))
-		for i, m := range mappings {
-			o.GIDMaps[i] = idtools.IDMap{
-				ContainerID: m.ContainerID,
-				HostID:      m.HostID,
-				Size:        m.Size,
-			}
-		}
+	return func(o *TarConfig) {
+		o.GIDMaps = mappings
 	}
 }
 
 // ExtractTarbal extracts an OCI compatible tar file src to the folder dst, expecting the overlay whiteout format
 func ExtractTarbal(ctx context.Context, src io.Reader, dst string, opts ...TarOption) (err error) {
-	var cfg tarConfig
+	var cfg TarConfig
 	start := time.Now()
 	for _, opt := range opts {
 		opt(&cfg)
@@ -145,7 +128,7 @@ func ExtractTarbal(ctx context.Context, src io.Reader, dst string, opts ...TarOp
 	return nil
 }
 
-func toHostID(containerID int, idMap []idtools.IDMap) int {
+func toHostID(containerID int, idMap []IDMapping) int {
 	for _, m := range idMap {
 		if (containerID >= m.ContainerID) && (containerID <= (m.ContainerID + m.Size - 1)) {
 			hostID := m.HostID + (containerID - m.ContainerID)
@@ -154,98 +137,3 @@ func toHostID(containerID int, idMap []idtools.IDMap) int {
 	}
 	return containerID
 }
-
-// BuildTarbal creates an OCI compatible tar file dst from the folder src, expecting the overlay whiteout format
-func BuildTarbal(ctx context.Context, src string, dst string, opts ...TarOption) (err error) {
-	var cfg tarConfig
-	for _, opt := range opts {
-		opt(&cfg)
-	}
-
-	//nolint:staticcheck,ineffassign
-	span, ctx := opentracing.StartSpanFromContext(ctx, "buildTarbal")
-	span.LogKV("src", src, "dst", dst)
-	defer tracing.FinishSpan(span, &err)
-
-	// ensure the src actually exists before trying to tar it
-	if _, err := os.Stat(src); err != nil {
-		return fmt.Errorf("Unable to tar files: %v", err.Error())
-	}
-
-	tarout, err := archive.TarWithOptions(src, &archive.TarOptions{
-		Compression:    archive.Uncompressed,
-		WhiteoutFormat: archive.OverlayWhiteoutFormat,
-		InUserNS:       true,
-		UIDMaps:        cfg.UIDMaps,
-		GIDMaps:        cfg.GIDMaps,
-	})
-	if err != nil {
-		return xerrors.Errorf("cannot create tar: %w", err)
-	}
-
-	fout, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, 0744)
-	if err != nil {
-		return xerrors.Errorf("cannot open archive for writing: %w", err)
-	}
-	defer fout.Close()
-	fbout := bufio.NewWriter(fout)
-	defer fbout.Flush()
-
-	targetOut := newLimitWriter(fbout, cfg.MaxSizeBytes)
-	defer func(e *error) {
-		if targetOut.DidMaxOut() {
-			*e = ErrMaxSizeExceeded
-		}
-	}(&err)
-
-	_, err = io.Copy(targetOut, tarout)
-	if err != nil {
-		return xerrors.Errorf("cannot write tar file: %w")
-	}
-	if err = fbout.Flush(); err != nil {
-		return xerrors.Errorf("cannot flush tar out stream: %w", err)
-	}
-
-	return nil
-}
-
-// ErrMaxSizeExceeded is emitted by LimitWriter when a write tries to write beyond the max number of bytes allowed
-var ErrMaxSizeExceeded = fmt.Errorf("maximum size exceeded")
-
-// newLimitWriter wraps a writer such that a maximum of N bytes can be written. Once that limit is exceeded
-// the writer returns io.ErrClosedPipe
-func newLimitWriter(out io.Writer, maxSizeBytes int64) *limitWriter {
-	return &limitWriter{
-		MaxSizeBytes: maxSizeBytes,
-		Out:          out,
-	}
-}
-
-type limitWriter struct {
-	MaxSizeBytes int64
-	Out          io.Writer
-	BytesWritten int64
-
-	didMaxOut bool
-}
-
-func (s *limitWriter) Write(b []byte) (n int, err error) {
-	if s.MaxSizeBytes == 0 {
-		return s.Out.Write(b)
-	}
-
-	bsize := int64(len(b))
-	if bsize+s.BytesWritten > s.MaxSizeBytes {
-		s.didMaxOut = true
-		return 0, ErrMaxSizeExceeded
-	}
-
-	n, err = s.Out.Write(b)
-	s.BytesWritten += int64(n)
-
-	return n, err
-}
-
-func (s *limitWriter) DidMaxOut() bool {
-	return s.didMaxOut
-}

components/content-service/pkg/archive/tar.go

@@ -5,6 +5,8 @@
 package archive
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"io/ioutil"
 	"os"
@@ -13,91 +15,8 @@ import (
 	"testing"
 )
 
-func TestSizeLimitingWriter(t *testing.T) {
-	tests := []struct {
-		MaxSize       int64
-		BlockSize     int64
-		BlockCount    int
-		ExpectWritten int
-		Err           error
-	}{
-		{30, 10, 1, 10, nil},
-		{30, 10, 3, 30, nil},
-		{20, 10, 3, 20, ErrMaxSizeExceeded},
-	}
-
-	for _, test := range tests {
-		var (
-			written int
-			n       int
-			err     error
-		)
-		w := newLimitWriter(ioutil.Discard, test.MaxSize)
-
-		for i := 0; i < test.BlockCount; i++ {
-			n, err = w.Write(make([]byte, test.BlockSize))
-			written += n
-			if err != nil {
-				break
-			}
-		}
-
-		if err != test.Err {
-			t.Errorf("unexpected error: expected %v, actual %v", test.Err, err)
-		}
-		if written != test.ExpectWritten {
-			t.Errorf("wrote unexpected number of bytes: expected %v, actual %v", test.ExpectWritten, written)
-		}
-	}
-}
-
-func TestBuildTarbalMaxSize(t *testing.T) {
-	tests := []struct {
-		Name        string
-		MaxSize     int64
-		ContentSize int64
-		Err         error
-	}{
-		{"positive", 1024 * 1024, 512, nil},
-		{"too-big", 512, 1024, ErrMaxSizeExceeded},
-	}
-
-	var cleanup []string
-	for _, test := range tests {
-		wd, err := ioutil.TempDir("", "")
-		if err != nil {
-			t.Errorf("cannot prepare test: %v", err)
-			continue
-		}
-		cleanup = append(cleanup, wd)
-
-		err = ioutil.WriteFile(filepath.Join(wd, "content.txt"), make([]byte, test.ContentSize), 0644)
-		if err != nil {
-			t.Errorf("cannot prepare test: %v", err)
-			continue
-		}
-
-		tgt, err := ioutil.TempFile("", "")
-		if err != nil {
-			t.Errorf("cannot prepare test: %v", err)
-			continue
-		}
-		tgt.Close()
-		cleanup = append(cleanup, tgt.Name())
-
-		err = BuildTarbal(context.Background(), wd, tgt.Name(), TarbalMaxSize(test.MaxSize))
-		if (err == nil && test.Err != nil) || (err != nil && test.Err == nil) || (err != nil && test.Err != nil && err.Error() != test.Err.Error()) {
-			t.Errorf("%s: unexpected error: expected \"%v\", actual \"%v\"", test.Name, test.Err, err)
-		}
-	}
-
-	for _, c := range cleanup {
-		os.RemoveAll(c)
-	}
-}
-
 func TestExtractTarbal(t *testing.T) {
-	type file = struct {
+	type file struct {
 		Name        string
 		ContentSize int64
 		Uid         int
@@ -121,37 +40,38 @@ func TestExtractTarbal(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.Name, func(t *testing.T) {
-			wd, err := ioutil.TempDir("", "")
-			defer os.RemoveAll(wd)
-			if err != nil {
-				t.Errorf("cannot prepare test: %v", err)
-				t.FailNow()
-			}
-			sourceFolder := filepath.Join(wd, "source")
-			os.MkdirAll(sourceFolder, 0777)
-
+			var (
+				buf = bytes.NewBuffer(nil)
+				tw  = tar.NewWriter(buf)
+			)
 			for _, file := range test.Files {
-				fileName := filepath.Join(sourceFolder, file.Name)
-				err = ioutil.WriteFile(fileName, make([]byte, file.ContentSize), 0644)
+				err := tw.WriteHeader(&tar.Header{
+					Name:     file.Name,
+					Size:     file.ContentSize,
+					Uid:      file.Uid,
+					Gid:      file.Uid,
+					Mode:     0644,
+					Typeflag: tar.TypeReg,
+				})
 				if err != nil {
-					t.Errorf("cannot prepare test: %v", err)
-					continue
+					t.Fatalf("cannot prepare archive: %q", err)
 				}
-				err = os.Chown(fileName, file.Uid, file.Uid)
+				_, err = tw.Write(make([]byte, file.ContentSize))
 				if err != nil {
-					t.Errorf("Cannot chown %s to %d: %s", file.Name, file.Uid, err)
+					t.Fatalf("cannot prepare archive: %q", err)
 				}
 			}
-			tarFile := filepath.Join(wd, "my.tar")
-			BuildTarbal(context.Background(), sourceFolder, tarFile)
+			tw.Flush()
+			tw.Close()
 
-			reader, err := os.Open(tarFile)
+			wd, err := ioutil.TempDir("", "")
+			defer os.RemoveAll(wd)
 			if err != nil {
-				t.Errorf("Cannot open %s", tarFile)
+				t.Fatalf("cannot prepare test: %v", err)
 			}
 			targetFolder := filepath.Join(wd, "target")
 			os.MkdirAll(targetFolder, 0777)
-			ExtractTarbal(context.Background(), reader, targetFolder)
+			ExtractTarbal(context.Background(), buf, targetFolder)
 
 			for _, file := range test.Files {
 				stat, err := os.Stat(filepath.Join(targetFolder, file.Name))

components/content-service/pkg/archive/tar_test.go

@@ -125,6 +125,7 @@ github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.
 github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.4.1 h1:pASeJT3R3YyVn+94qEPk0SnU1OQ20Jd/T+SPKy9xehY=
 github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20200928162600-f2cc35102c2a h1:jEIoR0aA5GogXZ8pP3DUzE+zrhaF6/1rYZy+7KkYEWM=
 github.com/containerd/continuity v0.0.0-20200928162600-f2cc35102c2a/go.mod h1:W0qIOTD7mp2He++YVq+kgfXezRYqzP1uDuMVH1bITDY=