[content-init] apply UID mapping symmetrically

Author: svenefftinge

components/content-service/go.mod

@@ -11,6 +11,11 @@ replace github.com/Sirupsen/logrus v1.6.0 => github.com/sirupsen/logrus v1.6.0 /
 
 require (
 	cloud.google.com/go/storage v1.10.0
+	github.com/Microsoft/go-winio v0.4.16 // indirect
+	github.com/Microsoft/hcsshim v0.8.14 // indirect
+	github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7 // indirect
+	github.com/docker/docker v1.13.1
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/content-service/api v0.0.0-00010101000000-000000000000
 	github.com/go-ozzo/ozzo-validation v3.5.0+incompatible
@@ -22,6 +27,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.7
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
+	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/prometheus/client_golang v1.1.0
 	github.com/spf13/cobra v1.1.1
@@ -32,6 +38,7 @@ require (
 	golang.org/x/text v0.3.5 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	google.golang.org/api v0.32.0
+	google.golang.org/appengine v1.6.6
 	google.golang.org/grpc v1.34.0
 	google.golang.org/protobuf v1.25.0
 	gopkg.in/ini.v1 v1.62.0 // indirect

components/content-service/go.sum

@@ -5,31 +5,37 @@
 package archive
 
 import (
+	"archive/tar"
 	"bufio"
 	"context"
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
+	"path"
+	"sort"
+	"time"
 
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/idtools"
+	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/tracing"
 	"github.com/opentracing/opentracing-go"
 	"golang.org/x/xerrors"
 )
 
-type buildTarbalConfig struct {
+type tarConfig struct {
 	MaxSizeBytes int64
 	UIDMaps      []idtools.IDMap
 	GIDMaps      []idtools.IDMap
 }
 
 // BuildTarbalOption configures the tarbal creation
-type BuildTarbalOption func(o *buildTarbalConfig)
+type TarOption func(o *tarConfig)
 
 // TarbalMaxSize limits the size of a tarbal
-func TarbalMaxSize(n int64) BuildTarbalOption {
-	return func(o *buildTarbalConfig) {
+func TarbalMaxSize(n int64) TarOption {
+	return func(o *tarConfig) {
 		o.MaxSizeBytes = n
 	}
 }
@@ -42,8 +48,8 @@ type IDMapping struct {
 }
 
 // WithUIDMapping reverses the given user ID mapping during archive creation
-func WithUIDMapping(mappings []IDMapping) BuildTarbalOption {
-	return func(o *buildTarbalConfig) {
+func WithUIDMapping(mappings []IDMapping) TarOption {
+	return func(o *tarConfig) {
 		o.UIDMaps = make([]idtools.IDMap, len(mappings))
 		for i, m := range mappings {
 			o.UIDMaps[i] = idtools.IDMap{
@@ -56,8 +62,8 @@ func WithUIDMapping(mappings []IDMapping) BuildTarbalOption {
 }
 
 // WithGIDMapping reverses the given user ID mapping during archive creation
-func WithGIDMapping(mappings []IDMapping) BuildTarbalOption {
-	return func(o *buildTarbalConfig) {
+func WithGIDMapping(mappings []IDMapping) TarOption {
+	return func(o *tarConfig) {
 		o.GIDMaps = make([]idtools.IDMap, len(mappings))
 		for i, m := range mappings {
 			o.GIDMaps[i] = idtools.IDMap{
@@ -69,9 +75,89 @@ func WithGIDMapping(mappings []IDMapping) BuildTarbalOption {
 	}
 }
 
+// ExtractTarbal extracts an OCI compatible tar file src to the folder dst, expecting the overlay whiteout format
+func ExtractTarbal(ctx context.Context, src io.Reader, dst string, opts ...TarOption) (err error) {
+	var cfg tarConfig
+	start := time.Now()
+	for _, opt := range opts {
+		opt(&cfg)
+	}
+
+	//nolint:staticcheck,ineffassign
+	span, ctx := opentracing.StartSpanFromContext(ctx, "extractTarbal")
+	span.LogKV("src", src, "dst", dst)
+	defer tracing.FinishSpan(span, &err)
+
+	pr, pw := io.Pipe()
+	src = io.TeeReader(src, pw)
+	tarReader := tar.NewReader(pr)
+	type Info struct {
+		UID, GID int
+	}
+	finished := make(chan bool)
+	m := make(map[string]Info)
+	go func() {
+		defer close(finished)
+		for {
+			hdr, err := tarReader.Next()
+			if err == io.EOF {
+				finished <- true
+				return
+			}
+			if err != nil {
+				log.WithError(err).Error("error reading tar")
+				return
+			} else {
+				m[hdr.Name] = Info{
+					UID: hdr.Uid,
+					GID: hdr.Gid,
+				}
+			}
+		}
+	}()
+
+	tarcmd := exec.Command("tar", "x")
+	tarcmd.Dir = dst
+	tarcmd.Stdin = src
+
+	msg, err := tarcmd.CombinedOutput()
+	if err != nil {
+		return xerrors.Errorf("tar %s: %s", dst, err.Error()+";"+string(msg))
+	}
+	<-finished
+
+	// lets create a sorted list of pathes and chown depth first.
+	paths := make([]string, 0, len(m))
+	for path := range m {
+		paths = append(paths, path)
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(paths)))
+	for _, p := range paths {
+		v := m[p]
+		uid := toHostID(v.UID, cfg.UIDMaps)
+		gid := toHostID(v.GID, cfg.GIDMaps)
+		err = os.Lchown(path.Join(dst, p), uid, gid)
+		if err != nil {
+			log.WithError(err).WithField("uid", uid).WithField("gid", gid).WithField("path", p).Warn("cannot chown")
+		}
+	}
+	log.WithField("duration", time.Since(start).Milliseconds()).Debug("untar complete")
+	return nil
+}
+
+func toHostID(containerID int, idMap []idtools.IDMap) int {
+	for _, m := range idMap {
+		if (containerID >= m.ContainerID) && (containerID <= (m.ContainerID + m.Size - 1)) {
+			hostID := m.HostID + (containerID - m.ContainerID)
+			return hostID
+		}
+	}
+	return containerID
+}
+
 // BuildTarbal creates an OCI compatible tar file dst from the folder src, expecting the overlay whiteout format
-func BuildTarbal(ctx context.Context, src string, dst string, opts ...BuildTarbalOption) (err error) {
-	var cfg buildTarbalConfig
+func BuildTarbal(ctx context.Context, src string, dst string, opts ...TarOption) (err error) {
+	var cfg tarConfig
 	for _, opt := range opts {
 		opt(&cfg)
 	}

components/ws-daemon/pkg/archive/tar.go renamed to components/content-service/pkg/archive/tar.go

@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"syscall"
 	"testing"
 )
 
@@ -94,3 +95,81 @@ func TestBuildTarbalMaxSize(t *testing.T) {
 		os.RemoveAll(c)
 	}
 }
+
+func TestExtractTarbal(t *testing.T) {
+	type file = struct {
+		Name        string
+		ContentSize int64
+		Uid         int
+	}
+	tests := []struct {
+		Name  string
+		Files []file
+	}{
+		{
+			Name: "simple-test",
+			Files: []file{
+				{"file.txt", 1024, 33333},
+				{"file2.txt", 1024, 33333},
+			},
+		},
+		{
+			Name:  "empty-tar",
+			Files: []file{},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			wd, err := ioutil.TempDir("", "")
+			defer os.RemoveAll(wd)
+			if err != nil {
+				t.Errorf("cannot prepare test: %v", err)
+				t.FailNow()
+			}
+			sourceFolder := filepath.Join(wd, "source")
+			os.MkdirAll(sourceFolder, 0777)
+
+			for _, file := range test.Files {
+				fileName := filepath.Join(sourceFolder, file.Name)
+				err = ioutil.WriteFile(fileName, make([]byte, file.ContentSize), 0644)
+				if err != nil {
+					t.Errorf("cannot prepare test: %v", err)
+					continue
+				}
+				err = os.Chown(fileName, file.Uid, file.Uid)
+				if err != nil {
+					t.Errorf("Cannot chown %s to %d: %s", file.Name, file.Uid, err)
+				}
+			}
+			tarFile := filepath.Join(wd, "my.tar")
+			BuildTarbal(context.Background(), sourceFolder, tarFile)
+
+			reader, err := os.Open(tarFile)
+			if err != nil {
+				t.Errorf("Cannot open %s", tarFile)
+			}
+			targetFolder := filepath.Join(wd, "target")
+			os.MkdirAll(targetFolder, 0777)
+			ExtractTarbal(context.Background(), reader, targetFolder)
+
+			for _, file := range test.Files {
+				stat, err := os.Stat(filepath.Join(targetFolder, file.Name))
+				if err != nil {
+					t.Errorf("expected %s", file.Name)
+					continue
+				}
+				uid := stat.Sys().(*syscall.Stat_t).Uid
+				if uid != uint32(file.Uid) {
+					t.Errorf("expected uid %d", file.Uid)
+					continue
+				}
+				gid := stat.Sys().(*syscall.Stat_t).Gid
+				if gid != uint32(file.Uid) {
+					t.Errorf("expected gid %d", file.Uid)
+					continue
+				}
+			}
+		})
+	}
+}

components/ws-daemon/pkg/archive/tar_test.go renamed to components/content-service/pkg/archive/tar_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/tracing"
 	csapi "github.com/gitpod-io/gitpod/content-service/api"
+	"github.com/gitpod-io/gitpod/content-service/pkg/archive"
 	"github.com/gitpod-io/gitpod/content-service/pkg/git"
 	"github.com/opentracing/opentracing-go"
 )
@@ -46,7 +47,7 @@ type GitInitializer struct {
 }
 
 // Run initializes the workspace using Git
-func (ws *GitInitializer) Run(ctx context.Context) (src csapi.WorkspaceInitSource, err error) {
+func (ws *GitInitializer) Run(ctx context.Context, mappings []archive.IDMapping) (src csapi.WorkspaceInitSource, err error) {
 	isGitWS := git.IsWorkingCopy(ws.Location)
 
 	span, ctx := opentracing.StartSpanFromContext(ctx, "GitInitializer.Run")

components/content-service/pkg/initializer/git.go

@@ -11,14 +11,14 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/tracing"
 	csapi "github.com/gitpod-io/gitpod/content-service/api"
+	"github.com/gitpod-io/gitpod/content-service/pkg/archive"
 	"github.com/gitpod-io/gitpod/content-service/pkg/git"
 	"github.com/gitpod-io/gitpod/content-service/pkg/storage"
 	"github.com/opentracing/opentracing-go"
@@ -43,30 +43,17 @@ const (
 
 // Initializer can initialize a workspace with content
 type Initializer interface {
-	Run(ctx context.Context) (csapi.WorkspaceInitSource, error)
+	Run(ctx context.Context, mappings []archive.IDMapping) (csapi.WorkspaceInitSource, error)
 }
 
 // EmptyInitializer does nothing
 type EmptyInitializer struct{}
 
 // Run does nothing
-func (e *EmptyInitializer) Run(ctx context.Context) (csapi.WorkspaceInitSource, error) {
+func (e *EmptyInitializer) Run(ctx context.Context, mappings []archive.IDMapping) (csapi.WorkspaceInitSource, error) {
 	return csapi.WorkspaceInitFromOther, nil
 }
 
-// recursiveChown chown's the location and all its childen to
-func recursiveChown(ctx context.Context, location string, uid, gid int) (err error) {
-	span, _ := opentracing.StartSpanFromContext(ctx, "recursiveChown")
-	defer tracing.FinishSpan(span, &err)
-
-	out, err := exec.Command("chown", "-R", fmt.Sprintf("%d:%d", uid, gid), location).CombinedOutput()
-	if err != nil {
-		return xerrors.Errorf("chown -R %s: %s", location, out)
-	}
-
-	return nil
-}
-
 // NewFromRequest picks the initializer from the request but does not execute it.
 // Returns gRPC errors.
 func NewFromRequest(ctx context.Context, loc string, rs storage.DirectDownloader, req *csapi.WorkspaceInitializer) (i Initializer, err error) {
@@ -254,6 +241,14 @@ type initializeOpts struct {
 	InWorkspace bool
 	UID         int
 	GID         int
+	mappings    []archive.IDMapping
+}
+
+// WithMappings configures the UID mappings that're used during content initialization
+func WithMappings(mappings []archive.IDMapping) InitializeOpt {
+	return func(o *initializeOpts) {
+		o.mappings = mappings
+	}
 }
 
 // WithInitializer configures the initializer that's used during content initialization
@@ -331,7 +326,7 @@ func InitializeWorkspace(ctx context.Context, location string, remoteStorage sto
 	}
 
 	// Run the initializer
-	hasBackup, err := remoteStorage.Download(ctx, location, storage.DefaultBackup)
+	hasBackup, err := remoteStorage.Download(ctx, location, storage.DefaultBackup, cfg.mappings)
 	if err != nil {
 		return src, xerrors.Errorf("cannot restore backup: %w", err)
 	}
@@ -340,19 +335,12 @@ func InitializeWorkspace(ctx context.Context, location string, remoteStorage sto
 	if hasBackup {
 		src = csapi.WorkspaceInitFromBackup
 	} else {
-		src, err = cfg.Initializer.Run(ctx)
+		src, err = cfg.Initializer.Run(ctx, cfg.mappings)
 		if err != nil {
 			return src, xerrors.Errorf("cannot initialize workspace: %w", err)
 		}
 	}
 
-	if !cfg.InWorkspace {
-		err = recursiveChown(ctx, location, cfg.UID, cfg.GID)
-		if err != nil {
-			return src, xerrors.Errorf("cannot set workspace permissions: %w", err)
-		}
-	}
-
 	return
 }