Merge pull request moby#4035 from vito/pluggable-sources

Author: jedevc

solver/llbsolver/ops/source.go

@@ -60,7 +60,7 @@ func (s *SourceOp) instance(ctx context.Context) (source.SourceInstance, error)
 	if s.src != nil {
 		return s.src, nil
 	}
-	id, err := source.FromLLB(s.op, s.platform)
+	id, err := s.sm.Identifier(s.op, s.platform)
 	if err != nil {
 		return nil, err
 	}

solver/llbsolver/provenance.go

@@ -20,7 +20,6 @@ import (
 	"github.com/moby/buildkit/solver/llbsolver/ops"
 	"github.com/moby/buildkit/solver/llbsolver/provenance"
 	"github.com/moby/buildkit/solver/pb"
-	"github.com/moby/buildkit/source"
 	"github.com/moby/buildkit/worker"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -268,70 +267,9 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc
 		switch op := pp.(type) {
 		case *ops.SourceOp:
 			id, pin := op.Pin()
-			switch s := id.(type) {
-			case *source.ImageIdentifier:
-				dgst, err := digest.Parse(pin)
-				if err != nil {
-					return errors.Wrapf(err, "failed to parse image digest %s", pin)
-				}
-				c.AddImage(provenance.ImageSource{
-					Ref:      s.Reference.String(),
-					Platform: s.Platform,
-					Digest:   dgst,
-				})
-			case *source.LocalIdentifier:
-				c.AddLocal(provenance.LocalSource{
-					Name: s.Name,
-				})
-			case *source.GitIdentifier:
-				url := s.Remote
-				if s.Ref != "" {
-					url += "#" + s.Ref
-				}
-				c.AddGit(provenance.GitSource{
-					URL:    url,
-					Commit: pin,
-				})
-				if s.AuthTokenSecret != "" {
-					c.AddSecret(provenance.Secret{
-						ID:       s.AuthTokenSecret,
-						Optional: true,
-					})
-				}
-				if s.AuthHeaderSecret != "" {
-					c.AddSecret(provenance.Secret{
-						ID:       s.AuthHeaderSecret,
-						Optional: true,
-					})
-				}
-				if s.MountSSHSock != "" {
-					c.AddSSH(provenance.SSH{
-						ID:       s.MountSSHSock,
-						Optional: true,
-					})
-				}
-			case *source.HTTPIdentifier:
-				dgst, err := digest.Parse(pin)
-				if err != nil {
-					return errors.Wrapf(err, "failed to parse HTTP digest %s", pin)
-				}
-				c.AddHTTP(provenance.HTTPSource{
-					URL:    s.URL,
-					Digest: dgst,
-				})
-			case *source.OCIIdentifier:
-				dgst, err := digest.Parse(pin)
-				if err != nil {
-					return errors.Wrapf(err, "failed to parse OCI digest %s", pin)
-				}
-				c.AddImage(provenance.ImageSource{
-					Ref:      s.Reference.String(),
-					Platform: s.Platform,
-					Digest:   dgst,
-					Local:    true,
-				})
-			default:
-				return errors.Errorf("unknown source identifier %T", id)
+			err := id.Capture(c, pin)
+			if err != nil {
+				return err
 			}
 		case *ops.ExecOp:
 			pr := op.Proto()

solver/llbsolver/vertex.go

@@ -9,7 +9,6 @@ import (
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/llbsolver/ops/opsutils"
 	"github.com/moby/buildkit/solver/pb"
-	"github.com/moby/buildkit/source"
 	"github.com/moby/buildkit/util/entitlements"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -322,13 +321,6 @@ func loadLLB(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEval
 func llbOpName(pbOp *pb.Op, load func(digest.Digest) (solver.Vertex, error)) (string, error) {
 	switch op := pbOp.Op.(type) {
 	case *pb.Op_Source:
-		if id, err := source.FromLLB(op, nil); err == nil {
-			if id, ok := id.(*source.LocalIdentifier); ok {
-				if len(id.IncludePatterns) == 1 {
-					return op.Source.Identifier + " (" + id.IncludePatterns[0] + ")", nil
-				}
-			}
-		}
 		return op.Source.Identifier, nil
 	case *pb.Op_Exec:
 		return strings.Join(op.Exec.Meta.Args, " "), nil

source/containerimage/identifier.go

@@ -0,0 +1,92 @@
+package containerimage
+
+import (
+	"github.com/containerd/containerd/reference"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/solver/llbsolver/provenance"
+	"github.com/moby/buildkit/source"
+	srctypes "github.com/moby/buildkit/source/types"
+	"github.com/moby/buildkit/util/resolver"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+type ImageIdentifier struct {
+	Reference   reference.Spec
+	Platform    *ocispecs.Platform
+	ResolveMode resolver.ResolveMode
+	RecordType  client.UsageRecordType
+	LayerLimit  *int
+}
+
+func NewImageIdentifier(str string) (*ImageIdentifier, error) {
+	ref, err := reference.Parse(str)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	if ref.Object == "" {
+		return nil, errors.WithStack(reference.ErrObjectRequired)
+	}
+	return &ImageIdentifier{Reference: ref}, nil
+}
+
+var _ source.Identifier = (*ImageIdentifier)(nil)
+
+func (*ImageIdentifier) Scheme() string {
+	return srctypes.DockerImageScheme
+}
+
+func (id *ImageIdentifier) Capture(c *provenance.Capture, pin string) error {
+	dgst, err := digest.Parse(pin)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse image digest %s", pin)
+	}
+	c.AddImage(provenance.ImageSource{
+		Ref:      id.Reference.String(),
+		Platform: id.Platform,
+		Digest:   dgst,
+	})
+	return nil
+}
+
+type OCIIdentifier struct {
+	Reference  reference.Spec
+	Platform   *ocispecs.Platform
+	SessionID  string
+	StoreID    string
+	LayerLimit *int
+}
+
+func NewOCIIdentifier(str string) (*OCIIdentifier, error) {
+	ref, err := reference.Parse(str)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	if ref.Object == "" {
+		return nil, errors.WithStack(reference.ErrObjectRequired)
+	}
+	return &OCIIdentifier{Reference: ref}, nil
+}
+
+var _ source.Identifier = (*OCIIdentifier)(nil)
+
+func (*OCIIdentifier) Scheme() string {
+	return srctypes.OCIScheme
+}
+
+func (id *OCIIdentifier) Capture(c *provenance.Capture, pin string) error {
+	dgst, err := digest.Parse(pin)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse OCI digest %s", pin)
+	}
+	c.AddImage(provenance.ImageSource{
+		Ref:      id.Reference.String(),
+		Platform: id.Platform,
+		Digest:   dgst,
+		Local:    true,
+	})
+	return nil
+}

source/containerimage/pull.go

@@ -7,24 +7,18 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/diff"
 	containerderrdefs "github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/leases"
-	"github.com/containerd/containerd/platforms"
-	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/session"
-	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/errdefs"
-	"github.com/moby/buildkit/source"
-	srctypes "github.com/moby/buildkit/source/types"
 	"github.com/moby/buildkit/util/estargz"
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/moby/buildkit/util/imageutil"
@@ -39,164 +33,12 @@ import (
 	"github.com/pkg/errors"
 )
 
-// TODO: break apart containerd specifics like contentstore so the resolver
-// code can be used with any implementation
-
-type ResolverType int
-
-const (
-	ResolverTypeRegistry ResolverType = iota
-	ResolverTypeOCILayout
-)
-
-type SourceOpt struct {
-	Snapshotter   snapshot.Snapshotter
-	ContentStore  content.Store
-	Applier       diff.Applier
-	CacheAccessor cache.Accessor
-	ImageStore    images.Store // optional
-	RegistryHosts docker.RegistryHosts
-	ResolverType
-	LeaseManager leases.Manager
-}
-
-type resolveImageResult struct {
-	ref  string
-	dgst digest.Digest
-	dt   []byte
-}
-
-type Source struct {
-	SourceOpt
-	g flightcontrol.Group[*resolveImageResult]
-}
-
-var _ source.Source = &Source{}
-
-func NewSource(opt SourceOpt) (*Source, error) {
-	is := &Source{
-		SourceOpt: opt,
-	}
-
-	return is, nil
-}
-
-func (is *Source) ID() string {
-	if is.ResolverType == ResolverTypeOCILayout {
-		return srctypes.OCIScheme
-	}
-	return srctypes.DockerImageScheme
-}
-
-func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (string, digest.Digest, []byte, error) {
-	key := ref
-	if platform := opt.Platform; platform != nil {
-		key += platforms.Format(*platform)
-	}
-	var (
-		rm    source.ResolveMode
-		rslvr remotes.Resolver
-		err   error
-	)
-
-	switch is.ResolverType {
-	case ResolverTypeRegistry:
-		rm, err = source.ParseImageResolveMode(opt.ResolveMode)
-		if err != nil {
-			return "", "", nil, err
-		}
-		rslvr = resolver.DefaultPool.GetResolver(is.RegistryHosts, ref, "pull", sm, g).WithImageStore(is.ImageStore, rm)
-	case ResolverTypeOCILayout:
-		rm = source.ResolveModeForcePull
-		rslvr = getOCILayoutResolver(opt.Store, sm, g)
-	}
-	key += rm.String()
-	res, err := is.g.Do(ctx, key, func(ctx context.Context) (*resolveImageResult, error) {
-		newRef, dgst, dt, err := imageutil.Config(ctx, ref, rslvr, is.ContentStore, is.LeaseManager, opt.Platform, opt.SourcePolicies)
-		if err != nil {
-			return nil, err
-		}
-		return &resolveImageResult{dgst: dgst, dt: dt, ref: newRef}, nil
-	})
-	if err != nil {
-		return "", "", nil, err
-	}
-	return res.ref, res.dgst, res.dt, nil
-}
-
-func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, vtx solver.Vertex) (source.SourceInstance, error) {
-	var (
-		p          *puller
-		platform   = platforms.DefaultSpec()
-		pullerUtil *pull.Puller
-		mode       source.ResolveMode
-		recordType client.UsageRecordType
-		ref        reference.Spec
-		store      llb.ResolveImageConfigOptStore
-		layerLimit *int
-	)
-	switch is.ResolverType {
-	case ResolverTypeRegistry:
-		imageIdentifier, ok := id.(*source.ImageIdentifier)
-		if !ok {
-			return nil, errors.Errorf("invalid image identifier %v", id)
-		}
-
-		if imageIdentifier.Platform != nil {
-			platform = *imageIdentifier.Platform
-		}
-		mode = imageIdentifier.ResolveMode
-		recordType = imageIdentifier.RecordType
-		ref = imageIdentifier.Reference
-		layerLimit = imageIdentifier.LayerLimit
-	case ResolverTypeOCILayout:
-		ociIdentifier, ok := id.(*source.OCIIdentifier)
-		if !ok {
-			return nil, errors.Errorf("invalid OCI layout identifier %v", id)
-		}
-
-		if ociIdentifier.Platform != nil {
-			platform = *ociIdentifier.Platform
-		}
-		mode = source.ResolveModeForcePull // with OCI layout, we always just "pull"
-		store = llb.ResolveImageConfigOptStore{
-			SessionID: ociIdentifier.SessionID,
-			StoreID:   ociIdentifier.StoreID,
-		}
-		ref = ociIdentifier.Reference
-		layerLimit = ociIdentifier.LayerLimit
-	default:
-		return nil, errors.Errorf("unknown resolver type: %v", is.ResolverType)
-	}
-	pullerUtil = &pull.Puller{
-		ContentStore: is.ContentStore,
-		Platform:     platform,
-		Src:          ref,
-	}
-	p = &puller{
-		CacheAccessor:  is.CacheAccessor,
-		LeaseManager:   is.LeaseManager,
-		Puller:         pullerUtil,
-		RegistryHosts:  is.RegistryHosts,
-		ResolverType:   is.ResolverType,
-		ImageStore:     is.ImageStore,
-		Mode:           mode,
-		RecordType:     recordType,
-		Ref:            ref.String(),
-		SessionManager: sm,
-		vtx:            vtx,
-		store:          store,
-		layerLimit:     layerLimit,
-	}
-	return p, nil
-}
-
 type puller struct {
 	CacheAccessor  cache.Accessor
 	LeaseManager   leases.Manager
 	RegistryHosts  docker.RegistryHosts
 	ImageStore     images.Store
-	Mode           source.ResolveMode
+	Mode           resolver.ResolveMode
 	RecordType     client.UsageRecordType
 	Ref            string
 	SessionManager *session.Manager