Skip to content

Commit 5026d95

Browse files
tonistiigitonistiigi
committed
gateway: pass executor with build and not access worker directly
Running interactive container APIs was done by giving the gateway implementation access to worker controller directly, but it should be passed with a build job instead. Signed-off-by: Tonis Tiigi <[email protected]> (cherry picked from commit 0971dffaab93d91e51af984b44c745b35b3c5b4d) (cherry picked from commit 564f884e7bb6db9c63e03c3b081ea71e15aa7980)
1 parent 7718bd5 commit 5026d95

File tree

13 files changed

+98
-39
lines changed

13 files changed

+98
-39
lines changed

‎cmd/buildkitd/main.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -677,8 +677,8 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err
677677
return nil, err
678678
}
679679
frontends := map[string]frontend.Frontend{}
680-
frontends["dockerfile.v0"] = forwarder.NewGatewayForwarder(wc, dockerfile.Build)
681-
frontends["gateway.v0"] = gateway.NewGatewayFrontend(wc)
680+
frontends["dockerfile.v0"] = forwarder.NewGatewayForwarder(wc.Infos(), dockerfile.Build)
681+
frontends["gateway.v0"] = gateway.NewGatewayFrontend(wc.Infos())
682682

683683
cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(cfg.Root, "cache.db"))
684684
if err != nil {

‎executor/executor.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ import (
66
"net"
77
"syscall"
88

9+
"github.com/containerd/containerd/mount"
10+
"github.com/docker/docker/pkg/idtools"
911
resourcestypes "github.com/moby/buildkit/executor/resources/types"
10-
"github.com/moby/buildkit/snapshot"
1112
"github.com/moby/buildkit/solver/pb"
1213
)
1314

@@ -28,8 +29,13 @@ type Meta struct {
2829
RemoveMountStubsRecursive bool
2930
}
3031

32+
type MountableRef interface {
33+
Mount() ([]mount.Mount, func() error, error)
34+
IdentityMapping() *idtools.IdentityMapping
35+
}
36+
3137
type Mountable interface {
32-
Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error)
38+
Mount(ctx context.Context, readonly bool) (MountableRef, error)
3339
}
3440

3541
type Mount struct {

‎frontend/frontend.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
"github.com/moby/buildkit/client/llb"
7+
"github.com/moby/buildkit/executor"
78
gw "github.com/moby/buildkit/frontend/gateway/client"
89
"github.com/moby/buildkit/session"
910
"github.com/moby/buildkit/solver"
@@ -17,7 +18,7 @@ type Result = result.Result[solver.ResultProxy]
1718
type Attestation = result.Attestation[solver.ResultProxy]
1819

1920
type Frontend interface {
20-
Solve(ctx context.Context, llb FrontendLLBBridge, opt map[string]string, inputs map[string]*pb.Definition, sid string, sm *session.Manager) (*Result, error)
21+
Solve(ctx context.Context, llb FrontendLLBBridge, exec executor.Executor, opt map[string]string, inputs map[string]*pb.Definition, sid string, sm *session.Manager) (*Result, error)
2122
}
2223

2324
type FrontendLLBBridge interface {

‎frontend/gateway/container/container.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ type Mount struct {
4747
WorkerRef *worker.WorkerRef
4848
}
4949

50-
func NewContainer(ctx context.Context, w worker.Worker, sm *session.Manager, g session.Group, req NewContainerRequest) (client.Container, error) {
50+
func NewContainer(ctx context.Context, cm cache.Manager, exec executor.Executor, sm *session.Manager, g session.Group, req NewContainerRequest) (client.Container, error) {
5151
ctx, cancel := context.WithCancel(ctx)
5252
eg, ctx := errgroup.WithContext(ctx)
5353
platform := opspb.Platform{
@@ -63,7 +63,7 @@ func NewContainer(ctx context.Context, w worker.Worker, sm *session.Manager, g s
6363
hostname: req.Hostname,
6464
extraHosts: req.ExtraHosts,
6565
platform: platform,
66-
executor: w.Executor(),
66+
executor: exec,
6767
sm: sm,
6868
group: g,
6969
errGroup: eg,
@@ -86,9 +86,8 @@ func NewContainer(ctx context.Context, w worker.Worker, sm *session.Manager, g s
8686
}
8787

8888
name := fmt.Sprintf("container %s", req.ContainerID)
89-
mm := mounts.NewMountManager(name, w.CacheManager(), sm)
90-
p, err := PrepareMounts(ctx, mm, w.CacheManager(), g, "", mnts, refs, func(m *opspb.Mount, ref cache.ImmutableRef) (cache.MutableRef, error) {
91-
cm := w.CacheManager()
89+
mm := mounts.NewMountManager(name, cm, sm)
90+
p, err := PrepareMounts(ctx, mm, cm, g, "", mnts, refs, func(m *opspb.Mount, ref cache.ImmutableRef) (cache.MutableRef, error) {
9291
if m.Input != opspb.Empty {
9392
cm = refs[m.Input].Worker.CacheManager()
9493
}

‎frontend/gateway/forwarder/forward.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66

77
cacheutil "github.com/moby/buildkit/cache/util"
88
"github.com/moby/buildkit/client/llb"
9+
"github.com/moby/buildkit/executor"
910
"github.com/moby/buildkit/frontend"
1011
"github.com/moby/buildkit/frontend/gateway/client"
1112
"github.com/moby/buildkit/frontend/gateway/container"
@@ -26,7 +27,7 @@ import (
2627
"golang.org/x/sync/errgroup"
2728
)
2829

29-
func LLBBridgeToGatewayClient(ctx context.Context, llbBridge frontend.FrontendLLBBridge, opts map[string]string, inputs map[string]*opspb.Definition, w worker.Infos, sid string, sm *session.Manager) (*BridgeClient, error) {
30+
func LLBBridgeToGatewayClient(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, opts map[string]string, inputs map[string]*opspb.Definition, w worker.Infos, sid string, sm *session.Manager) (*BridgeClient, error) {
3031
bc := &BridgeClient{
3132
opts: opts,
3233
inputs: inputs,
@@ -35,6 +36,7 @@ func LLBBridgeToGatewayClient(ctx context.Context, llbBridge frontend.FrontendLL
3536
sm: sm,
3637
workers: w,
3738
workerRefByID: make(map[string]*worker.WorkerRef),
39+
executor: exec,
3840
}
3941
bc.buildOpts = bc.loadBuildOpts()
4042
return bc, nil
@@ -52,6 +54,7 @@ type BridgeClient struct {
5254
workerRefByID map[string]*worker.WorkerRef
5355
buildOpts client.BuildOpts
5456
ctrs []client.Container
57+
executor executor.Executor
5558
}
5659

5760
func (c *BridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*client.Result, error) {
@@ -293,13 +296,13 @@ func (c *BridgeClient) NewContainer(ctx context.Context, req client.NewContainer
293296
return nil, err
294297
}
295298

296-
w, err := c.workers.GetDefault()
299+
cm, err := c.workers.DefaultCacheManager()
297300
if err != nil {
298301
return nil, err
299302
}
300303

301304
group := session.NewGroup(c.sid)
302-
ctr, err := container.NewContainer(ctx, w, c.sm, group, ctrReq)
305+
ctr, err := container.NewContainer(ctx, cm, c.executor, c.sm, group, ctrReq)
303306
if err != nil {
304307
return nil, err
305308
}

‎frontend/gateway/forwarder/frontend.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package forwarder
33
import (
44
"context"
55

6+
"github.com/moby/buildkit/executor"
67
"github.com/moby/buildkit/frontend"
78
"github.com/moby/buildkit/frontend/gateway/client"
89
"github.com/moby/buildkit/session"
@@ -22,8 +23,8 @@ type GatewayForwarder struct {
2223
f client.BuildFunc
2324
}
2425

25-
func (gf *GatewayForwarder) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, opts map[string]string, inputs map[string]*pb.Definition, sid string, sm *session.Manager) (retRes *frontend.Result, retErr error) {
26-
c, err := LLBBridgeToGatewayClient(ctx, llbBridge, opts, inputs, gf.workers, sid, sm)
26+
func (gf *GatewayForwarder) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, opts map[string]string, inputs map[string]*pb.Definition, sid string, sm *session.Manager) (retRes *frontend.Result, retErr error) {
27+
c, err := LLBBridgeToGatewayClient(ctx, llbBridge, exec, opts, inputs, gf.workers, sid, sm)
2728
if err != nil {
2829
return nil, err
2930
}

‎frontend/gateway/gateway.go

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ func filterPrefix(opts map[string]string, pfx string) map[string]string {
8686
return m
8787
}
8888

89-
func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, opts map[string]string, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*frontend.Result, error) {
89+
func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, opts map[string]string, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*frontend.Result, error) {
9090
source, ok := opts[keySource]
9191
if !ok {
9292
return nil, errors.Errorf("no source specified for gateway")
@@ -141,7 +141,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
141141
}
142142
}
143143
} else {
144-
c, err := forwarder.LLBBridgeToGatewayClient(ctx, llbBridge, opts, inputs, gf.workers, sid, sm)
144+
c, err := forwarder.LLBBridgeToGatewayClient(ctx, llbBridge, exec, opts, inputs, gf.workers, sid, sm)
145145
if err != nil {
146146
return nil, err
147147
}
@@ -281,18 +281,13 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
281281
}
282282
}
283283

284-
lbf, ctx, err := serveLLBBridgeForwarder(ctx, llbBridge, gf.workers, inputs, sid, sm)
284+
lbf, ctx, err := serveLLBBridgeForwarder(ctx, llbBridge, exec, gf.workers, inputs, sid, sm)
285285
defer lbf.conn.Close() //nolint
286286
if err != nil {
287287
return nil, err
288288
}
289289
defer lbf.Discard()
290290

291-
w, err := gf.workers.GetDefault()
292-
if err != nil {
293-
return nil, err
294-
}
295-
296291
mdmnt, release, err := metadataMount(frontendDef)
297292
if err != nil {
298293
return nil, err
@@ -305,7 +300,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
305300
mnts = append(mnts, *mdmnt)
306301
}
307302

308-
_, err = w.Executor().Run(ctx, "", container.MountWithSession(rootFS, session.NewGroup(sid)), mnts, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil)
303+
_, err = exec.Run(ctx, "", container.MountWithSession(rootFS, session.NewGroup(sid)), mnts, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil)
309304
if err != nil {
310305
if errdefs.IsCanceled(ctx, err) && lbf.isErrServerClosed {
311306
err = errors.Errorf("frontend grpc server closed unexpectedly")
@@ -434,11 +429,11 @@ func (lbf *llbBridgeForwarder) Result() (*frontend.Result, error) {
434429
return lbf.result, nil
435430
}
436431

437-
func NewBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) LLBBridgeForwarder {
438-
return newBridgeForwarder(ctx, llbBridge, workers, inputs, sid, sm)
432+
func NewBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) LLBBridgeForwarder {
433+
return newBridgeForwarder(ctx, llbBridge, exec, workers, inputs, sid, sm)
439434
}
440435

441-
func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) *llbBridgeForwarder {
436+
func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) *llbBridgeForwarder {
442437
lbf := &llbBridgeForwarder{
443438
callCtx: ctx,
444439
llbBridge: llbBridge,
@@ -451,13 +446,14 @@ func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg
451446
sid: sid,
452447
sm: sm,
453448
ctrs: map[string]gwclient.Container{},
449+
executor: exec,
454450
}
455451
return lbf
456452
}
457453

458-
func serveLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*llbBridgeForwarder, context.Context, error) {
454+
func serveLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*llbBridgeForwarder, context.Context, error) {
459455
ctx, cancel := context.WithCancel(ctx)
460-
lbf := newBridgeForwarder(ctx, llbBridge, workers, inputs, sid, sm)
456+
lbf := newBridgeForwarder(ctx, llbBridge, exec, workers, inputs, sid, sm)
461457
server := grpc.NewServer(grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor))
462458
grpc_health_v1.RegisterHealthServer(server, health.NewServer())
463459
pb.RegisterLLBBridgeServer(server, lbf)
@@ -552,6 +548,7 @@ type llbBridgeForwarder struct {
552548
isErrServerClosed bool
553549
sid string
554550
sm *session.Manager
551+
executor executor.Executor
555552
*pipe
556553
ctrs map[string]gwclient.Container
557554
ctrsMu sync.Mutex
@@ -1042,7 +1039,7 @@ func (lbf *llbBridgeForwarder) NewContainer(ctx context.Context, in *pb.NewConta
10421039
// and we want the context to live for the duration of the container.
10431040
group := session.NewGroup(lbf.sid)
10441041

1045-
w, err := lbf.workers.GetDefault()
1042+
cm, err := lbf.workers.DefaultCacheManager()
10461043
if err != nil {
10471044
return nil, stack.Enable(err)
10481045
}
@@ -1052,7 +1049,7 @@ func (lbf *llbBridgeForwarder) NewContainer(ctx context.Context, in *pb.NewConta
10521049
return nil, stack.Enable(err)
10531050
}
10541051

1055-
ctr, err := container.NewContainer(context.Background(), w, lbf.sm, group, ctrReq)
1052+
ctr, err := container.NewContainer(context.Background(), cm, lbf.executor, lbf.sm, group, ctrReq)
10561053
if err != nil {
10571054
return nil, stack.Enable(err)
10581055
}

‎snapshot/snapshotter.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,11 @@ import (
1010
"github.com/containerd/containerd/pkg/userns"
1111
"github.com/containerd/containerd/snapshots"
1212
"github.com/docker/docker/pkg/idtools"
13+
"github.com/moby/buildkit/executor"
1314
"github.com/pkg/errors"
1415
)
1516

16-
type Mountable interface {
17-
// ID() string
18-
Mount() ([]mount.Mount, func() error, error)
19-
IdentityMapping() *idtools.IdentityMapping
20-
}
17+
type Mountable = executor.MountableRef
2118

2219
// Snapshotter defines interface that any snapshot implementation should satisfy
2320
type Snapshotter interface {

‎solver/llbsolver/bridge.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import (
1111
"github.com/moby/buildkit/cache/remotecache"
1212
"github.com/moby/buildkit/client"
1313
"github.com/moby/buildkit/client/llb"
14+
"github.com/moby/buildkit/executor"
15+
resourcestypes "github.com/moby/buildkit/executor/resources/types"
1416
"github.com/moby/buildkit/frontend"
1517
gw "github.com/moby/buildkit/frontend/gateway/client"
1618
"github.com/moby/buildkit/identity"
@@ -39,6 +41,10 @@ type llbBridge struct {
3941
cms map[string]solver.CacheManager
4042
cmsMu sync.Mutex
4143
sm *session.Manager
44+
45+
executorOnce sync.Once
46+
executorErr error
47+
executor executor.Executor
4248
}
4349

4450
func (b *llbBridge) Warn(ctx context.Context, dgst digest.Digest, msg string, opts frontend.WarnOpts) error {
@@ -159,6 +165,32 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp
159165
return res, nil
160166
}
161167

168+
func (b *llbBridge) Run(ctx context.Context, id string, rootfs executor.Mount, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (resourcestypes.Recorder, error) {
169+
if err := b.loadExecutor(); err != nil {
170+
return nil, err
171+
}
172+
return b.executor.Run(ctx, id, rootfs, mounts, process, started)
173+
}
174+
175+
func (b *llbBridge) Exec(ctx context.Context, id string, process executor.ProcessInfo) error {
176+
if err := b.loadExecutor(); err != nil {
177+
return err
178+
}
179+
return b.executor.Exec(ctx, id, process)
180+
}
181+
182+
func (b *llbBridge) loadExecutor() error {
183+
b.executorOnce.Do(func() {
184+
w, err := b.resolveWorker()
185+
if err != nil {
186+
b.executorErr = err
187+
return
188+
}
189+
b.executor = w.Executor()
190+
})
191+
return b.executorErr
192+
}
193+
162194
type resultProxy struct {
163195
id string
164196
b *provenanceBridge

‎solver/llbsolver/provenance.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest,
165165
return nil, errors.Errorf("invalid frontend: %s", req.Frontend)
166166
}
167167
wb := &provenanceBridge{llbBridge: b.llbBridge, req: &req}
168-
res, err = f.Solve(ctx, wb, req.FrontendOpt, req.FrontendInputs, sid, b.llbBridge.sm)
168+
res, err = f.Solve(ctx, wb, b.llbBridge, req.FrontendOpt, req.FrontendInputs, sid, b.llbBridge.sm)
169169
if err != nil {
170170
return nil, err
171171
}

0 commit comments

Comments
 (0)