monitor: breakpoint debugger on terminal and on IDEs (via DAP)

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

Author: ktock

build/build.go

@@ -664,7 +664,7 @@ func Build(ctx context.Context, nodes []builder.Node, opt map[string]Options, do
 	return BuildWithResultHandler(ctx, nodes, opt, docker, configDir, w, nil)
 }
 
-func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[string]Options, docker *dockerutil.Client, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext)) (resp map[string]*client.SolveResponse, err error) {
+func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[string]Options, docker *dockerutil.Client, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext) error) (resp map[string]*client.SolveResponse, err error) {
 	if len(nodes) == 0 {
 		return nil, errors.Errorf("driver required for build")
 	}
@@ -929,7 +929,9 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
 							if resultHandleFunc != nil {
 								resultCtx, err := NewResultContext(cc, so, res)
 								if err == nil {
-									resultHandleFunc(dp.driverIndex, resultCtx)
+									if err := resultHandleFunc(dp.driverIndex, resultCtx); err != nil {
+										return nil, err
+									}
 								} else {
 									logrus.Warnf("failed to record result: %s", err)
 								}

build/invoke.go

@@ -97,22 +97,22 @@ func (c *Container) markUnavailable() {
 	c.isUnavailable.Store(true)
 }
 
-func (c *Container) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
+func (c *Container) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser, signalCh <-chan syscall.Signal, resizeCh <-chan gateway.WinSize) error {
 	if isInit := c.initStarted.CompareAndSwap(false, true); isInit {
 		defer func() {
 			// container can't be used after init exits
 			c.markUnavailable()
 		}()
 	}
-	err := exec(ctx, c.resultCtx, cfg, c.container, stdin, stdout, stderr)
+	err := exec(ctx, c.resultCtx, cfg, c.container, stdin, stdout, stderr, signalCh, resizeCh)
 	if err != nil {
 		// Container becomes unavailable if one of the processes fails in it.
 		c.markUnavailable()
 	}
 	return err
 }
 
-func exec(ctx context.Context, resultCtx *ResultContext, cfg *controllerapi.InvokeConfig, ctr gateway.Container, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
+func exec(ctx context.Context, resultCtx *ResultContext, cfg *controllerapi.InvokeConfig, ctr gateway.Container, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser, signalCh <-chan syscall.Signal, resizeCh <-chan gateway.WinSize) error {
 	processCfg, err := resultCtx.getProcessConfig(cfg, stdin, stdout, stderr)
 	if err != nil {
 		return err
@@ -125,12 +125,24 @@ func exec(ctx context.Context, resultCtx *ResultContext, cfg *controllerapi.Invo
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 	go func() {
-		select {
-		case <-ctx.Done():
-			if err := proc.Signal(ctx, syscall.SIGKILL); err != nil {
-				logrus.Warnf("failed to kill process: %v", err)
+		for {
+			select {
+			case s := <-signalCh:
+				if err := proc.Signal(ctx, s); err != nil {
+					logrus.Warnf("failed to send signal %v %v", s, err)
+				}
+			case w := <-resizeCh:
+				if err := proc.Resize(ctx, w); err != nil {
+					logrus.Warnf("failed to resize %v: %v", w, err)
+				}
+			case <-ctx.Done():
+				if err := proc.Signal(ctx, syscall.SIGKILL); err != nil {
+					logrus.Warnf("failed to kill process: %v", err)
+				}
+				return
+			case <-doneCh:
+				return
 			}
-		case <-doneCh:
 		}
 	}()
 

build/result.go

@@ -5,11 +5,13 @@ import (
 	_ "crypto/sha256" // ensure digests can be computed
 	"encoding/json"
 	"io"
+	"path/filepath"
 	"sync"
 	"sync/atomic"
 
 	controllerapi "github.com/docker/buildx/controller/pb"
 	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	gateway "github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/solver/errdefs"
@@ -44,6 +46,17 @@ func getDefinition(ctx context.Context, res *gateway.Result) (*pb.Definition, er
 	return def.ToPB(), nil
 }
 
+func DefinitionFromResultContext(ctx context.Context, res *ResultContext) (*pb.Definition, error) {
+	if res.def != nil {
+		return res.def, nil
+	}
+	return nil, errors.Errorf("result context doesn't contain build definition")
+}
+
+func GetResultAtFromResultContext(ctx context.Context, resultCtx *ResultContext, target *pb.Definition, statusChan chan *client.SolveStatus) (*ResultContext, error) {
+	return getResultAt(ctx, resultCtx.client, resultCtx.solveOpt, target, statusChan)
+}
+
 func getResultAt(ctx context.Context, c *client.Client, solveOpt client.SolveOpt, target *pb.Definition, statusChan chan *client.SolveStatus) (*ResultContext, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
@@ -76,6 +89,7 @@ func getResultAt(ctx context.Context, c *client.Client, solveOpt client.SolveOpt
 		resultCtx := ResultContext{
 			client:   c,
 			solveOpt: solveOpt,
+			def:      target,
 		}
 		_, err := c.Build(context.Background(), solveOpt, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
 			ctx, cancel := context.WithCancel(ctx)
@@ -124,6 +138,7 @@ func getResultAt(ctx context.Context, c *client.Client, solveOpt client.SolveOpt
 type ResultContext struct {
 	client   *client.Client
 	res      *gateway.Result
+	def      *pb.Definition
 	solveOpt client.SolveOpt
 
 	solveErr *errdefs.SolveError
@@ -150,6 +165,10 @@ func (r *ResultContext) Done() {
 	})
 }
 
+func (r *ResultContext) SolveError() *errdefs.SolveError {
+	return r.solveErr
+}
+
 func (r *ResultContext) registerCleanup(f func()) {
 	r.cleanupsMu.Lock()
 	r.cleanups = append(r.cleanups, f)
@@ -177,6 +196,28 @@ func (r *ResultContext) getContainerConfig(ctx context.Context, c gateway.Client
 		}
 		containerCfg = *ccfg
 	}
+	if img := cfg.Image; img != "" {
+		def, err := llb.Image(img).Marshal(ctx)
+		if err != nil {
+			return containerCfg, err
+		}
+		r, err := c.Solve(ctx, gateway.SolveRequest{
+			Definition: def.ToPB(),
+		})
+		if err != nil {
+			return containerCfg, err
+		}
+		for i := range containerCfg.Mounts {
+			containerCfg.Mounts[i].Dest = filepath.Join(cfg.ResultMountPath, containerCfg.Mounts[i].Dest)
+		}
+		containerCfg.Mounts = append([]gateway.Mount{
+			{
+				Dest:      "/",
+				MountType: pb.MountType_BIND,
+				Ref:       r.Ref,
+			},
+		}, containerCfg.Mounts...)
+	}
 	return containerCfg, nil
 }
 

cmd/buildx/main.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/containerd/containerd/pkg/seed"
+	"github.com/containerd/containerd/pkg/seed" //nolint:staticcheck // Global math/rand seed is deprecated, but still used by external dependencies
 	"github.com/docker/buildx/commands"
 	"github.com/docker/buildx/version"
 	"github.com/docker/cli/cli"
@@ -28,6 +28,7 @@ import (
 )
 
 func init() {
+	//nolint:staticcheck // Global math/rand seed is deprecated, but still used by external dependencies
 	seed.WithTimeAndRand()
 	stack.SetVersionInfo(version.Version, version.Revision)
 }

commands/build.go

@@ -34,6 +34,7 @@ import (
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+	solverpb "github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/moby/buildkit/util/grpcerrors"
 	"github.com/pkg/errors"
@@ -510,6 +511,7 @@ func launchControllerAndRunBuild(dockerCli command.Cli, options buildOptions) er
 	opts = *optsP
 
 	var ref string
+	var def *solverpb.Definition
 	var retErr error
 	f := ioset.NewSingleForwarder()
 	f.SetReader(os.Stdin)
@@ -528,12 +530,20 @@ func launchControllerAndRunBuild(dockerCli command.Cli, options buildOptions) er
 			}
 		}
 
+		if options.invoke == "debug-step" {
+			// Special mode where we don't get the result but get only the build definition.
+			// In this mode, Build() doesn't perform the build therefore always fails.
+			// The error returned by Build() contains *pb.Definition wrapped with *controllererror.BuildError.
+			opts.Debug = true
+		}
+
 		var resp *client.SolveResponse
-		ref, resp, err = c.Build(ctx, opts, pr, os.Stdout, os.Stderr, progress)
+		ref, resp, def, err = c.Build(ctx, opts, pr, os.Stdout, os.Stderr, progress)
 		if err != nil {
 			var be *controllererrors.BuildError
 			if errors.As(err, &be) {
 				ref = be.Ref
+				def = be.Definition
 				retErr = err
 				// We can proceed to monitor
 			} else {
@@ -574,7 +584,7 @@ func launchControllerAndRunBuild(dockerCli command.Cli, options buildOptions) er
 			}
 			return errors.Errorf("failed to configure terminal: %v", err)
 		}
-		err = monitor.RunMonitor(ctx, ref, &opts, invokeConfig, c, progress, pr2, os.Stdout, os.Stderr)
+		err = monitor.RunMonitor(ctx, ref, def, &opts, invokeConfig, c, progress, pr2, os.Stdout, os.Stderr)
 		con.Reset()
 		if err := pw2.Close(); err != nil {
 			logrus.Debug("failed to close monitor stdin pipe reader")
@@ -587,12 +597,12 @@ func launchControllerAndRunBuild(dockerCli command.Cli, options buildOptions) er
 			logrus.Warnf("disconnect error: %v", err)
 		}
 	}
-	return nil
+	return retErr
 }
 
 func needsMonitor(invokeFlag string, retErr error) bool {
 	switch invokeFlag {
-	case "debug-shell":
+	case "debug-shell", "debug-step":
 		return true
 	case "on-error":
 		return retErr != nil
@@ -606,8 +616,7 @@ func parseInvokeConfig(invoke string) (cfg controllerapi.InvokeConfig, err error
 	switch invoke {
 	case "default", "debug-shell":
 		return cfg, nil
-	case "on-error":
-		// NOTE: we overwrite the command to run because the original one should fail on the failed step.
+	case "on-error", "debug-step":
 		cfg.Cmd = []string{"/bin/sh"}
 		return cfg, nil
 	}

commands/debug-shell.go

@@ -38,7 +38,7 @@ func debugShellCmd(dockerCli command.Cli) *cobra.Command {
 			if err := con.SetRaw(); err != nil {
 				return errors.Errorf("failed to configure terminal: %v", err)
 			}
-			err = monitor.RunMonitor(ctx, "", nil, controllerapi.InvokeConfig{
+			err = monitor.RunMonitor(ctx, "", nil, nil, controllerapi.InvokeConfig{
 				Tty: true,
 			}, c, progress, os.Stdin, os.Stdout, os.Stderr)
 			con.Reset()

commands/root.go

@@ -5,6 +5,7 @@ import (
 
 	imagetoolscmd "github.com/docker/buildx/commands/imagetools"
 	"github.com/docker/buildx/controller/remote"
+	"github.com/docker/buildx/monitor/dap"
 	"github.com/docker/buildx/util/logutil"
 	"github.com/docker/cli-docs-tool/annotation"
 	"github.com/docker/cli/cli"
@@ -91,6 +92,7 @@ func addCommands(cmd *cobra.Command, dockerCli command.Cli) {
 		remote.AddControllerCommands(cmd, dockerCli)
 		addDebugShellCommand(cmd, dockerCli)
 	}
+	dap.AddDAPCommands(cmd, dockerCli) // hidden command; we need it for emacs DAP support
 }
 
 func rootFlags(options *rootOptions, flags *pflag.FlagSet) {

controller/build/build.go

@@ -31,6 +31,7 @@ import (
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/solver/errdefs"
+	solverpb "github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/grpcerrors"
 	"github.com/moby/buildkit/util/progress/progressui"
 	"github.com/morikuni/aec"
@@ -174,15 +175,15 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in controllerapi.Build
 		return nil, nil, err
 	}
 
-	resp, res, err := buildTargets(ctx, dockerCli, b.NodeGroup, nodes, map[string]build.Options{defaultTargetName: opts}, progressMode, in.Opts.MetadataFile, statusChan)
+	resp, res, err := buildTargets(ctx, dockerCli, b.NodeGroup, nodes, map[string]build.Options{defaultTargetName: opts}, progressMode, in.Opts.MetadataFile, statusChan, in.Debug)
 	err = wrapBuildError(err, false)
 	if err != nil {
 		return nil, nil, err
 	}
 	return resp, res, nil
 }
 
-func buildTargets(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, nodes []builder.Node, opts map[string]build.Options, progressMode string, metadataFile string, statusChan chan *client.SolveStatus) (*client.SolveResponse, *build.ResultContext, error) {
+func buildTargets(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, nodes []builder.Node, opts map[string]build.Options, progressMode string, metadataFile string, statusChan chan *client.SolveStatus, debug bool) (*client.SolveResponse, *build.ResultContext, error) {
 	ctx2, cancel := context.WithCancel(context.TODO())
 	defer cancel()
 
@@ -197,12 +198,18 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, ng *store.NodeGrou
 	var res *build.ResultContext
 	var mu sync.Mutex
 	var idx int
-	resp, err := build.BuildWithResultHandler(ctx, nodes, opts, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), progress.Tee(printer, statusChan), func(driverIndex int, gotRes *build.ResultContext) {
+	pw, done := progress.Tee(printer, statusChan)
+	defer done()
+	resp, err := build.BuildWithResultHandler(ctx, nodes, opts, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), pw, func(driverIndex int, gotRes *build.ResultContext) error {
 		mu.Lock()
 		defer mu.Unlock()
 		if res == nil || driverIndex < idx {
 			idx, res = driverIndex, gotRes
 		}
+		if debug {
+			return errors.Errorf("debug mode")
+		}
+		return nil
 	})
 	err1 := printer.Wait()
 	if err == nil {
@@ -385,6 +392,7 @@ func controllerUlimitOpt2DockerUlimit(u *controllerapi.UlimitOpt) *dockeropts.Ul
 
 type ResultContextError struct {
 	ResultContext *build.ResultContext
+	Definition    *solverpb.Definition
 	error
 }
 
@@ -396,5 +404,10 @@ func wrapResultContext(wErr error, res *build.ResultContext) error {
 	if wErr == nil {
 		return nil
 	}
-	return &ResultContextError{ResultContext: res, error: wErr}
+	def, err := build.DefinitionFromResultContext(context.Background(), res)
+	if err != nil {
+		logrus.Warnf("failed to get definition from result: %v", err)
+		return wErr
+	}
+	return &ResultContextError{ResultContext: res, Definition: def, error: wErr}
 }

controller/control/controller.go

@@ -3,30 +3,38 @@ package control
 import (
 	"context"
 	"io"
+	"syscall"
 
 	"github.com/containerd/console"
 	controllerapi "github.com/docker/buildx/controller/pb"
 	"github.com/moby/buildkit/client"
+	solverpb "github.com/moby/buildkit/solver/pb"
 )
 
 type BuildxController interface {
-	Build(ctx context.Context, options controllerapi.BuildOptions, in io.ReadCloser, w io.Writer, out console.File, progressMode string) (ref string, resp *client.SolveResponse, err error)
+	Build(ctx context.Context, options controllerapi.BuildOptions, in io.ReadCloser, w io.Writer, out console.File, progressMode string) (ref string, resp *client.SolveResponse, def *solverpb.Definition, err error)
 	// Invoke starts an IO session into the specified process.
 	// If pid doesn't matche to any running processes, it starts a new process with the specified config.
 	// If there is no container running or InvokeConfig.Rollback is speicfied, the process will start in a newly created container.
 	// NOTE: If needed, in the future, we can split this API into three APIs (NewContainer, NewProcess and Attach).
-	Invoke(ctx context.Context, ref, pid string, options controllerapi.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error
+	Invoke(ctx context.Context, ref, pid string, options controllerapi.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser, signalCh <-chan syscall.Signal, resizeCh <-chan WinSize) error
 	Kill(ctx context.Context) error
 	Close() error
 	List(ctx context.Context) (refs []string, _ error)
 	Disconnect(ctx context.Context, ref string) error
 	ListProcesses(ctx context.Context, ref string) (infos []*controllerapi.ProcessInfo, retErr error)
 	DisconnectProcess(ctx context.Context, ref, pid string) error
 	Inspect(ctx context.Context, ref string) (*controllerapi.InspectResponse, error)
+	Continue(ctx context.Context, ref string, def *solverpb.Definition, w io.Writer, out console.File, progressMode string) error
 }
 
 type ControlOptions struct {
 	ServerConfig string
 	Root         string
 	Detach       bool
 }
+
+type WinSize struct {
+	Rows uint32
+	Cols uint32
+}