Skip to content

Commit 137cff7

Browse files
tonistiigitonistiigi
authored
Merge pull request moby#3530 from tonistiigi/history-api-fixes
fixes for history API
2 parents 5e520c7 + 216ee42 commit 137cff7

File tree

7 files changed

+254
-158
lines changed

7 files changed

+254
-158
lines changed

‎api/services/control/control.pb.go

Lines changed: 179 additions & 141 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/services/control/control.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ message BuildHistoryRecord {
206206
bool pinned = 14;
207207
int32 numCachedSteps = 15;
208208
int32 numTotalSteps = 16;
209+
int32 numCompletedSteps = 17;
209210
// TODO: tags
210211
// TODO: unclipped logs
211212
}

‎client/solve.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ type SolveOpt struct {
4949
SessionPreInitialized bool // TODO: refactor to better session syncing
5050
Internal bool
5151
SourcePolicy *spb.Policy
52+
Ref string
5253
}
5354

5455
type ExportEntry struct {
@@ -95,6 +96,9 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
9596
}
9697

9798
ref := identity.NewID()
99+
if opt.Ref != "" {
100+
ref = opt.Ref
101+
}
98102
eg, ctx := errgroup.WithContext(ctx)
99103

100104
statusContext, cancelStatus := context.WithCancel(context.Background())

‎cmd/buildctl/build.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/moby/buildkit/cmd/buildctl/build"
1717
bccommon "github.com/moby/buildkit/cmd/buildctl/common"
1818
gateway "github.com/moby/buildkit/frontend/gateway/client"
19+
"github.com/moby/buildkit/identity"
1920
"github.com/moby/buildkit/session"
2021
"github.com/moby/buildkit/session/auth/authprovider"
2122
"github.com/moby/buildkit/session/sshforward/sshprovider"
@@ -100,6 +101,10 @@ var buildCommand = cli.Command{
100101
Name: "source-policy-file",
101102
Usage: "Read source policy file from a JSON file",
102103
},
104+
cli.StringFlag{
105+
Name: "ref-file",
106+
Usage: "Write build ref to a file",
107+
},
103108
},
104109
}
105110

@@ -209,6 +214,8 @@ func buildAction(clicontext *cli.Context) error {
209214

210215
eg, ctx := errgroup.WithContext(bccommon.CommandContext(clicontext))
211216

217+
ref := identity.NewID()
218+
212219
solveOpt := client.SolveOpt{
213220
Exports: exports,
214221
// LocalDirs is set later
@@ -220,6 +227,7 @@ func buildAction(clicontext *cli.Context) error {
220227
Session: attachable,
221228
AllowedEntitlements: allowed,
222229
SourcePolicy: srcPol,
230+
Ref: ref,
223231
}
224232

225233
solveOpt.FrontendAttrs, err = build.ParseOpt(clicontext.StringSlice("opt"))
@@ -255,6 +263,13 @@ func buildAction(clicontext *cli.Context) error {
255263
}
256264
}
257265

266+
refFile := clicontext.String("ref-file")
267+
if refFile != "" {
268+
defer func() {
269+
continuity.AtomicWriteFile(refFile, []byte(ref), 0666)
270+
}()
271+
}
272+
258273
// not using shared context to not disrupt display but let is finish reporting errors
259274
pw, err := progresswriter.NewPrinter(context.TODO(), os.Stderr, clicontext.String("progress"))
260275
if err != nil {

‎control/control.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import (
4444
"golang.org/x/sync/errgroup"
4545
"google.golang.org/grpc"
4646
"google.golang.org/grpc/codes"
47+
"google.golang.org/grpc/metadata"
4748
"google.golang.org/grpc/status"
4849
)
4950

@@ -248,6 +249,9 @@ func (c *Controller) Export(ctx context.Context, req *tracev1.ExportTraceService
248249
}
249250

250251
func (c *Controller) ListenBuildHistory(req *controlapi.BuildHistoryRequest, srv controlapi.Control_ListenBuildHistoryServer) error {
252+
if err := sendTimestampHeader(srv); err != nil {
253+
return err
254+
}
251255
return c.history.Listen(srv.Context(), req, func(h *controlapi.BuildHistoryEvent) error {
252256
if err := srv.Send(h); err != nil {
253257
return err
@@ -426,6 +430,8 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
426430
}, llbsolver.ExporterRequest{
427431
Exporter: expi,
428432
CacheExporters: cacheExporters,
433+
Type: req.Exporter,
434+
Attrs: req.ExporterAttrs,
429435
}, req.Entitlements, procs, req.Internal, req.SourcePolicy)
430436
if err != nil {
431437
return nil, err
@@ -436,6 +442,9 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
436442
}
437443

438444
func (c *Controller) Status(req *controlapi.StatusRequest, stream controlapi.Control_StatusServer) error {
445+
if err := sendTimestampHeader(stream); err != nil {
446+
return err
447+
}
439448
ch := make(chan *client.SolveStatus, 8)
440449

441450
eg, ctx := errgroup.WithContext(stream.Context())
@@ -645,3 +654,9 @@ func (cs *roContentStore) Update(ctx context.Context, info content.Info, fieldpa
645654
func (cs *roContentStore) Abort(ctx context.Context, ref string) error {
646655
return errors.Errorf("read-only content store")
647656
}
657+
658+
const timestampKey = "buildkit-current-timestamp"
659+
660+
func sendTimestampHeader(srv grpc.ServerStream) error {
661+
return srv.SendHeader(metadata.Pairs(timestampKey, time.Now().Format(time.RFC3339Nano)))
662+
}

‎solver/llbsolver/history.go

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,10 @@ type HistoryQueue struct {
4545
}
4646

4747
type StatusImportResult struct {
48-
Descriptor ocispecs.Descriptor
49-
NumCachedSteps int
50-
NumTotalSteps int
48+
Descriptor ocispecs.Descriptor
49+
NumCachedSteps int
50+
NumCompletedSteps int
51+
NumTotalSteps int
5152
}
5253

5354
func NewHistoryQueue(opt HistoryQueueOpt) *HistoryQueue {
@@ -461,16 +462,23 @@ func (h *HistoryQueue) ImportStatus(ctx context.Context, ch chan *client.SolveSt
461462
}
462463
}()
463464

464-
vtxMap := make(map[digest.Digest]bool)
465+
type vtxInfo struct {
466+
cached bool
467+
completed bool
468+
}
469+
vtxMap := make(map[digest.Digest]*vtxInfo)
465470

466471
buf := make([]byte, 32*1024)
467472
for st := range ch {
468473
for _, vtx := range st.Vertexes {
469474
if _, ok := vtxMap[vtx.Digest]; !ok {
470-
vtxMap[vtx.Digest] = false
475+
vtxMap[vtx.Digest] = &vtxInfo{}
471476
}
472477
if vtx.Cached {
473-
vtxMap[vtx.Digest] = true
478+
vtxMap[vtx.Digest].cached = true
479+
}
480+
if vtx.Completed != nil {
481+
vtxMap[vtx.Digest].completed = true
474482
}
475483
}
476484

@@ -502,16 +510,21 @@ func (h *HistoryQueue) ImportStatus(ctx context.Context, ch chan *client.SolveSt
502510
}
503511

504512
numCached := 0
505-
for _, cached := range vtxMap {
506-
if cached {
513+
numCompleted := 0
514+
for _, info := range vtxMap {
515+
if info.cached {
507516
numCached++
508517
}
518+
if info.completed {
519+
numCompleted++
520+
}
509521
}
510522

511523
return &StatusImportResult{
512-
Descriptor: *desc,
513-
NumCachedSteps: numCached,
514-
NumTotalSteps: len(vtxMap),
524+
Descriptor: *desc,
525+
NumCachedSteps: numCached,
526+
NumCompletedSteps: numCompleted,
527+
NumTotalSteps: len(vtxMap),
515528
}, release, nil
516529
}
517530

‎solver/llbsolver/solver.go

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ const (
5151
)
5252

5353
type ExporterRequest struct {
54+
Type string
55+
Attrs map[string]string
5456
Exporter exporter.ExporterInstance
5557
CacheExporters []RemoteCacheExporter
5658
}
@@ -139,7 +141,7 @@ func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge {
139141
return s.bridge(b)
140142
}
141143

142-
func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend.SolveRequest, j *solver.Job) (func(*Result, exporter.DescriptorReference, error) error, error) {
144+
func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend.SolveRequest, exp ExporterRequest, j *solver.Job) (func(*Result, exporter.DescriptorReference, error) error, error) {
143145
var stopTrace func() []tracetest.SpanStub
144146

145147
if s := trace.SpanFromContext(ctx); s.SpanContext().IsValid() {
@@ -157,6 +159,14 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend
157159
FrontendAttrs: req.FrontendOpt,
158160
CreatedAt: &st,
159161
}
162+
163+
if exp.Type != "" {
164+
rec.Exporters = []*controlapi.Exporter{{
165+
Type: exp.Type,
166+
Attrs: exp.Attrs,
167+
}}
168+
}
169+
160170
if err := s.history.Update(ctx, &controlapi.BuildHistoryEvent{
161171
Type: controlapi.BuildHistoryEventType_STARTED,
162172
Record: rec,
@@ -177,6 +187,9 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend
177187
}
178188
}
179189

190+
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
191+
defer cancel()
192+
180193
var mu sync.Mutex
181194
ch := make(chan *client.SolveStatus)
182195
eg, ctx2 := errgroup.WithContext(ctx)
@@ -282,6 +295,7 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend
282295
MediaType: st.Descriptor.MediaType,
283296
}
284297
rec.NumCachedSteps = int32(st.NumCachedSteps)
298+
rec.NumCompletedSteps = int32(st.NumCompletedSteps)
285299
rec.NumTotalSteps = int32(st.NumTotalSteps)
286300
mu.Unlock()
287301
return nil
@@ -334,10 +348,6 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend
334348
}
335349
}
336350

337-
if err != nil {
338-
return err
339-
}
340-
341351
if stopTrace == nil {
342352
logrus.Warn("no trace recorder found, skipping")
343353
return err
@@ -414,7 +424,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
414424
if internal {
415425
defer j.CloseProgress()
416426
} else {
417-
rec, err1 := s.recordBuildHistory(ctx, id, req, j)
427+
rec, err1 := s.recordBuildHistory(ctx, id, req, exp, j)
418428
if err != nil {
419429
defer j.CloseProgress()
420430
return nil, err1

0 commit comments

Comments
 (0)