Skip to content

Commit ba96bfb

Browse files
tonistiigitonistiigi
authored
Merge pull request moby#3949 from tonistiigi/flightcontrol-generics
move flightcontrol to use generics
2 parents 5d078f9 + 8ffc03b commit ba96bfb

File tree

19 files changed

+180
-214
lines changed

19 files changed

+180
-214
lines changed

‎cache/blobs.go

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ import (
2222
"golang.org/x/sync/errgroup"
2323
)
2424

25-
var g flightcontrol.Group
25+
var g flightcontrol.Group[struct{}]
26+
var gFileList flightcontrol.Group[[]string]
2627

2728
const containerdUncompressed = "containerd.io/uncompressed"
2829

@@ -86,12 +87,12 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
8687

8788
if _, ok := filter[sr.ID()]; ok {
8889
eg.Go(func() error {
89-
_, err := g.Do(ctx, fmt.Sprintf("%s-%t", sr.ID(), createIfNeeded), func(ctx context.Context) (interface{}, error) {
90+
_, err := g.Do(ctx, fmt.Sprintf("%s-%t", sr.ID(), createIfNeeded), func(ctx context.Context) (struct{}, error) {
9091
if sr.getBlob() != "" {
91-
return nil, nil
92+
return struct{}{}, nil
9293
}
9394
if !createIfNeeded {
94-
return nil, errors.WithStack(ErrNoBlobs)
95+
return struct{}{}, errors.WithStack(ErrNoBlobs)
9596
}
9697

9798
compressorFunc, finalize := comp.Type.Compress(ctx, comp)
@@ -108,12 +109,12 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
108109
if lowerRef != nil {
109110
m, err := lowerRef.Mount(ctx, true, s)
110111
if err != nil {
111-
return nil, err
112+
return struct{}{}, err
112113
}
113114
var release func() error
114115
lower, release, err = m.Mount()
115116
if err != nil {
116-
return nil, err
117+
return struct{}{}, err
117118
}
118119
if release != nil {
119120
defer release()
@@ -131,12 +132,12 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
131132
if upperRef != nil {
132133
m, err := upperRef.Mount(ctx, true, s)
133134
if err != nil {
134-
return nil, err
135+
return struct{}{}, err
135136
}
136137
var release func() error
137138
upper, release, err = m.Mount()
138139
if err != nil {
139-
return nil, err
140+
return struct{}{}, err
140141
}
141142
if release != nil {
142143
defer release()
@@ -151,7 +152,7 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
151152
if forceOvlStr := os.Getenv("BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF"); forceOvlStr != "" && sr.kind() != Diff {
152153
enableOverlay, err = strconv.ParseBool(forceOvlStr)
153154
if err != nil {
154-
return nil, errors.Wrapf(err, "invalid boolean in BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF")
155+
return struct{}{}, errors.Wrapf(err, "invalid boolean in BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF")
155156
}
156157
fallback = false // prohibit fallback on debug
157158
} else if !isTypeWindows(sr) {
@@ -173,10 +174,10 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
173174
if !ok || err != nil {
174175
if !fallback {
175176
if !ok {
176-
return nil, errors.Errorf("overlay mounts not detected (lower=%+v,upper=%+v)", lower, upper)
177+
return struct{}{}, errors.Errorf("overlay mounts not detected (lower=%+v,upper=%+v)", lower, upper)
177178
}
178179
if err != nil {
179-
return nil, errors.Wrapf(err, "failed to compute overlay diff")
180+
return struct{}{}, errors.Wrapf(err, "failed to compute overlay diff")
180181
}
181182
}
182183
if logWarnOnErr {
@@ -209,7 +210,7 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
209210
diff.WithCompressor(compressorFunc),
210211
)
211212
if err != nil {
212-
return nil, err
213+
return struct{}{}, err
213214
}
214215
}
215216

@@ -219,29 +220,29 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
219220
if finalize != nil {
220221
a, err := finalize(ctx, sr.cm.ContentStore)
221222
if err != nil {
222-
return nil, errors.Wrapf(err, "failed to finalize compression")
223+
return struct{}{}, errors.Wrapf(err, "failed to finalize compression")
223224
}
224225
for k, v := range a {
225226
desc.Annotations[k] = v
226227
}
227228
}
228229
info, err := sr.cm.ContentStore.Info(ctx, desc.Digest)
229230
if err != nil {
230-
return nil, err
231+
return struct{}{}, err
231232
}
232233

233234
if diffID, ok := info.Labels[containerdUncompressed]; ok {
234235
desc.Annotations[containerdUncompressed] = diffID
235236
} else if mediaType == ocispecs.MediaTypeImageLayer {
236237
desc.Annotations[containerdUncompressed] = desc.Digest.String()
237238
} else {
238-
return nil, errors.Errorf("unknown layer compression type")
239+
return struct{}{}, errors.Errorf("unknown layer compression type")
239240
}
240241

241242
if err := sr.setBlob(ctx, desc); err != nil {
242-
return nil, err
243+
return struct{}{}, err
243244
}
244-
return nil, nil
245+
return struct{}{}, nil
245246
})
246247
if err != nil {
247248
return err
@@ -415,29 +416,29 @@ func isTypeWindows(sr *immutableRef) bool {
415416

416417
// ensureCompression ensures the specified ref has the blob of the specified compression Type.
417418
func ensureCompression(ctx context.Context, ref *immutableRef, comp compression.Config, s session.Group) error {
418-
_, err := g.Do(ctx, fmt.Sprintf("%s-%s", ref.ID(), comp.Type), func(ctx context.Context) (interface{}, error) {
419+
_, err := g.Do(ctx, fmt.Sprintf("ensureComp-%s-%s", ref.ID(), comp.Type), func(ctx context.Context) (struct{}, error) {
419420
desc, err := ref.ociDesc(ctx, ref.descHandlers, true)
420421
if err != nil {
421-
return nil, err
422+
return struct{}{}, err
422423
}
423424

424425
// Resolve converters
425426
layerConvertFunc, err := getConverter(ctx, ref.cm.ContentStore, desc, comp)
426427
if err != nil {
427-
return nil, err
428+
return struct{}{}, err
428429
} else if layerConvertFunc == nil {
429430
if isLazy, err := ref.isLazy(ctx); err != nil {
430-
return nil, err
431+
return struct{}{}, err
431432
} else if isLazy {
432433
// This ref can be used as the specified compressionType. Keep it lazy.
433-
return nil, nil
434+
return struct{}{}, nil
434435
}
435-
return nil, ref.linkBlob(ctx, desc)
436+
return struct{}{}, ref.linkBlob(ctx, desc)
436437
}
437438

438439
// First, lookup local content store
439440
if _, err := ref.getBlobWithCompression(ctx, comp.Type); err == nil {
440-
return nil, nil // found the compression variant. no need to convert.
441+
return struct{}{}, nil // found the compression variant. no need to convert.
441442
}
442443

443444
// Convert layer compression type
@@ -447,18 +448,18 @@ func ensureCompression(ctx context.Context, ref *immutableRef, comp compression.
447448
dh: ref.descHandlers[desc.Digest],
448449
session: s,
449450
}).Unlazy(ctx); err != nil {
450-
return nil, err
451+
return struct{}{}, err
451452
}
452453
newDesc, err := layerConvertFunc(ctx, ref.cm.ContentStore, desc)
453454
if err != nil {
454-
return nil, errors.Wrapf(err, "failed to convert")
455+
return struct{}{}, errors.Wrapf(err, "failed to convert")
455456
}
456457

457458
// Start to track converted layer
458459
if err := ref.linkBlob(ctx, *newDesc); err != nil {
459-
return nil, errors.Wrapf(err, "failed to add compression blob")
460+
return struct{}{}, errors.Wrapf(err, "failed to add compression blob")
460461
}
461-
return nil, nil
462+
return struct{}{}, nil
462463
})
463464
return err
464465
}

‎cache/filelist.go

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ const keyFileList = "filelist"
2020
// are in the tar stream (AUFS whiteout format). If the reference does not have a
2121
// a blob associated with it, the list is empty.
2222
func (sr *immutableRef) FileList(ctx context.Context, s session.Group) ([]string, error) {
23-
res, err := g.Do(ctx, fmt.Sprintf("filelist-%s", sr.ID()), func(ctx context.Context) (interface{}, error) {
23+
return gFileList.Do(ctx, fmt.Sprintf("filelist-%s", sr.ID()), func(ctx context.Context) ([]string, error) {
2424
dt, err := sr.GetExternal(keyFileList)
2525
if err == nil && dt != nil {
2626
var files []string
@@ -80,11 +80,4 @@ func (sr *immutableRef) FileList(ctx context.Context, s session.Group) ([]string
8080
}
8181
return files, nil
8282
})
83-
if err != nil {
84-
return nil, err
85-
}
86-
if res == nil {
87-
return nil, nil
88-
}
89-
return res.([]string), nil
9083
}

‎cache/manager.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ type cacheManager struct {
9393
mountPool sharableMountPool
9494

9595
muPrune sync.Mutex // make sure parallel prune is not allowed so there will not be inconsistent results
96-
unlazyG flightcontrol.Group
96+
unlazyG flightcontrol.Group[struct{}]
9797
}
9898

9999
func NewManager(opt ManagerOpt) (Manager, error) {

‎cache/refs.go

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ type cacheRecord struct {
9090

9191
mountCache snapshot.Mountable
9292

93-
sizeG flightcontrol.Group
93+
sizeG flightcontrol.Group[int64]
9494

9595
// these are filled if multiple refs point to same data
9696
equalMutable *mutableRef
@@ -325,7 +325,7 @@ func (cr *cacheRecord) viewSnapshotID() string {
325325

326326
func (cr *cacheRecord) size(ctx context.Context) (int64, error) {
327327
// this expects that usage() is implemented lazily
328-
s, err := cr.sizeG.Do(ctx, cr.ID(), func(ctx context.Context) (interface{}, error) {
328+
return cr.sizeG.Do(ctx, cr.ID(), func(ctx context.Context) (int64, error) {
329329
cr.mu.Lock()
330330
s := cr.getSize()
331331
if s != sizeUnknown {
@@ -346,7 +346,7 @@ func (cr *cacheRecord) size(ctx context.Context) (int64, error) {
346346
isDead := cr.isDead()
347347
cr.mu.Unlock()
348348
if isDead {
349-
return int64(0), nil
349+
return 0, nil
350350
}
351351
if !errors.Is(err, errdefs.ErrNotFound) {
352352
return s, errors.Wrapf(err, "failed to get usage for %s", cr.ID())
@@ -379,10 +379,6 @@ func (cr *cacheRecord) size(ctx context.Context) (int64, error) {
379379
cr.mu.Unlock()
380380
return usage.Size, nil
381381
})
382-
if err != nil {
383-
return 0, err
384-
}
385-
return s.(int64), nil
386382
}
387383

388384
// caller must hold cr.mu
@@ -1057,7 +1053,7 @@ func (sr *immutableRef) withRemoteSnapshotLabelsStargzMode(ctx context.Context,
10571053
}
10581054

10591055
func (sr *immutableRef) prepareRemoteSnapshotsStargzMode(ctx context.Context, s session.Group) error {
1060-
_, err := sr.sizeG.Do(ctx, sr.ID()+"-prepare-remote-snapshot", func(ctx context.Context) (_ interface{}, rerr error) {
1056+
_, err := g.Do(ctx, sr.ID()+"-prepare-remote-snapshot", func(ctx context.Context) (_ struct{}, rerr error) {
10611057
dhs := sr.descHandlers
10621058
for _, r := range sr.layerChain() {
10631059
r := r
@@ -1069,7 +1065,7 @@ func (sr *immutableRef) prepareRemoteSnapshotsStargzMode(ctx context.Context, s
10691065
dh := dhs[digest.Digest(r.getBlob())]
10701066
if dh == nil {
10711067
// We cannot prepare remote snapshots without descHandler.
1072-
return nil, nil
1068+
return struct{}{}, nil
10731069
}
10741070

10751071
// tmpLabels contains dh.SnapshotLabels + session IDs. All keys contain
@@ -1121,7 +1117,7 @@ func (sr *immutableRef) prepareRemoteSnapshotsStargzMode(ctx context.Context, s
11211117
break
11221118
}
11231119

1124-
return nil, nil
1120+
return struct{}{}, nil
11251121
})
11261122
return err
11271123
}
@@ -1144,18 +1140,18 @@ func makeTmpLabelsStargzMode(labels map[string]string, s session.Group) (fields
11441140
}
11451141

11461142
func (sr *immutableRef) unlazy(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, topLevel bool) error {
1147-
_, err := sr.sizeG.Do(ctx, sr.ID()+"-unlazy", func(ctx context.Context) (_ interface{}, rerr error) {
1143+
_, err := g.Do(ctx, sr.ID()+"-unlazy", func(ctx context.Context) (_ struct{}, rerr error) {
11481144
if _, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID()); err == nil {
1149-
return nil, nil
1145+
return struct{}{}, nil
11501146
}
11511147

11521148
switch sr.kind() {
11531149
case Merge, Diff:
1154-
return nil, sr.unlazyDiffMerge(ctx, dhs, pg, s, topLevel)
1150+
return struct{}{}, sr.unlazyDiffMerge(ctx, dhs, pg, s, topLevel)
11551151
case Layer, BaseLayer:
1156-
return nil, sr.unlazyLayer(ctx, dhs, pg, s)
1152+
return struct{}{}, sr.unlazyLayer(ctx, dhs, pg, s)
11571153
}
1158-
return nil, nil
1154+
return struct{}{}, nil
11591155
})
11601156
return err
11611157
}

‎cache/remote.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -305,11 +305,11 @@ func (p lazyRefProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor)
305305
}
306306

307307
func (p lazyRefProvider) Unlazy(ctx context.Context) error {
308-
_, err := p.ref.cm.unlazyG.Do(ctx, string(p.desc.Digest), func(ctx context.Context) (_ interface{}, rerr error) {
308+
_, err := p.ref.cm.unlazyG.Do(ctx, string(p.desc.Digest), func(ctx context.Context) (_ struct{}, rerr error) {
309309
if isLazy, err := p.ref.isLazy(ctx); err != nil {
310-
return nil, err
310+
return struct{}{}, err
311311
} else if !isLazy {
312-
return nil, nil
312+
return struct{}{}, nil
313313
}
314314
defer func() {
315315
if rerr == nil {
@@ -320,7 +320,7 @@ func (p lazyRefProvider) Unlazy(ctx context.Context) error {
320320
if p.dh == nil {
321321
// shouldn't happen, if you have a lazy immutable ref it already should be validated
322322
// that descriptor handlers exist for it
323-
return nil, errors.New("unexpected nil descriptor handler")
323+
return struct{}{}, errors.New("unexpected nil descriptor handler")
324324
}
325325

326326
if p.dh.Progress != nil {
@@ -337,20 +337,20 @@ func (p lazyRefProvider) Unlazy(ctx context.Context) error {
337337
Manager: p.ref.cm.ContentStore,
338338
}, p.desc, p.dh.Ref, logs.LoggerFromContext(ctx))
339339
if err != nil {
340-
return nil, err
340+
return struct{}{}, err
341341
}
342342

343343
if imageRefs := p.ref.getImageRefs(); len(imageRefs) > 0 {
344344
// just use the first image ref, it's arbitrary
345345
imageRef := imageRefs[0]
346346
if p.ref.GetDescription() == "" {
347347
if err := p.ref.SetDescription("pulled from " + imageRef); err != nil {
348-
return nil, err
348+
return struct{}{}, err
349349
}
350350
}
351351
}
352352

353-
return nil, nil
353+
return struct{}{}, nil
354354
})
355355
return err
356356
}

‎client/llb/async.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ type asyncState struct {
1515
target State
1616
set bool
1717
err error
18-
g flightcontrol.Group
18+
g flightcontrol.Group[State]
1919
}
2020

2121
func (as *asyncState) Output() Output {
@@ -53,7 +53,7 @@ func (as *asyncState) ToInput(ctx context.Context, c *Constraints) (*pb.Input, e
5353
}
5454

5555
func (as *asyncState) Do(ctx context.Context, c *Constraints) error {
56-
_, err := as.g.Do(ctx, "", func(ctx context.Context) (interface{}, error) {
56+
_, err := as.g.Do(ctx, "", func(ctx context.Context) (State, error) {
5757
if as.set {
5858
return as.target, as.err
5959
}

‎executor/oci/hosts.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ func GetHostsFile(ctx context.Context, stateDir string, extraHosts []executor.Ho
2020
return makeHostsFile(stateDir, extraHosts, idmap, hostname)
2121
}
2222

23-
_, err := g.Do(ctx, stateDir, func(ctx context.Context) (interface{}, error) {
23+
_, err := g.Do(ctx, stateDir, func(ctx context.Context) (struct{}, error) {
2424
_, _, err := makeHostsFile(stateDir, nil, idmap, hostname)
25-
return nil, err
25+
return struct{}{}, err
2626
})
2727
if err != nil {
2828
return "", nil, err

0 commit comments

Comments
 (0)