Skip to content

Commit 3f7b522

Browse files
tonistiigitonistiigi
authored
Merge pull request #6288 from tonistiigi/solver-jobctx
solver: add jobcontext to ops caller
2 parents 395b2f8 + 30a3486 commit 3f7b522

File tree

10 files changed

+61
-31
lines changed

10 files changed

+61
-31
lines changed

‎solver/jobs.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ type state struct {
7373
solver *Solver
7474
}
7575

76+
func (s *state) Session() session.Group {
77+
return s
78+
}
79+
7680
func (s *state) SessionIterator() session.Iterator {
7781
return s.sessionIterator()
7882
}

‎solver/llbsolver/ops/build.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88
"github.com/containerd/continuity/fs"
99
"github.com/moby/buildkit/client/llb"
1010
"github.com/moby/buildkit/frontend"
11-
"github.com/moby/buildkit/session"
1211
"github.com/moby/buildkit/snapshot"
1312
"github.com/moby/buildkit/solver"
1413
"github.com/moby/buildkit/solver/llbsolver/ops/opsutils"
@@ -40,7 +39,7 @@ func NewBuildOp(v solver.Vertex, op *pb.Op_Build, b frontend.FrontendLLBBridge,
4039
}, nil
4140
}
4241

43-
func (b *BuildOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) {
42+
func (b *BuildOp) CacheMap(ctx context.Context, job solver.JobContext, index int) (*solver.CacheMap, bool, error) {
4443
dt, err := json.Marshal(struct {
4544
Type string
4645
Exec *pb.BuildOp
@@ -66,7 +65,7 @@ func (b *BuildOp) CacheMap(ctx context.Context, g session.Group, index int) (*so
6665
}, true, nil
6766
}
6867

69-
func (b *BuildOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) (outputs []solver.Result, retErr error) {
68+
func (b *BuildOp) Exec(ctx context.Context, job solver.JobContext, inputs []solver.Result) (outputs []solver.Result, retErr error) {
7069
if b.op.Builder != int64(pb.LLBBuilder) {
7170
return nil, errors.Errorf("only LLB builder is currently allowed")
7271
}
@@ -88,6 +87,7 @@ func (b *BuildOp) Exec(ctx context.Context, g session.Group, inputs []solver.Res
8887
return nil, errors.Errorf("invalid reference for build %T", inp.Sys())
8988
}
9089

90+
g := job.Session()
9191
mount, err := ref.ImmutableRef.Mount(ctx, true, g)
9292
if err != nil {
9393
return nil, err

‎solver/llbsolver/ops/diff.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88
"github.com/pkg/errors"
99

1010
"github.com/moby/buildkit/cache"
11-
"github.com/moby/buildkit/session"
1211
"github.com/moby/buildkit/solver"
1312
"github.com/moby/buildkit/solver/llbsolver/ops/opsutils"
1413
"github.com/moby/buildkit/solver/pb"
@@ -34,7 +33,7 @@ func NewDiffOp(v solver.Vertex, op *pb.Op_Diff, w worker.Worker) (solver.Op, err
3433
}, nil
3534
}
3635

37-
func (d *diffOp) CacheMap(ctx context.Context, group session.Group, index int) (*solver.CacheMap, bool, error) {
36+
func (d *diffOp) CacheMap(ctx context.Context, jobCtx solver.JobContext, index int) (*solver.CacheMap, bool, error) {
3837
dt, err := json.Marshal(struct {
3938
Type string
4039
Diff *pb.DiffOp
@@ -66,7 +65,7 @@ func (d *diffOp) CacheMap(ctx context.Context, group session.Group, index int) (
6665
return cm, true, nil
6766
}
6867

69-
func (d *diffOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) ([]solver.Result, error) {
68+
func (d *diffOp) Exec(ctx context.Context, jobCtx solver.JobContext, inputs []solver.Result) ([]solver.Result, error) {
7069
var curInput int
7170

7271
var lowerRef cache.ImmutableRef

‎solver/llbsolver/ops/exec.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ func checkShouldClearCacheOpts(m *pb.Mount) bool {
104104
return true
105105
}
106106

107-
func (e *ExecOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) {
107+
func (e *ExecOp) CacheMap(ctx context.Context, jobCtx solver.JobContext, index int) (*solver.CacheMap, bool, error) {
108108
op := cloneExecOp(e.op)
109109

110110
for i := range op.Meta.ExtraHosts {
@@ -362,7 +362,7 @@ func addDefaultEnvvar(env []string, k, v string) []string {
362362
return append(env, k+"="+v)
363363
}
364364

365-
func (e *ExecOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) (results []solver.Result, err error) {
365+
func (e *ExecOp) Exec(ctx context.Context, jobCtx solver.JobContext, inputs []solver.Result) (results []solver.Result, err error) {
366366
trace.SpanFromContext(ctx).AddEvent("ExecOp started")
367367

368368
refs := make([]*worker.WorkerRef, len(inputs))
@@ -378,6 +378,7 @@ func (e *ExecOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu
378378
if e.platform != nil {
379379
platformOS = e.platform.OS
380380
}
381+
g := jobCtx.Session()
381382
p, err := container.PrepareMounts(ctx, e.mm, e.cm, g, e.op.Meta.Cwd, e.op.Mounts, refs, func(m *pb.Mount, ref cache.ImmutableRef) (cache.MutableRef, error) {
382383
desc := fmt.Sprintf("mount %s from exec %s", m.Dest, strings.Join(e.op.Meta.Args, " "))
383384
return e.cm.New(ctx, ref, g, cache.WithDescription(desc))

‎solver/llbsolver/ops/exec_test.go‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66

77
"github.com/moby/buildkit/identity"
88
"github.com/moby/buildkit/session"
9+
"github.com/moby/buildkit/solver"
910
"github.com/moby/buildkit/solver/pb"
1011
"github.com/stretchr/testify/require"
1112
)
@@ -110,11 +111,11 @@ func TestExecOpCacheMap(t *testing.T) {
110111
t.Run(tc.name, func(t *testing.T) {
111112
t.Parallel()
112113

113-
m1, ok, err := tc.op1.CacheMap(ctx, session.NewGroup(t.Name()), 1)
114+
m1, ok, err := tc.op1.CacheMap(ctx, testJobContext(t), 1)
114115
require.NoError(t, err)
115116
require.True(t, ok)
116117

117-
m2, ok, err := tc.op2.CacheMap(ctx, session.NewGroup(t.Name()), 1)
118+
m2, ok, err := tc.op2.CacheMap(ctx, testJobContext(t), 1)
118119
require.NoError(t, err)
119120
require.True(t, ok)
120121

@@ -195,7 +196,7 @@ func TestExecOpContentCache(t *testing.T) {
195196
t.Parallel()
196197

197198
// default is always valid, and can sometimes have slow-cache
198-
m, ok, err := tc.op.CacheMap(ctx, session.NewGroup(t.Name()), 1)
199+
m, ok, err := tc.op.CacheMap(ctx, testJobContext(t), 1)
199200
require.NoError(t, err)
200201
require.True(t, ok)
201202
for _, dep := range m.Deps {
@@ -210,7 +211,7 @@ func TestExecOpContentCache(t *testing.T) {
210211
for _, mnt := range tc.op.op.Mounts {
211212
mnt.ContentCache = pb.MountContentCache_OFF
212213
}
213-
m, ok, err = tc.op.CacheMap(ctx, session.NewGroup(t.Name()), 1)
214+
m, ok, err = tc.op.CacheMap(ctx, testJobContext(t), 1)
214215
require.NoError(t, err)
215216
require.True(t, ok)
216217
for _, dep := range m.Deps {
@@ -221,7 +222,7 @@ func TestExecOpContentCache(t *testing.T) {
221222
for _, mnt := range tc.op.op.Mounts {
222223
mnt.ContentCache = pb.MountContentCache_ON
223224
}
224-
m, ok, err = tc.op.CacheMap(ctx, session.NewGroup(t.Name()), 1)
225+
m, ok, err = tc.op.CacheMap(ctx, testJobContext(t), 1)
225226
if tc.cacheIsSafe {
226227
require.NoError(t, err)
227228
require.True(t, ok)
@@ -289,3 +290,15 @@ func withoutOutput() func(*pb.Mount) {
289290
m.Output = int64(pb.SkipOutput)
290291
}
291292
}
293+
294+
func testJobContext(t *testing.T) solver.JobContext {
295+
return &jobCtx{g: session.NewGroup(t.Name())}
296+
}
297+
298+
type jobCtx struct {
299+
g session.Group
300+
}
301+
302+
func (j *jobCtx) Session() session.Group {
303+
return j.g
304+
}

‎solver/llbsolver/ops/file.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func NewFileOp(v solver.Vertex, op *pb.Op_File, cm cache.Manager, parallelism *s
5252
}, nil
5353
}
5454

55-
func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) {
55+
func (f *fileOp) CacheMap(ctx context.Context, jobCtx solver.JobContext, index int) (*solver.CacheMap, bool, error) {
5656
selectors := map[int][]opsutils.Selector{}
5757
invalidSelectors := map[int]struct{}{}
5858

@@ -173,7 +173,7 @@ func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol
173173
return cm, true, nil
174174
}
175175

176-
func (f *fileOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) ([]solver.Result, error) {
176+
func (f *fileOp) Exec(ctx context.Context, jobCtx solver.JobContext, inputs []solver.Result) ([]solver.Result, error) {
177177
inpRefs := make([]fileoptypes.Ref, 0, len(inputs))
178178
for _, inp := range inputs {
179179
workerRef, ok := inp.Sys().(*worker.WorkerRef)
@@ -189,7 +189,7 @@ func (f *fileOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu
189189
}
190190

191191
fs := NewFileOpSolver(f.w, backend, f.refManager)
192-
outs, err := fs.Solve(ctx, inpRefs, f.op.Actions, g)
192+
outs, err := fs.Solve(ctx, inpRefs, f.op.Actions, jobCtx.Session())
193193
if err != nil {
194194
return nil, err
195195
}

‎solver/llbsolver/ops/merge.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"github.com/pkg/errors"
1010

1111
"github.com/moby/buildkit/cache"
12-
"github.com/moby/buildkit/session"
1312
"github.com/moby/buildkit/solver"
1413
"github.com/moby/buildkit/solver/llbsolver/ops/opsutils"
1514
"github.com/moby/buildkit/solver/pb"
@@ -35,7 +34,7 @@ func NewMergeOp(v solver.Vertex, op *pb.Op_Merge, w worker.Worker) (solver.Op, e
3534
}, nil
3635
}
3736

38-
func (m *mergeOp) CacheMap(ctx context.Context, group session.Group, index int) (*solver.CacheMap, bool, error) {
37+
func (m *mergeOp) CacheMap(ctx context.Context, jobCtx solver.JobContext, index int) (*solver.CacheMap, bool, error) {
3938
dt, err := json.Marshal(struct {
4039
Type string
4140
Merge *pb.MergeOp
@@ -63,7 +62,7 @@ func (m *mergeOp) CacheMap(ctx context.Context, group session.Group, index int)
6362
return cm, true, nil
6463
}
6564

66-
func (m *mergeOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) ([]solver.Result, error) {
65+
func (m *mergeOp) Exec(ctx context.Context, jobCtx solver.JobContext, inputs []solver.Result) ([]solver.Result, error) {
6766
refs := make([]cache.ImmutableRef, len(inputs))
6867
var index int
6968
for _, inp := range inputs {

‎solver/llbsolver/ops/source.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,13 @@ func (s *SourceOp) instance(ctx context.Context) (source.SourceInstance, error)
7474
return s.src, nil
7575
}
7676

77-
func (s *SourceOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) {
77+
func (s *SourceOp) CacheMap(ctx context.Context, jobCtx solver.JobContext, index int) (*solver.CacheMap, bool, error) {
7878
src, err := s.instance(ctx)
7979
if err != nil {
8080
return nil, false, err
8181
}
8282

83-
k, pin, cacheOpts, done, err := src.CacheKey(ctx, g, index)
83+
k, pin, cacheOpts, done, err := src.CacheKey(ctx, jobCtx.Session(), index)
8484
if err != nil {
8585
return nil, false, err
8686
}
@@ -104,12 +104,12 @@ func (s *SourceOp) CacheMap(ctx context.Context, g session.Group, index int) (*s
104104
}, done, nil
105105
}
106106

107-
func (s *SourceOp) Exec(ctx context.Context, g session.Group, _ []solver.Result) (outputs []solver.Result, err error) {
107+
func (s *SourceOp) Exec(ctx context.Context, jobCtx solver.JobContext, _ []solver.Result) (outputs []solver.Result, err error) {
108108
src, err := s.instance(ctx)
109109
if err != nil {
110110
return nil, err
111111
}
112-
ref, err := src.Snapshot(ctx, g)
112+
ref, err := src.Snapshot(ctx, jobCtx.Session())
113113
if err != nil {
114114
return nil, err
115115
}

‎solver/scheduler_test.go‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3678,6 +3678,8 @@ type vertex struct {
36783678
execCallCount *int64
36793679
}
36803680

3681+
var _ Op = &vertex{}
3682+
36813683
func (v *vertex) Digest() digest.Digest {
36823684
return digest.FromBytes([]byte(v.opt.name))
36833685
}
@@ -3752,7 +3754,7 @@ func (v *vertex) cacheMap(ctx context.Context) error {
37523754
return nil
37533755
}
37543756

3755-
func (v *vertex) CacheMap(ctx context.Context, g session.Group, index int) (*CacheMap, bool, error) {
3757+
func (v *vertex) CacheMap(ctx context.Context, jobCtx JobContext, index int) (*CacheMap, bool, error) {
37563758
if index == 0 {
37573759
if err := v.cacheMap(ctx); err != nil {
37583760
return nil, false, err
@@ -3789,7 +3791,7 @@ func (v *vertex) exec(ctx context.Context, inputs []Result) error {
37893791
return nil
37903792
}
37913793

3792-
func (v *vertex) Exec(ctx context.Context, g session.Group, inputs []Result) (outputs []Result, err error) {
3794+
func (v *vertex) Exec(ctx context.Context, job JobContext, inputs []Result) (outputs []Result, err error) {
37933795
if err := v.exec(ctx, inputs); err != nil {
37943796
return nil, err
37953797
}
@@ -3834,11 +3836,13 @@ type vertexConst struct {
38343836
value int
38353837
}
38363838

3839+
var _ Op = &vertexConst{}
3840+
38373841
func (v *vertexConst) Sys() any {
38383842
return v
38393843
}
38403844

3841-
func (v *vertexConst) Exec(ctx context.Context, g session.Group, inputs []Result) (outputs []Result, err error) {
3845+
func (v *vertexConst) Exec(ctx context.Context, jobCtx JobContext, inputs []Result) (outputs []Result, err error) {
38423846
if err := v.exec(ctx, inputs); err != nil {
38433847
return nil, err
38443848
}
@@ -3865,11 +3869,13 @@ type vertexSum struct {
38653869
value int
38663870
}
38673871

3872+
var _ Op = &vertexSum{}
3873+
38683874
func (v *vertexSum) Sys() any {
38693875
return v
38703876
}
38713877

3872-
func (v *vertexSum) Exec(ctx context.Context, g session.Group, inputs []Result) (outputs []Result, err error) {
3878+
func (v *vertexSum) Exec(ctx context.Context, jobCtx JobContext, inputs []Result) (outputs []Result, err error) {
38733879
if err := v.exec(ctx, inputs); err != nil {
38743880
return nil, err
38753881
}
@@ -3904,11 +3910,13 @@ type vertexAdd struct {
39043910
value int
39053911
}
39063912

3913+
var _ Op = &vertexAdd{}
3914+
39073915
func (v *vertexAdd) Sys() any {
39083916
return v
39093917
}
39103918

3911-
func (v *vertexAdd) Exec(ctx context.Context, g session.Group, inputs []Result) (outputs []Result, err error) {
3919+
func (v *vertexAdd) Exec(ctx context.Context, jobCtx JobContext, inputs []Result) (outputs []Result, err error) {
39123920
if err := v.exec(ctx, inputs); err != nil {
39133921
return nil, err
39143922
}
@@ -3942,11 +3950,13 @@ type vertexSubBuild struct {
39423950
b Builder
39433951
}
39443952

3953+
var _ Op = &vertexSubBuild{}
3954+
39453955
func (v *vertexSubBuild) Sys() any {
39463956
return v
39473957
}
39483958

3949-
func (v *vertexSubBuild) Exec(ctx context.Context, g session.Group, inputs []Result) (outputs []Result, err error) {
3959+
func (v *vertexSubBuild) Exec(ctx context.Context, jobCtx JobContext, inputs []Result) (outputs []Result, err error) {
39503960
if err := v.exec(ctx, inputs); err != nil {
39513961
return nil, err
39523962
}

‎solver/types.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,15 +166,19 @@ type ReleaseFunc func()
166166
type Op interface {
167167
// CacheMap returns structure describing how the operation is cached.
168168
// Currently only roots are allowed to return multiple cache maps per op.
169-
CacheMap(context.Context, session.Group, int) (*CacheMap, bool, error)
169+
CacheMap(context.Context, JobContext, int) (*CacheMap, bool, error)
170170

171171
// Exec runs an operation given results from previous operations.
172-
Exec(ctx context.Context, g session.Group, inputs []Result) (outputs []Result, err error)
172+
Exec(ctx context.Context, jobCtx JobContext, inputs []Result) (outputs []Result, err error)
173173

174174
// Acquire acquires the necessary resources to execute the `Op`.
175175
Acquire(ctx context.Context) (release ReleaseFunc, err error)
176176
}
177177

178+
type JobContext interface {
179+
Session() session.Group
180+
}
181+
178182
type ProvenanceProvider interface {
179183
IsProvenanceProvider()
180184
}

0 commit comments

Comments
 (0)