Skip to content

Commit 2cdf012

Browse files
fuweidfuweid
authored
Merge pull request containerd#9617 from abel-von/sandbox-plugin-0109
sandbox: use sandboxService in CRI plugin instead of calling controller API directly
2 parents f0c64a9 + a60e52f commit 2cdf012

23 files changed

+307
-169
lines changed

‎integration/build_local_containerd_helper_test.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,6 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string, tweakInitFn tweakPl
129129
containerd.WithDefaultNamespace(constants.K8sContainerdNamespace),
130130
containerd.WithDefaultPlatform(platforms.Default()),
131131
containerd.WithInMemoryServices(lastInitContext),
132-
containerd.WithInMemorySandboxControllers(lastInitContext),
133132
)
134133
require.NoError(t, err)
135134

‎internal/cri/config/config.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@ type ImageConfig struct {
277277
// "base": "docker.io/library/ubuntu:latest"
278278
// Migrated from:
279279
// (PluginConfig).SandboxImage string `toml:"sandbox_image" json:"sandboxImage"`
280-
PinnedImages map[string]string
280+
PinnedImages map[string]string `toml:"pinned_images" json:"pinned_images"`
281281

282282
// RuntimePlatforms is map between the runtime and the image platform to
283283
// use for that runtime. When resolving an image for a runtime, this

‎internal/cri/server/container_create.go

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
6363
return nil, fmt.Errorf("failed to find sandbox id %q: %w", r.GetPodSandboxId(), err)
6464
}
6565

66-
controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler)
67-
if err != nil {
68-
return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
69-
}
70-
71-
cstatus, err := controller.Status(ctx, sandbox.ID, false)
66+
cstatus, err := c.sandboxService.SandboxStatus(ctx, sandbox.Sandboxer, sandbox.ID, false)
7267
if err != nil {
7368
return nil, fmt.Errorf("failed to get controller status: %w", err)
7469
}
@@ -150,7 +145,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
150145
}
151146
}()
152147

153-
platform, err := controller.Platform(ctx, sandboxID)
148+
platform, err := c.sandboxService.SandboxPlatform(ctx, sandbox.Sandboxer, sandboxID)
154149
if err != nil {
155150
return nil, fmt.Errorf("failed to query sandbox platform: %w", err)
156151
}

‎internal/cri/server/container_stats_list.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,15 +68,12 @@ func (c *criService) getMetricsHandler(ctx context.Context, sandboxID string) (m
6868
if err != nil {
6969
return nil, fmt.Errorf("failed to find sandbox id %q: %w", sandboxID, err)
7070
}
71-
controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler)
72-
if err != nil {
73-
return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
74-
}
71+
7572
// Grab the platform that this containers sandbox advertises. Reason being, even if
7673
// the host may be {insert platform}, if it virtualizes or emulates a different platform
7774
// it will return stats in that format, and we need to handle the conversion logic based
7875
// off of this info.
79-
p, err := controller.Platform(ctx, sandboxID)
76+
p, err := c.sandboxService.SandboxPlatform(ctx, sandbox.Sandboxer, sandboxID)
8077
if err != nil {
8178
return nil, err
8279
}

‎internal/cri/server/podsandbox/controller.go

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -158,10 +158,10 @@ func (c *Controller) Wait(ctx context.Context, sandboxID string) (sandbox.ExitSt
158158

159159
}
160160

161-
func (c *Controller) waitSandboxExit(ctx context.Context, p *types.PodSandbox, exitCh <-chan containerd.ExitStatus) (exitStatus uint32, exitedAt time.Time, err error) {
161+
func (c *Controller) waitSandboxExit(ctx context.Context, p *types.PodSandbox, exitCh <-chan containerd.ExitStatus) error {
162162
select {
163163
case e := <-exitCh:
164-
exitStatus, exitedAt, err = e.Result()
164+
exitStatus, exitedAt, err := e.Result()
165165
if err != nil {
166166
log.G(ctx).WithError(err).Errorf("failed to get task exit status for %q", p.ID)
167167
exitStatus = unknownExitCode
@@ -171,12 +171,16 @@ func (c *Controller) waitSandboxExit(ctx context.Context, p *types.PodSandbox, e
171171
dctx, dcancel := context.WithTimeout(dctx, handleEventTimeout)
172172
defer dcancel()
173173
event := &eventtypes.TaskExit{ExitStatus: exitStatus, ExitedAt: protobuf.ToTimestamp(exitedAt)}
174-
if cleanErr := handleSandboxTaskExit(dctx, p, event); cleanErr != nil {
174+
if err := handleSandboxTaskExit(dctx, p, event); err != nil {
175+
// TODO will backoff the event to the controller's own EventMonitor, but not cri's,
176+
// because we should call handleSandboxTaskExit again the next time
177+
// eventMonitor handle this event. but now it goes into cri's EventMonitor,
178+
// the handleSandboxTaskExit will not be called anymore
175179
c.cri.BackOffEvent(p.ID, e)
176180
}
177-
return
181+
return nil
178182
case <-ctx.Done():
179-
return unknownExitCode, time.Now(), ctx.Err()
183+
return ctx.Err()
180184
}
181185
}
182186

@@ -196,5 +200,8 @@ func handleSandboxTaskExit(ctx context.Context, sb *types.PodSandbox, e *eventty
196200
}
197201
}
198202
}
203+
if err := sb.Exit(e.ExitStatus, protobuf.FromTimestamp(e.ExitedAt)); err != nil {
204+
return err
205+
}
199206
return nil
200207
}

‎internal/cri/server/podsandbox/controller_test.go

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323

2424
"github.com/stretchr/testify/assert"
2525

26-
containerd "github.com/containerd/containerd/v2/client"
2726
criconfig "github.com/containerd/containerd/v2/internal/cri/config"
2827
"github.com/containerd/containerd/v2/internal/cri/server/podsandbox/types"
2928
sandboxstore "github.com/containerd/containerd/v2/internal/cri/store/sandbox"
@@ -63,10 +62,7 @@ func Test_Status(t *testing.T) {
6362
CreatedAt: createdAt,
6463
})
6564
sb.Metadata = sandboxstore.Metadata{ID: sandboxID}
66-
err := controller.store.Save(sb)
67-
if err != nil {
68-
t.Fatal(err)
69-
}
65+
assert.NoError(t, controller.store.Save(sb))
7066
s, err := controller.Status(context.Background(), sandboxID, false)
7167
if err != nil {
7268
t.Fatal(err)
@@ -75,7 +71,8 @@ func Test_Status(t *testing.T) {
7571
assert.Equal(t, s.CreatedAt, createdAt)
7672
assert.Equal(t, s.State, sandboxstore.StateReady.String())
7773

78-
sb.Exit(*containerd.NewExitStatus(exitStatus, exitedAt, nil))
74+
assert.NoError(t, sb.Exit(exitStatus, exitedAt))
75+
7976
exit, err := controller.Wait(context.Background(), sandboxID)
8077
if err != nil {
8178
t.Fatal(err)

‎internal/cri/server/podsandbox/recover.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,17 @@ import (
2222
goruntime "runtime"
2323
"time"
2424

25+
"github.com/containerd/errdefs"
2526
"github.com/containerd/log"
2627
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
2728

2829
containerd "github.com/containerd/containerd/v2/client"
2930
sandbox2 "github.com/containerd/containerd/v2/core/sandbox"
31+
"github.com/containerd/containerd/v2/internal/cri/config"
3032
"github.com/containerd/containerd/v2/internal/cri/server/podsandbox/types"
3133
sandboxstore "github.com/containerd/containerd/v2/internal/cri/store/sandbox"
3234
ctrdutil "github.com/containerd/containerd/v2/internal/cri/util"
3335
"github.com/containerd/containerd/v2/pkg/netns"
34-
"github.com/containerd/errdefs"
3536
)
3637

3738
// loadContainerTimeout is the default timeout for loading a container/sandbox.
@@ -133,8 +134,9 @@ func (c *Controller) RecoverContainer(ctx context.Context, cntr containerd.Conta
133134
}
134135
if ch != nil {
135136
go func() {
136-
code, exitTime, err := c.waitSandboxExit(ctrdutil.NamespacedContext(), podSandbox, ch)
137-
podSandbox.Exit(*containerd.NewExitStatus(code, exitTime, err))
137+
if err := c.waitSandboxExit(ctrdutil.NamespacedContext(), podSandbox, ch); err != nil {
138+
log.G(context.Background()).Warnf("failed to wait pod sandbox exit %v", err)
139+
}
138140
}()
139141
}
140142

@@ -144,6 +146,7 @@ func (c *Controller) RecoverContainer(ctx context.Context, cntr containerd.Conta
144146

145147
sandbox = sandboxstore.NewSandbox(*meta, s)
146148
sandbox.Container = cntr
149+
sandbox.Sandboxer = string(config.ModePodSandbox)
147150

148151
// Load network namespace.
149152
sandbox.NetNS = getNetNS(meta)

‎internal/cri/server/podsandbox/recover_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ func TestRecoverContainer(t *testing.T) {
403403

404404
pSb := controller.store.Get(cont.ID())
405405
assert.NotNil(t, pSb)
406-
assert.Equal(t, c.expectedState, pSb.State, "%s state is not expected", cont.ID())
406+
assert.Equal(t, c.expectedState, pSb.Status.Get().State, "%s state is not expected", cont.ID())
407407

408408
if c.expectedExitCode > 0 {
409409
cont.t.waitExitCh <- struct{}{}

‎internal/cri/server/podsandbox/sandbox_run.go

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,6 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
216216
if err != nil {
217217
return cin, fmt.Errorf("failed to get sandbox container info: %w", err)
218218
}
219-
podSandbox.CreatedAt = info.CreatedAt
220219

221220
// Create sandbox task in containerd.
222221
log.G(ctx).Tracef("Create sandbox container (id=%q, name=%q).", id, metadata.Name)
@@ -242,7 +241,6 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
242241
}
243242
}
244243
}()
245-
podSandbox.Pid = task.Pid()
246244

247245
// wait is a long running background request, no timeout needed.
248246
exitCh, err := task.Wait(ctrdutil.NamespacedContext())
@@ -267,16 +265,25 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
267265
if err := task.Start(ctx); err != nil {
268266
return cin, fmt.Errorf("failed to start sandbox container task %q: %w", id, err)
269267
}
270-
podSandbox.State = sandboxstore.StateReady
268+
pid := task.Pid()
269+
if err := podSandbox.Status.Update(func(status sandboxstore.Status) (sandboxstore.Status, error) {
270+
status.Pid = pid
271+
status.State = sandboxstore.StateReady
272+
status.CreatedAt = info.CreatedAt
273+
return status, nil
274+
}); err != nil {
275+
return cin, fmt.Errorf("failed to update status of pod sandbox %q: %w", id, err)
276+
}
271277

272278
cin.SandboxID = id
273279
cin.Pid = task.Pid()
274280
cin.CreatedAt = info.CreatedAt
275281
cin.Labels = labels
276282

277283
go func() {
278-
code, exitTime, err := c.waitSandboxExit(ctrdutil.NamespacedContext(), podSandbox, exitCh)
279-
podSandbox.Exit(*containerd.NewExitStatus(code, exitTime, err))
284+
if err := c.waitSandboxExit(ctrdutil.NamespacedContext(), podSandbox, exitCh); err != nil {
285+
log.G(context.Background()).Warnf("failed to wait pod sandbox exit %v", err)
286+
}
280287
}()
281288

282289
return

‎internal/cri/server/podsandbox/sandbox_status.go

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,18 +36,15 @@ func (c *Controller) Status(ctx context.Context, sandboxID string, verbose bool)
3636
if sb == nil {
3737
return sandbox.ControllerStatus{}, fmt.Errorf("unable to find sandbox %q: %w", sandboxID, errdefs.ErrNotFound)
3838
}
39-
39+
status := sb.Status.Get()
4040
cstatus := sandbox.ControllerStatus{
4141
SandboxID: sandboxID,
42-
Pid: sb.Pid,
43-
State: sb.State.String(),
44-
CreatedAt: sb.CreatedAt,
42+
Pid: status.Pid,
43+
State: status.State.String(),
44+
CreatedAt: status.CreatedAt,
45+
ExitedAt: status.ExitedAt,
4546
Extra: nil,
4647
}
47-
exitStatus := sb.GetExitStatus()
48-
if exitStatus != nil {
49-
cstatus.ExitedAt = exitStatus.ExitTime()
50-
}
5148

5249
if verbose {
5350
info, err := toCRISandboxInfo(ctx, sb)
@@ -64,7 +61,7 @@ func (c *Controller) Status(ctx context.Context, sandboxID string, verbose bool)
6461
// toCRISandboxInfo converts internal container object information to CRI sandbox status response info map.
6562
func toCRISandboxInfo(ctx context.Context, sb *types.PodSandbox) (map[string]string, error) {
6663
si := &critypes.SandboxInfo{
67-
Pid: sb.Pid,
64+
Pid: sb.Status.Get().Pid,
6865
Config: sb.Metadata.Config,
6966
RuntimeHandler: sb.Metadata.RuntimeHandler,
7067
CNIResult: sb.Metadata.CNIResult,

0 commit comments

Comments
 (0)