Merge pull request containerd#9644 from abel-von/fix-sandbox-status

sandbox: fix podsandbox recover status issue

Author: fuweid

integration/restart_linux_test.go

@@ -0,0 +1,78 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package integration
+
+import (
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+func TestContainerdRestartSandboxRecover(t *testing.T) {
+	sbStatuses := map[string]runtime.PodSandboxState{
+		// Sandbox with unknown status will be NotReady when returned from ListPodSandbox
+		"sandbox_unknown":   runtime.PodSandboxState_SANDBOX_NOTREADY,
+		"sandbox_not_ready": runtime.PodSandboxState_SANDBOX_NOTREADY,
+		"sandbox_ready":     runtime.PodSandboxState_SANDBOX_READY,
+	}
+
+	sbReadyConfig := PodSandboxConfig("sandbox_ready", "sandbox_ready")
+	_, err := runtimeService.RunPodSandbox(sbReadyConfig, *runtimeHandler)
+	assert.NoError(t, err)
+
+	sbNotReadyConfig := PodSandboxConfig("sandbox_not_ready", "sandbox_not_ready")
+	notReadyID, err := runtimeService.RunPodSandbox(sbNotReadyConfig, *runtimeHandler)
+	assert.NoError(t, err)
+	err = runtimeService.StopPodSandbox(notReadyID)
+	assert.NoError(t, err)
+
+	t.Logf("Create a pod config with shim create delay")
+	sbUnknownConfig := PodSandboxConfig("sandbox_unknown", "sandbox_unknown_status")
+	injectShimFailpoint(t, sbUnknownConfig, map[string]string{
+		"Create": "1*delay(2000)",
+	})
+	waitCh := make(chan struct{})
+	go func() {
+		time.Sleep(time.Second)
+		t.Logf("Create a sandbox with shim create delay")
+		RestartContainerd(t, syscall.SIGTERM)
+		waitCh <- struct{}{}
+	}()
+	t.Logf("Create a sandbox with shim create delay")
+	_, err = runtimeService.RunPodSandbox(sbUnknownConfig, failpointRuntimeHandler)
+	assert.Error(t, err)
+	<-waitCh
+	sbs, err := runtimeService.ListPodSandbox(nil)
+	assert.NoError(t, err)
+	foundUnkownSb := false
+	for _, sb := range sbs {
+		if sb.Metadata.Name == "sandbox_unknown" {
+			foundUnkownSb = true
+		}
+		if status, ok := sbStatuses[sb.Metadata.Name]; ok {
+			assert.Equal(t, status, sb.State)
+			err = runtimeService.StopPodSandbox(sb.Id)
+			assert.NoError(t, err)
+			err = runtimeService.RemovePodSandbox(sb.Id)
+			assert.NoError(t, err)
+		}
+	}
+	assert.True(t, foundUnkownSb)
+}

internal/cri/server/podsandbox/recover.go

@@ -96,13 +96,17 @@ func (c *Controller) RecoverContainer(ctx context.Context, cntr containerd.Conta
 			status.State = sandboxstore.StateNotReady
 		} else {
 			if taskStatus.Status == containerd.Running {
-				status.State = sandboxstore.StateReady
-				status.Pid = t.Pid()
 				exitCh, err := t.Wait(ctrdutil.NamespacedContext())
 				if err != nil {
-					return status, channel, fmt.Errorf("failed to wait for sandbox container task: %w", err)
+					if !errdefs.IsNotFound(err) {
+						return status, channel, fmt.Errorf("failed to wait for sandbox container task: %w", err)
+					}
+					status.State = sandboxstore.StateNotReady
+				} else {
+					status.State = sandboxstore.StateReady
+					status.Pid = t.Pid()
+					channel = exitCh
 				}
-				channel = exitCh
 			} else {
 				// Task is not running. Delete the task and set sandbox state as NOTREADY.
 				if _, err := t.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {