force kill first

Author: iQQBot

components/ws-daemon/pkg/container/container.go

@@ -57,6 +57,8 @@ type Runtime interface {
 	DisposeContainer(ctx context.Context, workspaceInstanceID string)
 
 	GetContainerTaskInfo(ctx context.Context, id ID) (*task.Process, error)
+
+	ForceKillContainerTask(ctx context.Context, id ID) error
 }
 
 var (

components/ws-daemon/pkg/container/containerd.go

@@ -587,15 +587,16 @@ func (s *Containerd) GetContainerTaskInfo(ctx context.Context, id ID) (*task.Pro
 	if task.Process == nil {
 		return nil, fmt.Errorf("task has no process")
 	}
+	return task.Process, nil
+}
 
-	// try send last kill signal to the task
-	_, _ = s.Client.TaskService().Kill(ctx, &tasks.KillRequest{
+func (s *Containerd) ForceKillContainerTask(ctx context.Context, id ID) error {
+	_, err := s.Client.TaskService().Kill(ctx, &tasks.KillRequest{
 		ContainerID: string(id),
 		Signal:      9,
 		All:         true,
 	})
-
-	return task.Process, nil
+	return err
 }
 
 var kubepodsQoSRegexp = regexp.MustCompile(`([^/]+)-([^/]+)-pod`)

components/ws-daemon/pkg/controller/workspace_controller.go

@@ -358,12 +358,30 @@ func (wsc *WorkspaceController) doWorkspaceContentBackup(ctx context.Context, sp
 			// We should get an event when the condition changes, but requeue
 			// anyways to make sure we act on it in time.
 			return ctrl.Result{RequeueAfter: 500 * time.Millisecond}, nil
-		} else {
-			glog.WithFields(ws.OWI()).WithField("workspace", req.NamespacedName).Warn("workspace container is still running after 5 minutes of deletion, starting backup anyway")
-			err = wsc.dumpWorkspaceContainerInfo(ctx, ws)
+		}
+
+		if !ws.IsConditionTrue(workspacev1.WorkspaceConditionForceKilledTask) {
+			err = wsc.forceKillContainerTask(ctx, ws)
+			if err != nil {
+				return ctrl.Result{}, fmt.Errorf("failed to force kill container task: %w", err)
+			}
+			err = retry.RetryOnConflict(retryParams, func() error {
+				if err := wsc.Get(ctx, req.NamespacedName, ws); err != nil {
+					return err
+				}
+				ws.Status.SetCondition(workspacev1.NewWorkspaceConditionForceKilledTask())
+				return wsc.Client.Status().Update(ctx, ws)
+			})
 			if err != nil {
-				glog.WithFields(ws.OWI()).WithField("workspace", req.NamespacedName).Errorf("failed to dump container info: %v", err)
+				return ctrl.Result{}, fmt.Errorf("failed to set force killed task condition: %w", err)
 			}
+			return ctrl.Result{Requeue: true, RequeueAfter: 2 * time.Second}, nil
+		}
+
+		glog.WithFields(ws.OWI()).WithField("workspace", req.NamespacedName).Warn("workspace container is still running after 5 minutes of deletion, starting backup anyway")
+		err = wsc.dumpWorkspaceContainerInfo(ctx, ws)
+		if err != nil {
+			glog.WithFields(ws.OWI()).WithField("workspace", req.NamespacedName).Errorf("failed to dump container info: %v", err)
 		}
 	}
 
@@ -474,6 +492,14 @@ func (wsc *WorkspaceController) dumpWorkspaceContainerInfo(ctx context.Context,
 	return nil
 }
 
+func (wsc *WorkspaceController) forceKillContainerTask(ctx context.Context, ws *workspacev1.Workspace) error {
+	id, err := wsc.runtime.WaitForContainer(ctx, ws.Name)
+	if err != nil {
+		return fmt.Errorf("failed to wait for container: %w", err)
+	}
+	return wsc.runtime.ForceKillContainerTask(ctx, id)
+}
+
 func (wsc *WorkspaceController) prepareInitializer(ctx context.Context, ws *workspacev1.Workspace) (*csapi.WorkspaceInitializer, error) {
 	var init csapi.WorkspaceInitializer
 	err := proto.Unmarshal(ws.Spec.Initializer, &init)

components/ws-manager-api/go/crd/v1/workspace_types.go

@@ -286,6 +286,9 @@ const (
 
 	// WorkspaceConditionStateWiped is true once all state has successfully been wiped by ws-daemon. This is only set if PodRejected=true, and the rejected workspace has been deleted.
 	WorkspaceConditionStateWiped WorkspaceCondition = "StateWiped"
+
+	// WorkspaceConditionForceKilledTask is true if we send a SIGKILL to the task
+	WorkspaceConditionForceKilledTask WorkspaceCondition = "ForceKilledTask"
 )
 
 func NewWorkspaceConditionDeployed() metav1.Condition {
@@ -440,6 +443,14 @@ func NewWorkspaceConditionContainerRunning(status metav1.ConditionStatus) metav1
 	}
 }
 
+func NewWorkspaceConditionForceKilledTask() metav1.Condition {
+	return metav1.Condition{
+		Type:               string(WorkspaceConditionForceKilledTask),
+		LastTransitionTime: metav1.Now(),
+		Status:             metav1.ConditionTrue,
+	}
+}
+
 // +kubebuilder:validation:Enum:=Unknown;Pending;Imagebuild;Creating;Initializing;Running;Stopping;Stopped
 type WorkspacePhase string