StopVM to do forceful shutdown if VM is in paused state

StopVM calls an agent inside the VM. If the VM is paused StopVM can
be stuck as it cannot connect to the agent. To fix this, we implemented
api in go-sdk to read instance state and here we call it internally to
read the VM state. If it is paused then we do a forceful shutdown of the
VM.

Signed-off-by: Alakesh Haloi <[email protected]>

Author: alakesh

runtime/service.go

@@ -636,7 +636,8 @@ func (s *service) mountDrives(requestCtx context.Context) error {
 
 // StopVM will shutdown the VMM. Unlike Shutdown, this method is exposed to containerd clients.
 // If the VM has not been created yet and the timeout is hit waiting for it to exist, an error will be returned
-// but the shim will continue to shutdown.
+// but the shim will continue to shutdown. Similarly if we detect that the VM is in pause state, then
+// we are unable to communicate to the in-VM agent. In this case, we do a forceful shutdown.
 func (s *service) StopVM(requestCtx context.Context, request *proto.StopVMRequest) (_ *empty.Empty, err error) {
 	defer logPanicAndDie(s.logger)
 	s.logger.WithFields(logrus.Fields{"timeout_seconds": request.TimeoutSeconds}).Debug("StopVM")
@@ -646,6 +647,20 @@ func (s *service) StopVM(requestCtx context.Context, request *proto.StopVMReques
 		timeout = time.Duration(request.TimeoutSeconds) * time.Second
 	}
 
+	info, err := s.machine.DescribeInstanceInfo(requestCtx)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get instance info %v", info)
+	}
+
+	if *info.State == models.InstanceInfoStatePaused {
+		s.logger.Debug("Instance is in Paused state, force shutdown in progress")
+		err = s.jailer.Stop(true)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to stop VM in paused State")
+		}
+		return &empty.Empty{}, nil
+	}
+
 	err = s.waitVMReady()
 	if err != nil {
 		return nil, err

runtime/service_integ_test.go

@@ -1537,6 +1537,7 @@ func TestStopVM_Isolated(t *testing.T) {
 		stopFunc        func(ctx context.Context, fcClient fccontrol.FirecrackerService, req proto.CreateVMRequest)
 		withStopVM      bool
 	}{
+
 		{
 			name:       "Successful",
 			withStopVM: true,
@@ -1626,6 +1627,22 @@ func TestStopVM_Isolated(t *testing.T) {
 				}
 			},
 		},
+
+		// Test that StopVM returns success if the VM is in paused state, instead of hanging forever.
+		// VM is force shutdown in this case, so we expect no Error or hang.
+		{
+			name:       "PauseStop",
+			withStopVM: true,
+
+			createVMRequest: proto.CreateVMRequest{},
+			stopFunc: func(ctx context.Context, fcClient fccontrol.FirecrackerService, req proto.CreateVMRequest) {
+				_, err = fcClient.PauseVM(ctx, &proto.PauseVMRequest{VMID: req.VMID})
+				require.Equal(status.Code(err), codes.OK)
+
+				_, err = fcClient.StopVM(ctx, &proto.StopVMRequest{VMID: req.VMID})
+				require.NoError(err)
+			},
+		},
 	}
 
 	fcClient, err := newFCControlClient(containerdSockPath)