pillar/qmp: do several attempts to retrieve valid QEMU status

QEMU status is very crucial to EVE and any error or unexpected status
leads to QEMU process is being stopped.

There is an issue in the 3rd QMP library:

   digitalocean/go-qemu#210

And in order to be on a safe side and avoid these kind of problems in
the future repeat status qeury several times.

Signed-off-by: Roman Penyaev <[email protected]>

Author: rouming

pkg/pillar/hypervisor/qmp.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/digitalocean/go-qemu/qmp"
 	"github.com/lf-edge/eve/pkg/pillar/types"
@@ -102,7 +103,21 @@ func getQemuStatus(socket string) (types.SwState, error) {
 		"preconfig":      types.PAUSED,
 	}
 
-	if raw, err := execRawCmd(socket, `{ "execute": "query-status" }`); err == nil {
+	// We do several retries, because correct QEMU status is very crucial to EVE
+	// and if for some reason (https://github.com/digitalocean/go-qemu/pull/210)
+	// the status is unexpected, EVE stops QEMU and game over.
+	var errs error
+	state := types.UNKNOWN
+	for attempt := 1; attempt <= 3; attempt++ {
+		raw, err := execRawCmd(socket, `{ "execute": "query-status" }`)
+		if err != nil {
+			err = fmt.Errorf("[attempt %d] qmp status failed for QMP socket '%s': err: '%v'; (JSON response: '%s')",
+				attempt, socket, err, raw)
+			errs = errors.Join(errs, err)
+			time.Sleep(time.Second)
+			continue
+		}
+
 		var result struct {
 			ID     string `json:"id"`
 			Return struct {
@@ -114,18 +129,27 @@ func getQemuStatus(socket string) (types.SwState, error) {
 		dec := json.NewDecoder(bytes.NewReader(raw))
 		dec.DisallowUnknownFields()
 		err = dec.Decode(&result)
-		var matched bool
-		var state types.SwState
 		if err != nil {
-			err = fmt.Errorf("%v; (JSON received: '%s')", err, raw)
-		} else if state, matched = qmpStatusMap[result.Return.Status]; !matched {
-			err = fmt.Errorf("unknown QMP status '%s' for QMP socket '%s'; (JSON response: '%s')",
-				result.Return.Status, socket, raw)
+			err = fmt.Errorf("[attempt %d] failed to parse QMP status response for QMP socket '%s': err: '%v'; (JSON response: '%s')",
+				attempt, socket, err, raw)
+			errs = errors.Join(errs, err)
+			time.Sleep(time.Second)
+			continue
 		}
-		return state, err
-	} else {
-		return types.UNKNOWN, err
+		var matched bool
+		if state, matched = qmpStatusMap[result.Return.Status]; !matched {
+			err = fmt.Errorf("[attempt %d] unknown QMP status '%s' for QMP socket '%s'; (JSON response: '%s')",
+				attempt, result.Return.Status, socket, raw)
+			errs = errors.Join(errs, err)
+			time.Sleep(time.Second)
+			continue
+		}
+
+		// Success
+		break
 	}
+
+	return state, errs
 }
 
 func qmpEventHandler(listenerSocket, executorSocket string) {