OCPBUGS-33789: [Nutanix] installer intermittent failed to upload bootstrap image data to prism-central when CAPI enabled

Author: yanhua121

pkg/infrastructure/nutanix/clusterapi/clusterapi.go

@@ -3,9 +3,12 @@ package clusterapi
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
 
 	nutanixclientv3 "github.com/nutanix-cloud-native/prism-go-client/v3"
 	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/utils/ptr"
 
 	infracapi "github.com/openshift/installer/pkg/infrastructure/clusterapi"
@@ -146,44 +149,110 @@ func (p Provider) Ignition(ctx context.Context, in infracapi.IgnitionInput) ([]b
 	}
 	imgReq.Metadata = imgMeta
 
+	// Wait for successful creation of the bootstrap image object and upload the image data to it in PC.
+	// Put both createImage() and uploadImageData() in the same wait.ExponentialBackoffWithContext().
+	// Because if createImage() succeeds but uploadImageData() fails, we need to delete the image object
+	// and retry to call both createImage() and uploadImageData() again. The old-version prism-api server sometimes
+	// returns error for the uploadImage call and does not allow to retry the uploadImage call to the same image object.
+	timeout := 20 * time.Minute
+	if err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{
+		Duration: time.Minute * 4,
+		Factor:   float64(1.0),
+		Steps:    5,
+		Cap:      timeout,
+	}, func(ctx context.Context) (bool, error) {
+		// create the bootstrap image object in PC
+		imgUUID, err1 := createImage(ctx, nutanixCl, imgReq, imgName)
+		if err1 != nil {
+			logrus.Errorf("failed to create the bootstrap image object %s in PC: %v", imgName, err1)
+			// no need to retry if the error code is 401 or 403
+			if strings.Contains(err1.Error(), `"code": 401`) || strings.Contains(err1.Error(), `"code": 403`) {
+				return false, err1
+			}
+
+			// delete the image object if uuid is not empty
+			if imgUUID != "" {
+				if e2 := deleteImage(ctx, nutanixCl, imgUUID); e2 != nil {
+					logrus.Errorf("failed to delete image object %s (uuid: %s): %v", imgName, imgUUID, e2)
+				}
+			}
+			return false, nil
+		}
+
+		// upload the image data to the bootstrap image object in PC
+		err2 := uploadImageData(ctx, nutanixCl, imgName, imgUUID, imgPath)
+		if err2 != nil {
+			logrus.Errorf("failed to upload the bootstrap image %s data: %v", imgName, err2)
+			// no need to retry if the error code is 401 or 403
+			if strings.Contains(err2.Error(), `"code": 401`) || strings.Contains(err2.Error(), `"code": 403`) {
+				return false, err2
+			}
+
+			// delete the image object
+			if e2 := deleteImage(ctx, nutanixCl, imgUUID); e2 != nil {
+				logrus.Errorf("failed to delete image object %s (uuid: %s): %v", imgName, imgUUID, e2)
+			}
+			return false, nil
+		}
+
+		return true, nil
+	}); err != nil {
+		if wait.Interrupted(err) {
+			err = fmt.Errorf("timeout/interrupt to create/upload the bootstrap image object %s in PC within %v: %w", imgName, timeout, err)
+		} else {
+			err = fmt.Errorf("failed to create/upload the bootstrap image object %s in PC: %w", imgName, err)
+		}
+
+		return in.BootstrapIgnData, err
+	}
+	logrus.Infof("Successfully created the bootstrap image object %s and uploaded its image data", imgName)
+
+	return in.BootstrapIgnData, nil
+}
+
+// createImage creates the image object in PC, with the provided request input.
+// Returns the imageUUID if the image is created.
+func createImage(ctx context.Context, nutanixCl *nutanixclientv3.Client, imgReq *nutanixclientv3.ImageIntentInput, imgName string) (string, error) {
+	t1 := time.Now()
+
 	// create the image object.
 	respi, err := nutanixCl.V3.CreateImage(ctx, imgReq)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create the bootstrap image %q: %w", imgName, err)
+		return "", fmt.Errorf("failed to create the image %q: %w", imgName, err)
 	}
 	imgUUID := *respi.Metadata.UUID
 
 	if taskUUID, ok := respi.Status.ExecutionContext.TaskUUID.(string); ok {
-		logrus.Infof("creating the bootstrap image %s (uuid: %s), taskUUID: %s.", imgName, imgUUID, taskUUID)
+		logrus.Infof("creating the image %s (uuid: %s), taskUUID: %s", imgName, imgUUID, taskUUID)
 
-		// Wait till the image creation task is successed.
+		// Wait for the image creation task
 		if err = nutanixtypes.WaitForTask(nutanixCl.V3, taskUUID); err != nil {
-			err = fmt.Errorf("failed to create the bootstrap image %q: %w", imgName, err)
-			logrus.Errorf(err.Error())
-			return nil, err
+			err = fmt.Errorf("failed to create the image %s (uuid: %s), taskUUID: %s: %w", imgName, imgUUID, taskUUID, err)
+		} else {
+			logrus.Infof("created the image %s (uuid: %s). used_time %v", imgName, imgUUID, time.Since(t1))
 		}
-		logrus.Infof("created the bootstrap image %s (uuid: %s).", imgName, imgUUID)
 	} else {
 		err = fmt.Errorf("failed to convert the task UUID %v to string", respi.Status.ExecutionContext.TaskUUID)
-		logrus.Error(err)
-		return nil, err
 	}
 
+	return imgUUID, err
+}
+
+// uploadImageData upload the image data from the specified file path to the image object in PC.
+func uploadImageData(ctx context.Context, nutanixCl *nutanixclientv3.Client, imgName, imgUUID, imgPath string) error {
 	// upload the image data.
-	logrus.Infof("preparing to upload the bootstrap image %s (uuid: %s) data from file %s", imgName, imgUUID, imgPath)
-	err = nutanixCl.V3.UploadImage(ctx, imgUUID, imgPath)
+	logrus.Infof("preparing to upload the image %s (uuid: %s) data from file %s", imgName, imgUUID, imgPath)
+	t1 := time.Now()
+	err := nutanixCl.V3.UploadImage(ctx, imgUUID, imgPath)
 	if err != nil {
-		e1 := fmt.Errorf("failed to upload the bootstrap image data %q from filepath %s: %w", imgName, imgPath, err)
-		logrus.Error(e1)
-		return nil, e1
+		return fmt.Errorf("failed to upload the image data %q from filepath %s: %w  used_time %v", imgName, imgPath, err, time.Since(t1))
 	}
-	logrus.Infof("uploading the bootstrap image %s data", imgName)
+	logrus.Infof("uploading the image %s data. used_time %v", imgName, time.Since(t1))
+
 	// wait for the image data uploading task to complete.
 	respb, err := nutanixCl.V3.GetImage(ctx, imgUUID)
 	if err != nil {
-		e1 := fmt.Errorf("failed to get the bootstrap image %q. %w", imgName, err)
-		logrus.Error(e1)
-		return nil, e1
+		return fmt.Errorf("failed to get the image %q. %w", imgName, err)
 	}
 
 	if taskUUIDs, ok := respb.Status.ExecutionContext.TaskUUID.([]interface{}); ok {
@@ -193,18 +262,36 @@ func (p Provider) Ignition(ctx context.Context, in infracapi.IgnitionInput) ([]b
 				tUUIDs = append(tUUIDs, tUUIDstr)
 			}
 		}
-		logrus.Infof("waiting for the bootstrap image data uploading task to complete,  taskUUIDs: %v", tUUIDs)
+		logrus.Infof("waiting for the image data uploading task to complete,  taskUUIDs: %v", tUUIDs)
 		if err = nutanixtypes.WaitForTasks(nutanixCl.V3, tUUIDs); err != nil {
-			e1 := fmt.Errorf("failed to upload the bootstrap image data %q from filepath %s: %w", imgName, imgPath, err)
-			logrus.Error(e1)
-			return nil, e1
+			return fmt.Errorf("failed to upload the bootstrap image data %q from filepath %s: %w", imgName, imgPath, err)
 		}
-		logrus.Infof("completed uploading the bootstrap image data %s (uuid: %s)", imgName, imgUUID)
 	} else {
-		err = fmt.Errorf("failed to convert the taskUUIDs %v to array", respb.Status.ExecutionContext.TaskUUID)
-		logrus.Error(err)
-		return nil, err
+		return fmt.Errorf("failed to convert the taskUUIDs %v to array", respb.Status.ExecutionContext.TaskUUID)
 	}
 
-	return in.BootstrapIgnData, nil
+	return nil
+}
+
+// deleteImage deletes the image object with the given uuid in PC.
+func deleteImage(ctx context.Context, nutanixCl *nutanixclientv3.Client, imgUUID string) error {
+	logrus.Infof("preparing to delete the image with uuid %s", imgUUID)
+
+	respd, err := nutanixCl.V3.DeleteImage(ctx, imgUUID)
+	if err != nil {
+		return fmt.Errorf("failed to delete the image with uuid %s: %w", imgUUID, err)
+	}
+
+	if taskUUID, ok := respd.Status.ExecutionContext.TaskUUID.(string); ok {
+		logrus.Infof("deleting the image with uuid %s, taskUUID: %s", imgUUID, taskUUID)
+
+		// Wait till the image deletion task is successed.
+		if err = nutanixtypes.WaitForTask(nutanixCl.V3, taskUUID); err != nil {
+			return fmt.Errorf("failed to delete the image with uuid: %s, taskUUID: %s: %w", imgUUID, taskUUID, err)
+		}
+	} else {
+		return fmt.Errorf("failed to convert the task UUID %v to string", respd.Status.ExecutionContext.TaskUUID)
+	}
+
+	return nil
 }