Adding upgrade retry

Author: ewoutp

client/api.go

@@ -53,6 +53,10 @@ type API interface {
 	// StartDatabaseUpgrade is called to start the upgrade process
 	StartDatabaseUpgrade(ctx context.Context, force bool) error
 
+	// RetryDatabaseUpgrade resets a failure mark in the existing upgrade plan
+	// such that the starters will retry the upgrade once more.
+	RetryDatabaseUpgrade(ctx context.Context) error
+
 	// Status returns the status of any upgrade plan
 	UpgradeStatus(context.Context) (UpgradeStatus, error)
 }

client/client.go

@@ -25,7 +25,6 @@ package client
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -225,6 +224,29 @@ func (c *client) StartDatabaseUpgrade(ctx context.Context, force bool) error {
 	return nil
 }
 
+// RetryDatabaseUpgrade resets a failure mark in the existing upgrade plan
+// such that the starters will retry the upgrade once more.
+func (c *client) RetryDatabaseUpgrade(ctx context.Context) error {
+	url := c.createURL("/database-auto-upgrade", nil)
+
+	req, err := http.NewRequest("PUT", url, nil)
+	if err != nil {
+		return maskAny(err)
+	}
+	if ctx != nil {
+		req = req.WithContext(ctx)
+	}
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return maskAny(err)
+	}
+	if err := c.handleResponse(resp, "PUT", url, nil); err != nil {
+		return maskAny(err)
+	}
+
+	return nil
+}
+
 // Status returns the status of any upgrade plan
 func (c *client) UpgradeStatus(ctx context.Context) (UpgradeStatus, error) {
 	url := c.createURL("/database-auto-upgrade", nil)
@@ -258,11 +280,16 @@ func (c *client) handleResponse(resp *http.Response, method, url string, result
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		/*var er ErrorResponse
+		var er ErrorResponse
 		if err := json.Unmarshal(body, &er); err == nil {
-			return &er
-		}*/
-		return maskAny(fmt.Errorf("Invalid status %d", resp.StatusCode))
+			return maskAny(StatusError{
+				StatusCode: resp.StatusCode,
+				message:    er.Error,
+			})
+		}
+		return maskAny(StatusError{
+			StatusCode: resp.StatusCode,
+		})
 	}
 
 	// Got a success status

service/server.go

@@ -640,6 +640,31 @@ func (s *httpServer) databaseAutoUpgradeHandler(w http.ResponseWriter, r *http.R
 				w.Write([]byte("OK"))
 			}
 		}
+	case "PUT":
+		// Retry the upgrade process
+		if !isRunningMaster {
+			// We're not the starter leader.
+			// Forward the request to the leader.
+			c, err := createMasterClient()
+			if err != nil {
+				handleError(w, err)
+			} else {
+				if err := c.RetryDatabaseUpgrade(ctx); err != nil {
+					handleError(w, err)
+				} else {
+					w.WriteHeader(http.StatusOK)
+					w.Write([]byte("OK"))
+				}
+			}
+		} else {
+			// We're the starter leader, process the request
+			if err := s.context.UpgradeManager().RetryDatabaseUpgrade(ctx); err != nil {
+				handleError(w, err)
+			} else {
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte("OK"))
+			}
+		}
 	case "GET":
 		if status, err := s.context.UpgradeManager().Status(ctx); err != nil {
 			handleError(w, err)

service/upgrade_manager.go

@@ -44,6 +44,10 @@ type UpgradeManager interface {
 	// StartDatabaseUpgrade is called to start the upgrade process
 	StartDatabaseUpgrade(ctx context.Context, force bool) error
 
+	// RetryDatabaseUpgrade resets a failure mark in the existing upgrade plan
+	// such that the starters will retry the upgrade once more.
+	RetryDatabaseUpgrade(ctx context.Context) error
+
 	// Status returns the status of any upgrade plan
 	Status(context.Context) (client.UpgradeStatus, error)
 
@@ -124,6 +128,14 @@ func (p UpgradePlan) IsFailed() bool {
 	return false
 }
 
+// ResetFailures resets all Failures field to 0.
+func (p *UpgradePlan) ResetFailures() {
+	for _, e := range p.Entries {
+		e.Failures = 0
+		e.Reason = ""
+	}
+}
+
 // UpgradeEntryType is a strongly typed upgrade plan item
 type UpgradeEntryType string
 
@@ -352,6 +364,49 @@ func (m *upgradeManager) StartDatabaseUpgrade(ctx context.Context, force bool) e
 	return nil
 }
 
+// RetryDatabaseUpgrade resets a failure mark in the existing upgrade plan
+// such that the starters will retry the upgrade once more.
+func (m *upgradeManager) RetryDatabaseUpgrade(ctx context.Context) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	// Fetch mode
+	_, _, mode := m.upgradeManagerContext.ClusterConfig()
+
+	if !mode.HasAgency() {
+		// Without an agency there is not upgrade plan to retry
+		return errors.Wrap(client.BadRequestError, "Retry needs an agency")
+	}
+
+	// Retry upgrade with agency.
+	plan, err := m.readUpgradePlan(ctx)
+	if agency.IsKeyNotFound(err) {
+		// There is no upgrade plan
+		return errors.Wrap(client.BadRequestError, "There is no upgrade plan")
+	}
+	if err != nil {
+		// Failed to read upgrade plan
+		return errors.Wrap(err, "Failed to read upgrade plan")
+	}
+
+	// Check failure status
+	if !plan.IsFailed() {
+		return errors.Wrap(client.BadRequestError, "The upgrade plan has not failed")
+	}
+
+	// Reset failures and write plan
+	plan.ResetFailures()
+	overwrite := false
+	if _, err := m.writeUpgradePlan(ctx, plan, overwrite); err != nil {
+		return errors.Wrap(err, "Failed to write upgrade plan")
+	}
+
+	// Inform user
+	m.log.Info().Msg("Reset failures in upgrade plan so it can be retried")
+
+	return nil
+}
+
 // Status returns the current status of the upgrade process.
 func (m *upgradeManager) Status(ctx context.Context) (client.UpgradeStatus, error) {
 	plan, err := m.readUpgradePlan(ctx)

upgrade.go

@@ -38,31 +38,57 @@ import (
 var (
 	cmdUpgrade = &cobra.Command{
 		Use:   "upgrade",
-		Short: "Start the ArangoDB starter in the background",
+		Short: "Upgrade an ArangoDB deployment to a new version",
 		Run:   cmdUpgradeRun,
 	}
+	cmdRetry = &cobra.Command{
+		Use:   "retry",
+		Short: "Retry an operation",
+		Run:   cmdShowUsage,
+	}
+	cmdRetryUpgrade = &cobra.Command{
+		Use:   "upgrade",
+		Short: "Retry a failed upgrade of an ArangoDB deployment to a new version",
+		Run:   cmdRetryUpgradeRun,
+	}
 	upgradeOptions struct {
 		starterEndpoint string
 	}
+	retryUpgradeOptions struct {
+		starterEndpoint string
+	}
 )
 
 func init() {
 	f := cmdUpgrade.Flags()
 	f.StringVar(&upgradeOptions.starterEndpoint, "starter.endpoint", "", "The endpoint of the starter to connect to. E.g. http://localhost:8528")
 
+	f = cmdRetryUpgrade.Flags()
+	f.StringVar(&retryUpgradeOptions.starterEndpoint, "starter.endpoint", "", "The endpoint of the starter to connect to. E.g. http://localhost:8528")
+
 	cmdMain.AddCommand(cmdUpgrade)
+	cmdMain.AddCommand(cmdRetry)
+	cmdRetry.AddCommand(cmdRetryUpgrade)
 }
 
 func cmdUpgradeRun(cmd *cobra.Command, args []string) {
+	runUpgrade(upgradeOptions.starterEndpoint, false, false)
+}
+
+func cmdRetryUpgradeRun(cmd *cobra.Command, args []string) {
+	runUpgrade(retryUpgradeOptions.starterEndpoint, false, true)
+}
+
+func runUpgrade(starterEndpoint string, force, retry bool) {
 	// Setup logging
 	consoleOnly := true
 	configureLogging(consoleOnly)
 
 	// Check options
-	if upgradeOptions.starterEndpoint == "" {
+	if starterEndpoint == "" {
 		log.Fatal().Msg("--starter.endpoint must be set")
 	}
-	ep, err := url.Parse(upgradeOptions.starterEndpoint)
+	ep, err := url.Parse(starterEndpoint)
 	if err != nil {
 		log.Fatal().Err(err).Msg("--starter.endpoint is invalid")
 	}
@@ -73,11 +99,17 @@ func cmdUpgradeRun(cmd *cobra.Command, args []string) {
 		log.Fatal().Err(err).Msg("Failed to create Starter client")
 	}
 	ctx := context.Background()
-	force := false
-	if err := c.StartDatabaseUpgrade(ctx, force); err != nil {
-		log.Fatal().Err(err).Msg("Failed to starter database automatic upgrade")
+	if retry {
+		if err := c.RetryDatabaseUpgrade(ctx); err != nil {
+			log.Fatal().Err(err).Msg("Failed to retry database automatic upgrade")
+		}
+		log.Info().Msg("Database automatic upgrade has been restarted")
+	} else {
+		if err := c.StartDatabaseUpgrade(ctx, force); err != nil {
+			log.Fatal().Err(err).Msg("Failed to start database automatic upgrade")
+		}
+		log.Info().Msg("Database automatic upgrade has been started")
 	}
-	log.Info().Msg("Database automatic upgrade has been started")
 
 	// Wait for the upgrade to finish
 	remaining := ""