Retry enrollment requests when an error is returned, add enrollment timeout (elastic#8056)

Retry enrollment requests when an error is returned until a timeout is reached.
Add --enroll-timeout and FLEET_ENROLL_TIMEOUT to control how long the timeout is; default 10m.
A negative value disables the timeout.

Author: michel-laterman

changelog/fragments/1746113477-Retry-enrollment-for-all-errors.yaml

@@ -0,0 +1,36 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Retry enrollment requests on any error
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  If any error is encountered during an attempted enrollment, the elastic-agent
+  will backoff and retry. Add a new --enroll-timeout flag and
+  FLEET_ENROLL_TIMEOUT env var to set how long it tries for, default 10m. A
+  negative value disables the timeout.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/8056
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

internal/pkg/agent/cmd/container.go

@@ -81,6 +81,7 @@ The following actions are possible and grouped based on the actions.
   FLEET_ENROLL - set to 1 for enrollment into Fleet Server. If not set, Elastic Agent is run in standalone mode.
   FLEET_URL - URL of the Fleet Server to enroll into
   FLEET_ENROLLMENT_TOKEN - token to use for enrollment. This is not needed in case FLEET_SERVER_ENABLED and FLEET_ENROLL is set. Then the token is fetched from Kibana.
+  FLEET_ENROLL_TIMEOUT - The timeout duration for the enroll commnd. Defaults to 10m. A negative value disables the timeout.
   FLEET_CA - path to certificate authority to use with communicate with Fleet Server [$KIBANA_CA]
   FLEET_INSECURE - communicate with Fleet with either insecure HTTP or unverified HTTPS
   ELASTIC_AGENT_CERT - path to certificate to use for connecting to fleet-server.
@@ -390,7 +391,7 @@ func ensureServiceToken(streams *cli.IOStreams, cfg *setupConfig) error {
 	if err != nil {
 		return err
 	}
-	code, r, err := client.Connection.Request("POST", "/api/fleet/service_tokens", nil, nil, nil)
+	code, r, err := client.Request("POST", "/api/fleet/service_tokens", nil, nil, nil)
 	if err != nil {
 		return fmt.Errorf("request to get security token from Kibana failed: %w", err)
 	}
@@ -517,6 +518,10 @@ func buildEnrollArgs(cfg setupConfig, token string, policyID string) ([]string,
 		args = append(args, "--daemon-timeout")
 		args = append(args, cfg.Fleet.DaemonTimeout.String())
 	}
+	if cfg.Fleet.EnrollTimeout != 0 {
+		args = append(args, "--enroll-timeout")
+		args = append(args, cfg.Fleet.EnrollTimeout.String())
+	}
 	if cfg.Fleet.Cert != "" {
 		args = append(args, "--elastic-agent-cert", cfg.Fleet.Cert)
 	}
@@ -693,14 +698,14 @@ func isTrue(val string) bool {
 func performGET(cfg setupConfig, client *kibana.Client, path string, response interface{}, writer io.Writer, msg string) error {
 	var lastErr error
 	for i := 0; i < cfg.Kibana.RetryMaxCount; i++ {
-		code, result, err := client.Connection.Request("GET", path, nil, nil, nil)
+		code, result, err := client.Request("GET", path, nil, nil, nil)
 		if err != nil || code != 200 {
 			if err != nil {
 				err = fmt.Errorf("http GET request to %s%s fails: %w. Response: %s",
-					client.Connection.URL, path, err, truncateString(result))
+					client.URL, path, err, truncateString(result))
 			} else {
 				err = fmt.Errorf("http GET request to %s%s fails. StatusCode: %d Response: %s",
-					client.Connection.URL, path, code, truncateString(result))
+					client.URL, path, code, truncateString(result))
 			}
 			fmt.Fprintf(writer, "%s failed: %s\n", msg, err)
 			<-time.After(cfg.Kibana.RetrySleepDuration)
@@ -721,7 +726,7 @@ func truncateString(b []byte) string {
 		runes = append(runes[:maxLength], []rune("... (truncated)")...)
 	}
 
-	return strings.Replace(string(runes), "\n", " ", -1)
+	return strings.ReplaceAll(string(runes), "\n", " ")
 }
 
 // runLegacyAPMServer extracts the bundled apm-server from elastic-agent

internal/pkg/agent/cmd/enroll.go

@@ -14,6 +14,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -95,6 +96,7 @@ func addEnrollFlags(cmd *cobra.Command) {
 	cmd.Flags().StringSliceP("proxy-header", "", []string{}, "Proxy headers used with CONNECT request: when bootstrapping Fleet Server, it's the proxy used by Fleet Server to connect to Elasticsearch; when enrolling the Elastic Agent to Fleet Server, it's the proxy used by the Elastic Agent to connect to Fleet Server")
 	cmd.Flags().BoolP("delay-enroll", "", false, "Delays enrollment to occur on first start of the Elastic Agent service")
 	cmd.Flags().DurationP("daemon-timeout", "", 0, "Timeout waiting for Elastic Agent daemon")
+	cmd.Flags().DurationP("enroll-timeout", "", 10*time.Minute, "Timeout waiting for Elastic Agent enroll command. A negative value disables the timeout.")
 	cmd.Flags().DurationP("fleet-server-timeout", "", 0, "When bootstrapping Fleet Server, timeout waiting for Fleet Server to be ready to start enrollment")
 	cmd.Flags().Bool("skip-daemon-reload", false, "Skip daemon reload after enrolling")
 	cmd.Flags().StringSliceP("tag", "", []string{}, "User-set tags")
@@ -205,6 +207,7 @@ func buildEnrollmentFlags(cmd *cobra.Command, url string, token string) []string
 	fProxyHeaders, _ := cmd.Flags().GetStringSlice("proxy-header")
 	delayEnroll, _ := cmd.Flags().GetBool("delay-enroll")
 	daemonTimeout, _ := cmd.Flags().GetDuration("daemon-timeout")
+	enrollTimeout, _ := cmd.Flags().GetDuration("enroll-timeout")
 	fTimeout, _ := cmd.Flags().GetDuration("fleet-server-timeout")
 	skipDaemonReload, _ := cmd.Flags().GetBool("skip-daemon-reload")
 	fTags, _ := cmd.Flags().GetStringSlice("tag")
@@ -285,6 +288,10 @@ func buildEnrollmentFlags(cmd *cobra.Command, url string, token string) []string
 		args = append(args, "--daemon-timeout")
 		args = append(args, daemonTimeout.String())
 	}
+	if enrollTimeout != 0 {
+		args = append(args, "--enroll-timeout")
+		args = append(args, enrollTimeout.String())
+	}
 	if fTimeout != 0 {
 		args = append(args, "--fleet-server-timeout")
 		args = append(args, fTimeout.String())
@@ -461,6 +468,7 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command) error {
 	proxyHeaders, _ := cmd.Flags().GetStringSlice("proxy-header")
 	delayEnroll, _ := cmd.Flags().GetBool("delay-enroll")
 	daemonTimeout, _ := cmd.Flags().GetDuration("daemon-timeout")
+	enrollTimeout, _ := cmd.Flags().GetDuration("enroll-timeout")
 	fTimeout, _ := cmd.Flags().GetDuration("fleet-server-timeout")
 	skipDaemonReload, _ := cmd.Flags().GetBool("skip-daemon-reload")
 	tags, _ := cmd.Flags().GetStringSlice("tag")
@@ -475,6 +483,12 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command) error {
 
 	ctx := handleSignal(context.Background())
 
+	if enrollTimeout > 0 {
+		eCtx, cancel := context.WithTimeout(ctx, enrollTimeout)
+		defer cancel()
+		ctx = eCtx
+	}
+
 	// On MacOS Ventura and above, fixing the permissions on enrollment during installation fails with the error:
 	//  Error: failed to fix permissions: chown /Library/Elastic/Agent/data/elastic-agent-c13f91/elastic-agent.app: operation not permitted
 	// This is because we are fixing permissions twice, once during installation and again during the enrollment step.

internal/pkg/agent/cmd/enroll_cmd.go

@@ -543,25 +543,23 @@ func (c *enrollCmd) enrollWithBackoff(ctx context.Context, persistentConfig map[
 	defer close(signal)
 	backExp := c.backoffFactory(signal)
 
+RETRYLOOP:
 	for {
-		retry := false
 		switch {
 		case errors.Is(err, fleetapi.ErrTooManyRequests):
 			c.log.Warn("Too many requests on the remote server, will retry in a moment.")
-			retry = true
 		case errors.Is(err, fleetapi.ErrConnRefused):
 			c.log.Warn("Remote server is not ready to accept connections(Connection Refused), will retry in a moment.")
-			retry = true
 		case errors.Is(err, fleetapi.ErrTemporaryServerError):
 			c.log.Warnf("Remote server failed to handle the request(%s), will retry in a moment.", err.Error())
-			retry = true
+		case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), err == nil:
+			break RETRYLOOP
 		case err != nil:
-			c.log.Warnf("Enrollment failed: %s", err.Error())
+			c.log.Warnf("Error detected: %s, will retry in a moment.", err.Error())
 		}
-		if !retry {
-			break
+		if !backExp.Wait() {
+			break RETRYLOOP
 		}
-		backExp.Wait()
 		c.log.Infof("Retrying enrollment to URL: %s", c.client.URI())
 		err = c.enroll(ctx, persistentConfig)
 	}
@@ -597,9 +595,7 @@ func (c *enrollCmd) enroll(ctx context.Context, persistentConfig map[string]inte
 
 	resp, err := cmd.Execute(ctx, r)
 	if err != nil {
-		return errors.New(err,
-			"fail to execute request to fleet-server",
-			errors.TypeNetwork)
+		return fmt.Errorf("failed to execute request to fleet-server: %w", err)
 	}
 
 	fleetConfig, err := createFleetConfigFromEnroll(resp.Item.AccessAPIKey, c.options.EnrollAPIKey, c.options.ReplaceToken, c.remoteConfig)