Fix enroll to return 503 when ES returns 503. (#5232)

* Fix enroll to return 503 when ES returns 503.

* Fix import.

* Add changelog.

* Wrap error in invalidate.

Author: blakerouse

changelog/fragments/1754931919-Fix-503-handling-in-enrollment.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fix 503 handling in enrollment
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: fleet-server
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/fleet-server/pull/5232
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/fleet-server/issues/5197

internal/pkg/apikey/create.go

@@ -12,6 +12,8 @@ import (
 
 	"github.com/elastic/go-elasticsearch/v8"
 	"github.com/elastic/go-elasticsearch/v8/esapi"
+
+	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 )
 
 // Create generates a new APIKey in Elasticsearch using the given client.
@@ -48,7 +50,7 @@ func Create(ctx context.Context, client *elasticsearch.Client, name, ttl, refres
 	defer res.Body.Close()
 
 	if res.IsError() {
-		return nil, fmt.Errorf("fail CreateAPIKey: %s", res.String())
+		return nil, fmt.Errorf("fail CreateAPIKey: %w", es.TranslateError(res.StatusCode, nil))
 	}
 
 	type APIKeyResponse struct {

internal/pkg/apikey/invalidate.go

@@ -12,6 +12,8 @@ import (
 
 	"github.com/elastic/go-elasticsearch/v8"
 	"github.com/elastic/go-elasticsearch/v8/esapi"
+
+	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 )
 
 // Invalidate invalidates the provided API keys by ID.
@@ -44,7 +46,7 @@ func Invalidate(ctx context.Context, client *elasticsearch.Client, ids ...string
 	defer res.Body.Close()
 
 	if res.IsError() {
-		return fmt.Errorf("fail InvalidateAPIKey: %s", res.String())
+		return fmt.Errorf("fail InvalidateAPIKey: %w", es.TranslateError(res.StatusCode, nil))
 	}
 
 	return nil

testing/e2e/stand_alone_test.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 	"html/template"
+	"net/http"
 	"net/http/httptest"
 	"os"
 	"os/exec"
@@ -23,6 +24,7 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/elastic/elastic-agent-client/v7/pkg/client"
+	"github.com/elastic/fleet-server/pkg/api"
 	"github.com/elastic/fleet-server/testing/e2e/api_version"
 	"github.com/elastic/fleet-server/testing/e2e/scaffold"
 	"github.com/elastic/fleet-server/v7/version"
@@ -364,6 +366,135 @@ func (suite *StandAloneSuite) TestElasticsearch429OnStartup() {
 	cmd.Wait()
 }
 
+// TestElasticsearch503OnStartup will check to ensure fleet-server functions as expected (does not crash)
+// if Elasticsearch returns 503s on startup.
+func (suite *StandAloneSuite) TestElasticsearch503OnStartup() {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+
+	// Create a proxy that returns 503s
+	proxy := NewStatusProxy(suite.T(), http.StatusServiceUnavailable)
+	proxy.Enable()
+	server := httptest.NewServer(proxy)
+
+	// Create a config file from a template in the test temp dir
+	dir := suite.T().TempDir()
+	tpl, err := template.ParseFiles(filepath.Join("testdata", "stand-alone-http-proxy.tpl"))
+	suite.Require().NoError(err)
+	f, err := os.Create(filepath.Join(dir, "config.yml"))
+	suite.Require().NoError(err)
+	err = tpl.Execute(f, map[string]string{
+		"Hosts":        suite.ESHosts,
+		"ServiceToken": suite.ServiceToken,
+		"Proxy":        server.URL,
+	})
+	f.Close()
+	suite.Require().NoError(err)
+
+	// Run the fleet-server binary
+	cmd := exec.CommandContext(ctx, suite.binaryPath, "-c", filepath.Join(dir, "config.yml"))
+	//cmd.Stderr = os.Stderr // NOTE: This can be uncommented to put out logs
+	cmd.Cancel = func() error {
+		return cmd.Process.Signal(syscall.SIGTERM)
+	}
+	cmd.Env = []string{"GOCOVERDIR=" + suite.CoverPath}
+	suite.T().Log("Starting fleet-server")
+	err = cmd.Start()
+	suite.Require().NoError(err)
+
+	// FIXME timeout to make sure fleet-server has started
+	time.Sleep(5 * time.Second)
+	suite.T().Log("Checking fleet-server status")
+	// Wait to check that it is Starting.
+	suite.FleetServerStatusIs(ctx, "http://localhost:8220", client.UnitStateStarting) // fleet-server returns 503:starting if upstream ES returns 429.
+
+	// Disable proxy and ensure fleet-server recovers
+	suite.T().Log("Disable proxy")
+	proxy.Disable()
+	suite.FleetServerStatusIs(ctx, "http://localhost:8220", client.UnitStateHealthy)
+
+	cancel()
+	cmd.Wait()
+}
+
+// TestElasticsearch503OnEnroll will check to ensure fleet-server returns a 503 error when elasticsearch returns a
+// 503 gateway error.
+func (suite *StandAloneSuite) TestElasticsearch503OnEnroll() {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+
+	// Create a proxy that returns 503s
+	proxy := NewStatusProxy(suite.T(), http.StatusServiceUnavailable)
+	proxy.Disable() // start off
+	server := httptest.NewServer(proxy)
+
+	// Create a config file from a template in the test temp dir
+	dir := suite.T().TempDir()
+	tpl, err := template.ParseFiles(filepath.Join("testdata", "stand-alone-http-proxy.tpl"))
+	suite.Require().NoError(err)
+	f, err := os.Create(filepath.Join(dir, "config.yml"))
+	suite.Require().NoError(err)
+	err = tpl.Execute(f, map[string]any{
+		"Hosts":                    suite.ESHosts,
+		"ServiceToken":             suite.ServiceToken,
+		"Proxy":                    server.URL,
+		"StaticPolicyTokenEnabled": true,
+		"StaticTokenKey":           "abcdefg",
+		"StaticPolicyID":           "dummy-policy",
+	})
+	f.Close()
+	suite.Require().NoError(err)
+
+	// Run the fleet-server binary
+	cmd := exec.CommandContext(ctx, suite.binaryPath, "-c", filepath.Join(dir, "config.yml"))
+	cmd.Stderr = os.Stderr // NOTE: This can be uncommented to put out logs
+	cmd.Cancel = func() error {
+		return cmd.Process.Signal(syscall.SIGTERM)
+	}
+	cmd.Env = []string{"GOCOVERDIR=" + suite.CoverPath}
+	suite.T().Log("Starting fleet-server")
+	err = cmd.Start()
+	suite.Require().NoError(err)
+	defer func() {
+		cancel()
+		cmd.Wait()
+	}()
+
+	// FIXME timeout to make sure fleet-server has started
+	time.Sleep(5 * time.Second)
+	suite.T().Log("Checking fleet-server status")
+	// Should start healthy as the proxy is disabled.
+	suite.FleetServerStatusIs(ctx, "http://localhost:8220", client.UnitStateHealthy)
+
+	// Ensure enrollment works correctly
+	suite.T().Log("Checking enrollment works")
+	enrollmentToken := suite.GetEnrollmentTokenForPolicyID(ctx, "dummy-policy")
+	tester := api_version.NewClientAPITesterCurrent(
+		suite.Scaffold,
+		"http://localhost:8220",
+		enrollmentToken,
+	)
+	tester.Enroll(ctx, enrollmentToken)
+
+	// Enable the proxy which will cause enrollment to fail
+	suite.T().Log("Force 503 error from proxy")
+	proxy.Enable()
+
+	// Perform enrollment again should error with 503
+	suite.T().Log("Perform enrollment again")
+	client, err := api.NewClientWithResponses("http://localhost:8220", api.WithHTTPClient(tester.Client), api.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error {
+		req.Header.Set("Authorization", "ApiKey "+enrollmentToken)
+		return nil
+	}))
+	tester.Require().NoError(err)
+	enrollResp, err := client.AgentEnrollWithResponse(ctx,
+		&api.AgentEnrollParams{UserAgent: "elastic agent " + version.DefaultVersion},
+		api.AgentEnrollJSONRequestBody{
+			Type: api.PERMANENT,
+		},
+	)
+	tester.Require().NoError(err)
+	tester.Require().Equal(http.StatusServiceUnavailable, enrollResp.StatusCode())
+}
+
 // TestElasticsearchTimeoutOnStartup will check to ensure fleet-server functions as expected (does not crash)
 // if Elasticsearch times out on startup.
 func (suite *StandAloneSuite) TestElasticsearchTimeoutOnStartup() {