Add Fleet-managed Agent FIPS upgrade test

Author: ycombinator

pkg/testing/define/define.go

@@ -92,24 +92,40 @@ func NewFixtureFromLocalBuild(t *testing.T, version string, opts ...atesting.Fix
 		buildsDir = filepath.Join(projectDir, "build", "distributions")
 	}
 
-	return NewFixtureWithBinary(t, version, "elastic-agent", buildsDir, opts...)
+	return NewFixtureWithBinary(t, version, "elastic-agent", buildsDir, false, opts...)
+}
 
+// NewFixtureFromLocalFIPSBuild returns a new FIPS-capable Elastic Agent testing fixture with a LocalFetcher
+// and the agent logging to the test logger.
+func NewFixtureFromLocalFIPSBuild(t *testing.T, version string, opts ...atesting.FixtureOpt) (*atesting.Fixture, error) {
+	buildsDir := os.Getenv("AGENT_BUILD_DIR")
+	if buildsDir == "" {
+		projectDir, err := findProjectRoot()
+		if err != nil {
+			return nil, err
+		}
+		buildsDir = filepath.Join(projectDir, "build", "distributions")
+	}
+
+	return NewFixtureWithBinary(t, version, "elastic-agent", buildsDir, true, opts...)
 }
 
 // NewFixtureWithBinary returns a new Elastic Agent testing fixture with a LocalFetcher and
 // the agent logging to the test logger.
-func NewFixtureWithBinary(t *testing.T, version string, binary string, buildsDir string, opts ...atesting.FixtureOpt) (*atesting.Fixture, error) {
+func NewFixtureWithBinary(t *testing.T, version string, binary string, buildsDir string, fips bool, opts ...atesting.FixtureOpt) (*atesting.Fixture, error) {
 	ver, err := semver.ParseVersion(version)
 	if err != nil {
 		return nil, fmt.Errorf("%q is an invalid agent version: %w", version, err)
 	}
 
-	var binFetcher atesting.Fetcher
+	localFetcherOpts := []atesting.LocalFetcherOpt{atesting.WithCustomBinaryName(binary)}
 	if ver.IsSnapshot() {
-		binFetcher = atesting.LocalFetcher(buildsDir, atesting.WithLocalSnapshotOnly(), atesting.WithCustomBinaryName(binary))
-	} else {
-		binFetcher = atesting.LocalFetcher(buildsDir, atesting.WithCustomBinaryName(binary))
+		localFetcherOpts = append(localFetcherOpts, atesting.WithLocalSnapshotOnly())
+	}
+	if fips {
+		localFetcherOpts = append(localFetcherOpts, atesting.WithLocalFIPSOnly())
 	}
+	binFetcher := atesting.LocalFetcher(buildsDir, localFetcherOpts...)
 
 	opts = append(opts, atesting.WithFetcher(binFetcher), atesting.WithLogOutput())
 	if binary != "elastic-agent" {

pkg/testing/fetcher_local.go

@@ -18,27 +18,35 @@ import (
 type localFetcher struct {
 	dir          string
 	snapshotOnly bool
+	fipsOnly     bool
 	binaryName   string
 }
 
-type localFetcherOpt func(f *localFetcher)
+type LocalFetcherOpt func(f *localFetcher)
 
 // WithLocalSnapshotOnly sets the LocalFetcher to only pull the snapshot build.
-func WithLocalSnapshotOnly() localFetcherOpt {
+func WithLocalSnapshotOnly() LocalFetcherOpt {
 	return func(f *localFetcher) {
 		f.snapshotOnly = true
 	}
 }
 
+// WithLocalFIPSOnly sets the LocalFetcher to only pull a FIPS-compliant build.
+func WithLocalFIPSOnly() LocalFetcherOpt {
+	return func(f *localFetcher) {
+		f.fipsOnly = true
+	}
+}
+
 // WithCustomBinaryName sets the binary to a custom name, the default is `elastic-agent`
-func WithCustomBinaryName(name string) localFetcherOpt {
+func WithCustomBinaryName(name string) LocalFetcherOpt {
 	return func(f *localFetcher) {
 		f.binaryName = name
 	}
 }
 
 // LocalFetcher returns a fetcher that pulls the binary of the Elastic Agent from a local location.
-func LocalFetcher(dir string, opts ...localFetcherOpt) Fetcher {
+func LocalFetcher(dir string, opts ...LocalFetcherOpt) Fetcher {
 	f := &localFetcher{
 		dir:        dir,
 		binaryName: "elastic-agent",
@@ -56,6 +64,7 @@ func (f *localFetcher) Name() string {
 
 // Fetch fetches the Elastic Agent and places the resulting binary at the path.
 func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) {
+	prefix := GetPackagePrefix(f.fipsOnly)
 	suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat)
 	if err != nil {
 		return nil, err
@@ -66,7 +75,7 @@ func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architec
 		return nil, fmt.Errorf("invalid version: %q: %w", ver, err)
 	}
 
-	mainBuildfmt := "%s-%s-%s"
+	mainBuildfmt := "%s-%s%s-%s"
 	if f.snapshotOnly && !ver.IsSnapshot() {
 		if ver.Prerelease() == "" {
 			ver = semver.NewParsedSemVer(ver.Major(), ver.Minor(), ver.Patch(), "SNAPSHOT", ver.BuildMetadata())
@@ -85,10 +94,10 @@ func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architec
 	}
 
 	if ver.IsSnapshot() && !matchesEarlyReleaseVersion {
-		build := fmt.Sprintf(mainBuildfmt, f.binaryName, ver.VersionWithPrerelease(), suffix)
+		build := fmt.Sprintf(mainBuildfmt, f.binaryName, prefix, ver.VersionWithPrerelease(), suffix)
 		buildPath = filepath.Join(ver.BuildMetadata(), build)
 	} else {
-		buildPath = fmt.Sprintf(mainBuildfmt, f.binaryName, ver.String(), suffix)
+		buildPath = fmt.Sprintf(mainBuildfmt, f.binaryName, prefix, ver.String(), suffix)
 	}
 
 	fullPath := filepath.Join(f.dir, buildPath)

pkg/testing/fetcher_local_test.go

@@ -57,7 +57,7 @@ func TestLocalFetcher(t *testing.T) {
 	tcs := []struct {
 		name     string
 		version  string
-		opts     []localFetcherOpt
+		opts     []LocalFetcherOpt
 		want     []byte
 		wantHash []byte
 	}{
@@ -69,7 +69,7 @@ func TestLocalFetcher(t *testing.T) {
 		}, {
 			name:     "SnapshotOnly",
 			version:  baseVersion,
-			opts:     []localFetcherOpt{WithLocalSnapshotOnly()},
+			opts:     []LocalFetcherOpt{WithLocalSnapshotOnly()},
 			want:     snapshotContent,
 			wantHash: snapshotContentHash,
 		}, {
@@ -80,13 +80,13 @@ func TestLocalFetcher(t *testing.T) {
 		}, {
 			name:     "version with snapshot and SnapshotOnly",
 			version:  baseVersion + "-SNAPSHOT",
-			opts:     []localFetcherOpt{WithLocalSnapshotOnly()},
+			opts:     []LocalFetcherOpt{WithLocalSnapshotOnly()},
 			want:     snapshotContent,
 			wantHash: snapshotContentHash,
 		}, {
 			name:     "version with snapshot and build ID",
 			version:  baseVersion + "-SNAPSHOT+l5snflwr",
-			opts:     []localFetcherOpt{},
+			opts:     []LocalFetcherOpt{},
 			want:     snapshotContent,
 			wantHash: snapshotContentHash,
 		},

pkg/testing/fixture.go

@@ -50,6 +50,7 @@ type Fixture struct {
 	binaryName      string
 	runLength       time.Duration
 	additionalArgs  []string
+	fipsArtifact    bool
 
 	srcPackage string
 	workDir    string
@@ -145,6 +146,12 @@ func WithAdditionalArgs(args []string) FixtureOpt {
 	}
 }
 
+func WithFIPSArtifact() FixtureOpt {
+	return func(f *Fixture) {
+		f.fipsArtifact = true
+	}
+}
+
 // NewFixture creates a new fixture to setup and manage Elastic Agent.
 func NewFixture(t *testing.T, version string, opts ...FixtureOpt) (*Fixture, error) {
 	// we store the caller so the fixture can find the cache directory for the artifacts that

testing/integration/beats_serverless_test.go

@@ -78,7 +78,7 @@ func (runner *BeatRunner) SetupSuite() {
 	}
 	runner.T().Logf("running serverless tests with %s", runner.testbeatName)
 
-	agentFixture, err := define.NewFixtureWithBinary(runner.T(), define.Version(), runner.testbeatName, "/home/ubuntu", atesting.WithRunLength(time.Minute*3), atesting.WithAdditionalArgs([]string{"-E", "output.elasticsearch.allow_older_versions=true"}))
+	agentFixture, err := define.NewFixtureWithBinary(runner.T(), define.Version(), runner.testbeatName, "/home/ubuntu", false, atesting.WithRunLength(time.Minute*3), atesting.WithAdditionalArgs([]string{"-E", "output.elasticsearch.allow_older_versions=true"}))
 	runner.agentFixture = agentFixture
 	require.NoError(runner.T(), err)
 

testing/integration/upgrade_fleet_test.go

@@ -51,7 +51,7 @@ func TestFleetManagedUpgradeUnprivileged(t *testing.T) {
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
 	})
-	testFleetManagedUpgrade(t, info, true)
+	testFleetManagedUpgrade(t, info, true, false)
 }
 
 // TestFleetManagedUpgradePrivileged tests that the build under test can retrieve an action from
@@ -65,16 +65,63 @@ func TestFleetManagedUpgradePrivileged(t *testing.T) {
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
 	})
-	testFleetManagedUpgrade(t, info, false)
+	testFleetManagedUpgrade(t, info, false, false)
 }
 
-func testFleetManagedUpgrade(t *testing.T, info *define.Info, unprivileged bool) {
+// TestFleetManagedUpgradeUnprivilegedFIPS tests that the build under test can retrieve an action from
+// Fleet and perform the upgrade as an unprivileged FIPS-capable Elastic Agent. It does not need to test
+// all the combinations of versions as the standalone tests already perform those tests and
+// would be redundant.
+func TestFleetManagedUpgradeUnprivilegedFIPS(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Group: Fleet,
+		Stack: &define.Stack{},
+		Local: false, // requires Agent installation
+		Sudo:  true,  // requires Agent installation
+		// FIPS: true // TODO: uncomment when https://github.com/elastic/elastic-agent/pull/8083 is merged
+	})
+	postWatcherSuccessHook := upgradetest.PostUpgradeAgentIsFIPSCapable
+	upgradeOpts := []upgradetest.UpgradeOpt{upgradetest.WithPostWatcherSuccessHook(postWatcherSuccessHook)}
+	testFleetManagedUpgrade(t, info, true, true, upgradeOpts...)
+}
+
+// TestFleetManagedUpgradePrivileged tests that the build under test can retrieve an action from
+// Fleet and perform the upgrade as a privileged FIPS-capable Elastic Agent. It does not need to test all
+// the combinations of  versions as the standalone tests already perform those tests and
+// would be redundant.
+func TestFleetManagedUpgradePrivilegedFIPS(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Group: FleetPrivileged,
+		Stack: &define.Stack{},
+		Local: false, // requires Agent installation
+		Sudo:  true,  // requires Agent installation
+		// FIPS: true // TODO: uncomment when https://github.com/elastic/elastic-agent/pull/8083 is merged
+	})
+
+	// Check that new (post-upgrade) Agent is also FIPS-capable
+	postWatcherSuccessHook := upgradetest.PostUpgradeAgentIsFIPSCapable
+	upgradeOpts := []upgradetest.UpgradeOpt{upgradetest.WithPostWatcherSuccessHook(postWatcherSuccessHook)}
+	testFleetManagedUpgrade(t, info, false, true, upgradeOpts...)
+}
+
+func testFleetManagedUpgrade(t *testing.T, info *define.Info, unprivileged bool, fips bool, upgradeOpts ...upgradetest.UpgradeOpts) {
 	ctx, cancel := context.WithCancel(context.TODO())
 	defer cancel()
 
 	// Start at the build version as we want to test the retry
 	// logic that is in the build.
-	startFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+	fixtureOpts := make([]atesting.FixtureOpt, 0)
+	if fips {
+		fixtureOpts = append(fixtureOpts, atesting.WithFetcher())
+	}
+
+	var startFixture *testing.Fixture
+	var err error
+	if fips {
+		startFixture, err = define.NewFixtureFromLocalFIPSBuild(t, define.Version(), fixtureOpts...)
+	} else {
+		startFixture, err = define.NewFixtureFromLocalBuild(t, define.Version(), fixtureOpts...)
+	}
 	require.NoError(t, err)
 	err = startFixture.Prepare(ctx)
 	require.NoError(t, err)
@@ -105,7 +152,7 @@ func testFleetManagedUpgrade(t *testing.T, info *define.Info, unprivileged bool)
 	t.Logf("Testing Elastic Agent upgrade from %s to %s with Fleet...",
 		define.Version(), endVersionInfo.Binary.String())
 
-	testUpgradeFleetManagedElasticAgent(ctx, t, info, startFixture, endFixture, defaultPolicy(), unprivileged)
+	testUpgradeFleetManagedElasticAgent(ctx, t, info, startFixture, endFixture, defaultPolicy(), unprivileged, upgradeOpts...)
 }
 
 func TestFleetAirGappedUpgradeUnprivileged(t *testing.T) {
@@ -348,7 +395,15 @@ func testUpgradeFleetManagedElasticAgent(
 	startFixture *atesting.Fixture,
 	endFixture *atesting.Fixture,
 	policy kibana.AgentPolicy,
-	unprivileged bool) {
+	unprivileged bool,
+	opts ...upgradetest.UpgradeOpt,
+) {
+
+	// use the passed in options to perform the upgrade
+	var upgradeOpts upgradetest.UpgradeOpts
+	for _, o := range opts {
+		o(&upgradeOpts)
+	}
 
 	kibClient := info.KibanaClient
 
@@ -470,6 +525,12 @@ func testUpgradeFleetManagedElasticAgent(
 	// version, otherwise it's possible that it was rolled back to the original version
 	err = upgradetest.CheckHealthyAndVersion(ctx, startFixture, endVersionInfo.Binary)
 	assert.NoError(t, err)
+
+	if upgradeOpts.postWatcherSuccessHook != nil {
+		if err := upgradeOpts.postWatcherSuccessHook(ctx, endFixture); err != nil {
+			return fmt.Errorf("post watcher success hook failed: %w", err)
+		}
+	}
 }
 
 func defaultPolicy() kibana.AgentPolicy {

testing/integration/upgrade_standalone_fips_test.go

@@ -7,8 +7,6 @@
 package integration
 
 import (
-	"context"
-	"errors"
 	"fmt"
 	"testing"
 
@@ -33,6 +31,7 @@ func TestStandaloneUpgradeFIPStoFIPS(t *testing.T) {
 		OS: []define.OS{
 			{Type: define.Linux},
 		},
+		// FIPS: true // TODO: uncomment when https://github.com/elastic/elastic-agent/pull/8083 is merged
 	})
 
 	// parse the version we are testing
@@ -59,24 +58,7 @@ func TestStandaloneUpgradeFIPStoFIPS(t *testing.T) {
 	require.NoError(t, err)
 
 	// Check that new (post-upgrade) Agent is also FIPS-capable
-	postWatcherSuccessHook := func(ctx context.Context, endFixture *atesting.Fixture) error {
-		client := endFixture.Client()
-		err := client.Connect(ctx)
-		if err != nil {
-			return err
-		}
-
-		ver, err := client.Version(ctx)
-		if err != nil {
-			return err
-		}
-
-		if !ver.Fips {
-			return errors.New("expected upgraded Agent to be FIPS-capable")
-		}
-
-		return nil
-	}
+	postWatcherSuccessHook := upgradetest.PostUpgradeAgentIsFIPSCapable
 
 	for _, startVersion := range versionList {
 		// We need to start the upgrade from a FIPS-capable version