[Integration Test] Ensure that upgrading a FIPS-capable Agent results in a FIPS-capable Agent (#7804)

* Adding skeleton for FIPS-to-FIPS upgrade test

* Expose FIPS compliance in GRPC client Version call response

* Test upgrade from FIPS to FIPS artifact

* Change assert to require

* Add postWatcherSuccessHook to upgrade test

* Refactor standalone upgrade test to take upgradeOpts

* Fix up FIPS upgrade test to use postWatcherSuccessHook to test FIPS compliance of upgraded Agent

* Add version constraint to test

* s/compliant/capable/

* s/compliant/capable/

* Append -fips to artifact name if current release of Agent is FIPS-capable

* Enable FIPS-capable to FIPS-capable Agent upgrades

* Running mage fmt

* Adding test to ensure FIPS-capable Agent cannot be upgraded to non-FIPS-capable Agent

* Adding return value

* Fixing function calls

* Remove post-upgrade success hook since we expect upgrade to fail

* Add minimum FIPS version check for start version

* Simplify upgradeOpts initialization

* Add version equality comparison method

* Fix version checks in tests

* Refactor version check into own helper function

* Fixing args

* No need to pass testing.T

* Remove redundant test case

* Restrict FIPS integration tests to Linux

* Add Fleet-managed Agent FIPS upgrade test

* Remove integration test trying to upgrade FIPS to non-FIPS

* Fix type of argument

* Refactoring: extract common logic into helper function

* Remove old code

* Require no error

* Fixing syntax errors

* Define tests as needing a FIPS environment

* FIPS upgrade tests should only run on Linux

* FIPS upgrade tests should start with FIPS-capable version

* Fixing comment + skip message

* Fix syntax errors

* Removing TestStandaloneUpgradeFIPStoFIPS test

* Removing TestFleetManagedUpgradePrivilegedFIPS test

* Add back accidentally-deleted function

* Combine less and equal unit tests

* Hash replaceToken only if its non-empty

* Use startFixture

(cherry picked from commit ac9ee9a)

# Conflicts:
#	internal/pkg/agent/application/upgrade/upgrade.go
#	testing/upgradetest/upgrader.go

Author: ycombinator

internal/pkg/agent/application/upgrade/artifact/artifact_test.go

@@ -0,0 +1,56 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package artifact
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	agtversion "github.com/elastic/elastic-agent/pkg/version"
+)
+
+func TestGetArtifactName(t *testing.T) {
+	version, err := agtversion.ParseVersion("9.1.0")
+	require.NoError(t, err)
+
+	tests := map[string]struct {
+		a               Artifact
+		version         agtversion.ParsedSemVer
+		operatingSystem string
+		arch            string
+		expectedName    string
+		expectedErr     string
+	}{
+		"no_fips": {
+			a:               Artifact{Cmd: "elastic-agent"},
+			version:         *version,
+			operatingSystem: "linux",
+			arch:            "arm64",
+			expectedName:    "elastic-agent-9.1.0-linux-arm64.tar.gz",
+		},
+		"fips": {
+			a:               Artifact{Cmd: "elastic-agent-fips"},
+			version:         *version,
+			operatingSystem: "linux",
+			arch:            "arm64",
+			expectedName:    "elastic-agent-fips-9.1.0-linux-arm64.tar.gz",
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			artifactName, err := GetArtifactName(test.a, test.version, test.operatingSystem, test.arch)
+			if test.expectedErr == "" {
+				require.NoError(t, err)
+				require.Equal(t, test.expectedName, artifactName)
+			} else {
+				require.Empty(t, artifactName)
+				require.Equal(t, test.expectedErr, err.Error())
+			}
+		})
+	}
+
+}

internal/pkg/agent/application/upgrade/step_unpack.go

@@ -419,7 +419,10 @@ func readCommitHash(reader io.Reader) (string, error) {
 }
 
 func getFileNamePrefix(archivePath string) string {
-	return strings.TrimSuffix(filepath.Base(archivePath), ".tar.gz") + "/" // omitting `elastic-agent-{version}-{os}-{arch}/` in filename
+	prefix := strings.TrimSuffix(filepath.Base(archivePath), ".tar.gz") + "/" // omitting `elastic-agent-{version}-{os}-{arch}/` in filename
+	prefix = strings.Replace(prefix, fipsPrefix, "", 1)
+
+	return prefix
 }
 
 func validFileName(p string) bool {

internal/pkg/agent/application/upgrade/step_unpack_test.go

@@ -419,3 +419,27 @@ func addEntryToZipArchive(af files, writer *zip.Writer) error {
 
 	return nil
 }
+
+func TestGetFileNamePrefix(t *testing.T) {
+	tests := map[string]struct {
+		archivePath    string
+		expectedPrefix string
+	}{
+		"fips": {
+			archivePath:    "/foo/bar/elastic-agent-fips-9.1.0-SNAPSHOT-linux-arm64.tar.gz",
+			expectedPrefix: "elastic-agent-9.1.0-SNAPSHOT-linux-arm64/",
+		},
+		"no_fips": {
+			archivePath:    "/foo/bar/elastic-agent-9.1.0-SNAPSHOT-linux-arm64.tar.gz",
+			expectedPrefix: "elastic-agent-9.1.0-SNAPSHOT-linux-arm64/",
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			prefix := getFileNamePrefix(test.archivePath)
+			require.Equal(t, test.expectedPrefix, prefix)
+		})
+	}
+
+}

internal/pkg/agent/application/upgrade/upgrade.go

@@ -46,6 +46,7 @@ const (
 	runDirMod          = 0770
 	snapshotSuffix     = "-SNAPSHOT"
 	watcherMaxWaitTime = 30 * time.Second
+	fipsPrefix         = "-fips"
 )
 
 var agentArtifact = artifact.Artifact{
@@ -60,6 +61,12 @@ var (
 	ErrFipsNotUpgradedToFips = errors.New("cannot upgrade from a fips compliant agent to a non-compliant one")
 )
 
+func init() {
+	if release.FIPSDistribution() {
+		agentArtifact.Cmd += fipsPrefix
+	}
+}
+
 // Upgrader performs an upgrade
 type Upgrader struct {
 	log            *logger.Logger
@@ -172,12 +179,23 @@ func checkUpgrade(log *logger.Logger, currentVersion, newVersion agentVersion, m
 		return ErrUpgradeSameVersion
 	}
 
+<<<<<<< HEAD
 	if currentVersion.fips && metadata.manifest.Package.Fips {
 		return nil
 	}
 
 	if !currentVersion.fips && !metadata.manifest.Package.Fips {
 		return nil
+=======
+	if currentVersion.fips && !metadata.manifest.Package.Fips {
+		log.Warnf("Upgrade action skipped because FIPS-capable Agent cannot be upgraded to non-FIPS-capable Agent")
+		return ErrFipsToNonFips
+	}
+
+	if !currentVersion.fips && metadata.manifest.Package.Fips {
+		log.Warnf("Upgrade action skipped because non-FIPS-capable Agent cannot be upgraded to FIPS-capable Agent")
+		return ErrNonFipsToFips
+>>>>>>> ac9ee9a34 ([Integration Test] Ensure that upgrading a FIPS-capable Agent results in a FIPS-capable Agent (#7804))
 	}
 
 	return ErrFipsNotUpgradedToFips

internal/pkg/agent/cmd/enroll_cmd.go

@@ -1054,10 +1054,18 @@ func createFleetConfigFromEnroll(accessAPIKey string, enrollmentToken string, re
 	if err != nil {
 		return nil, errors.New(err, "failed to generate enrollment hash", errors.TypeConfig)
 	}
-	cfg.ReplaceTokenHash, err = fleetHashToken(replaceToken)
-	if err != nil {
-		return nil, errors.New(err, "failed to generate replace token hash", errors.TypeConfig)
+
+	// Hash replaceToken if provided; it is not expected to be provided when an Agent
+	// is being enrolled for the very first time. Hashing an empty replaceToken with the
+	// FIPS-capable build of Elastic Agent results in an "invalid key length" error from
+	// OpenSSL's FIPS provider.
+	if replaceToken != "" {
+		cfg.ReplaceTokenHash, err = fleetHashToken(replaceToken)
+		if err != nil {
+			return nil, errors.New(err, "failed to generate replace token hash", errors.TypeConfig)
+		}
 	}
+
 	if err := cfg.Valid(); err != nil {
 		return nil, errors.New(err, "invalid enrollment options", errors.TypeConfig)
 	}

pkg/control/v2/client/client.go

@@ -300,6 +300,7 @@ func (c *client) Version(ctx context.Context) (Version, error) {
 		Commit:    res.Commit,
 		BuildTime: bt,
 		Snapshot:  res.Snapshot,
+		Fips:      res.Fips,
 	}, nil
 }
 

pkg/testing/define/define.go

@@ -18,6 +18,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/gofrs/uuid/v5"
 
 	"github.com/elastic/elastic-agent-libs/kibana"
@@ -83,33 +85,31 @@ func Version() string {
 // NewFixtureFromLocalBuild returns a new Elastic Agent testing fixture with a LocalFetcher and
 // the agent logging to the test logger.
 func NewFixtureFromLocalBuild(t *testing.T, version string, opts ...atesting.FixtureOpt) (*atesting.Fixture, error) {
-	buildsDir := os.Getenv("AGENT_BUILD_DIR")
-	if buildsDir == "" {
-		projectDir, err := findProjectRoot()
-		if err != nil {
-			return nil, err
-		}
-		buildsDir = filepath.Join(projectDir, "build", "distributions")
-	}
-
-	return NewFixtureWithBinary(t, version, "elastic-agent", buildsDir, opts...)
+	return NewFixtureWithBinary(t, version, "elastic-agent", buildsDir(t), false, opts...)
+}
 
+// NewFixtureFromLocalFIPSBuild returns a new FIPS-capable Elastic Agent testing fixture with a LocalFetcher
+// and the agent logging to the test logger.
+func NewFixtureFromLocalFIPSBuild(t *testing.T, version string, opts ...atesting.FixtureOpt) (*atesting.Fixture, error) {
+	return NewFixtureWithBinary(t, version, "elastic-agent", buildsDir(t), true, opts...)
 }
 
 // NewFixtureWithBinary returns a new Elastic Agent testing fixture with a LocalFetcher and
 // the agent logging to the test logger.
-func NewFixtureWithBinary(t *testing.T, version string, binary string, buildsDir string, opts ...atesting.FixtureOpt) (*atesting.Fixture, error) {
+func NewFixtureWithBinary(t *testing.T, version string, binary string, buildsDir string, fips bool, opts ...atesting.FixtureOpt) (*atesting.Fixture, error) {
 	ver, err := semver.ParseVersion(version)
 	if err != nil {
 		return nil, fmt.Errorf("%q is an invalid agent version: %w", version, err)
 	}
 
-	var binFetcher atesting.Fetcher
+	localFetcherOpts := []atesting.LocalFetcherOpt{atesting.WithCustomBinaryName(binary)}
 	if ver.IsSnapshot() {
-		binFetcher = atesting.LocalFetcher(buildsDir, atesting.WithLocalSnapshotOnly(), atesting.WithCustomBinaryName(binary))
-	} else {
-		binFetcher = atesting.LocalFetcher(buildsDir, atesting.WithCustomBinaryName(binary))
+		localFetcherOpts = append(localFetcherOpts, atesting.WithLocalSnapshotOnly())
 	}
+	if fips {
+		localFetcherOpts = append(localFetcherOpts, atesting.WithLocalFIPSOnly())
+	}
+	binFetcher := atesting.LocalFetcher(buildsDir, localFetcherOpts...)
 
 	opts = append(opts, atesting.WithFetcher(binFetcher), atesting.WithLogOutput())
 	if binary != "elastic-agent" {
@@ -301,3 +301,16 @@ func getKibanaClient() (*kibana.Client, error) {
 	}
 	return c, nil
 }
+
+func buildsDir(t *testing.T) string {
+	t.Helper()
+
+	buildsDir := os.Getenv("AGENT_BUILD_DIR")
+	if buildsDir == "" {
+		projectDir, err := findProjectRoot()
+		require.NoError(t, err)
+		buildsDir = filepath.Join(projectDir, "build", "distributions")
+	}
+
+	return buildsDir
+}

pkg/testing/fetcher_local.go

@@ -18,27 +18,35 @@ import (
 type localFetcher struct {
 	dir          string
 	snapshotOnly bool
+	fipsOnly     bool
 	binaryName   string
 }
 
-type localFetcherOpt func(f *localFetcher)
+type LocalFetcherOpt func(f *localFetcher)
 
 // WithLocalSnapshotOnly sets the LocalFetcher to only pull the snapshot build.
-func WithLocalSnapshotOnly() localFetcherOpt {
+func WithLocalSnapshotOnly() LocalFetcherOpt {
 	return func(f *localFetcher) {
 		f.snapshotOnly = true
 	}
 }
 
+// WithLocalFIPSOnly sets the LocalFetcher to only pull a FIPS-compliant build.
+func WithLocalFIPSOnly() LocalFetcherOpt {
+	return func(f *localFetcher) {
+		f.fipsOnly = true
+	}
+}
+
 // WithCustomBinaryName sets the binary to a custom name, the default is `elastic-agent`
-func WithCustomBinaryName(name string) localFetcherOpt {
+func WithCustomBinaryName(name string) LocalFetcherOpt {
 	return func(f *localFetcher) {
 		f.binaryName = name
 	}
 }
 
 // LocalFetcher returns a fetcher that pulls the binary of the Elastic Agent from a local location.
-func LocalFetcher(dir string, opts ...localFetcherOpt) Fetcher {
+func LocalFetcher(dir string, opts ...LocalFetcherOpt) Fetcher {
 	f := &localFetcher{
 		dir:        dir,
 		binaryName: "elastic-agent",
@@ -56,6 +64,7 @@ func (f *localFetcher) Name() string {
 
 // Fetch fetches the Elastic Agent and places the resulting binary at the path.
 func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) {
+	prefix := GetPackagePrefix(f.fipsOnly)
 	suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat)
 	if err != nil {
 		return nil, err
@@ -66,7 +75,7 @@ func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architec
 		return nil, fmt.Errorf("invalid version: %q: %w", ver, err)
 	}
 
-	mainBuildfmt := "%s-%s-%s"
+	mainBuildfmt := "%s-%s%s-%s"
 	if f.snapshotOnly && !ver.IsSnapshot() {
 		if ver.Prerelease() == "" {
 			ver = semver.NewParsedSemVer(ver.Major(), ver.Minor(), ver.Patch(), "SNAPSHOT", ver.BuildMetadata())
@@ -85,10 +94,10 @@ func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architec
 	}
 
 	if ver.IsSnapshot() && !matchesEarlyReleaseVersion {
-		build := fmt.Sprintf(mainBuildfmt, f.binaryName, ver.VersionWithPrerelease(), suffix)
+		build := fmt.Sprintf(mainBuildfmt, f.binaryName, prefix, ver.VersionWithPrerelease(), suffix)
 		buildPath = filepath.Join(ver.BuildMetadata(), build)
 	} else {
-		buildPath = fmt.Sprintf(mainBuildfmt, f.binaryName, ver.String(), suffix)
+		buildPath = fmt.Sprintf(mainBuildfmt, f.binaryName, prefix, ver.String(), suffix)
 	}
 
 	fullPath := filepath.Join(f.dir, buildPath)

pkg/testing/fetcher_local_test.go

@@ -57,7 +57,7 @@ func TestLocalFetcher(t *testing.T) {
 	tcs := []struct {
 		name     string
 		version  string
-		opts     []localFetcherOpt
+		opts     []LocalFetcherOpt
 		want     []byte
 		wantHash []byte
 	}{
@@ -69,7 +69,7 @@ func TestLocalFetcher(t *testing.T) {
 		}, {
 			name:     "SnapshotOnly",
 			version:  baseVersion,
-			opts:     []localFetcherOpt{WithLocalSnapshotOnly()},
+			opts:     []LocalFetcherOpt{WithLocalSnapshotOnly()},
 			want:     snapshotContent,
 			wantHash: snapshotContentHash,
 		}, {
@@ -80,13 +80,13 @@ func TestLocalFetcher(t *testing.T) {
 		}, {
 			name:     "version with snapshot and SnapshotOnly",
 			version:  baseVersion + "-SNAPSHOT",
-			opts:     []localFetcherOpt{WithLocalSnapshotOnly()},
+			opts:     []LocalFetcherOpt{WithLocalSnapshotOnly()},
 			want:     snapshotContent,
 			wantHash: snapshotContentHash,
 		}, {
 			name:     "version with snapshot and build ID",
 			version:  baseVersion + "-SNAPSHOT+l5snflwr",
-			opts:     []localFetcherOpt{},
+			opts:     []LocalFetcherOpt{},
 			want:     snapshotContent,
 			wantHash: snapshotContentHash,
 		},

pkg/testing/fixture.go

@@ -50,6 +50,7 @@ type Fixture struct {
 	binaryName      string
 	runLength       time.Duration
 	additionalArgs  []string
+	fipsArtifact    bool
 
 	srcPackage string
 	workDir    string
@@ -145,6 +146,12 @@ func WithAdditionalArgs(args []string) FixtureOpt {
 	}
 }
 
+func WithFIPSArtifact() FixtureOpt {
+	return func(f *Fixture) {
+		f.fipsArtifact = true
+	}
+}
+
 // NewFixture creates a new fixture to setup and manage Elastic Agent.
 func NewFixture(t *testing.T, version string, opts ...FixtureOpt) (*Fixture, error) {
 	// we store the caller so the fixture can find the cache directory for the artifacts that