diff --git a/internal/pkg/agent/application/application.go b/internal/pkg/agent/application/application.go index f887e37d0dc..45190169351 100644 --- a/internal/pkg/agent/application/application.go +++ b/internal/pkg/agent/application/application.go @@ -118,7 +118,7 @@ func New( // monitoring is not supported in bootstrap mode https://github.com/elastic/elastic-agent/issues/1761 isMonitoringSupported := !disableMonitoring && cfg.Settings.V1MonitoringEnabled - upgrader, err := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig, agentInfo) + upgrader, err := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig, cfg.Settings.Upgrade, agentInfo) if err != nil { return nil, nil, nil, fmt.Errorf("failed to create upgrader: %w", err) } diff --git a/internal/pkg/agent/application/coordinator/coordinator_unit_test.go b/internal/pkg/agent/application/coordinator/coordinator_unit_test.go index 1a24b6d33a2..94606208d4c 100644 --- a/internal/pkg/agent/application/coordinator/coordinator_unit_test.go +++ b/internal/pkg/agent/application/coordinator/coordinator_unit_test.go @@ -35,6 +35,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" + "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" "github.com/elastic/elastic-agent/internal/pkg/agent/transpiler" "github.com/elastic/elastic-agent/internal/pkg/config" monitoringCfg "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/config" @@ -446,6 +447,7 @@ func TestCoordinatorReportsInvalidPolicy(t *testing.T) { upgradeMgr, err := upgrade.NewUpgrader( log, &artifact.Config{}, + &configuration.UpgradeConfig{}, &info.AgentInfo{}, ) require.NoError(t, err, "errored when creating a new upgrader") diff --git a/internal/pkg/agent/application/upgrade/rollback.go b/internal/pkg/agent/application/upgrade/rollback.go index 90be1bbe2df..1fe62f7fefd 100644 --- a/internal/pkg/agent/application/upgrade/rollback.go +++ b/internal/pkg/agent/application/upgrade/rollback.go @@ -24,6 +24,7 @@ import ( "github.com/elastic/elastic-agent/pkg/control/v2/client" "github.com/elastic/elastic-agent/pkg/core/logger" "github.com/elastic/elastic-agent/pkg/utils" + "github.com/elastic/elastic-agent/pkg/version" ) const ( @@ -33,6 +34,9 @@ const ( restartBackoffMax = 90 * time.Second ) +// Rollback window feature is only available starting with version >= 9.1.0-SNAPSHOT. +var rollbackWindowMinVersion = version.NewParsedSemVer(9, 1, 0, "SNAPSHOT", "") + // Rollback rollbacks to previous version which was functioning before upgrade. func Rollback(ctx context.Context, log *logger.Logger, c client.Client, topDirPath, prevVersionedHome, prevHash string) error { symlinkPath := filepath.Join(topDirPath, agentName) @@ -144,13 +148,13 @@ func cleanup(log *logger.Logger, topDirPath, currentVersionedHome, currentHash s // InvokeWatcher invokes an agent instance using watcher argument for watching behavior of // agent during upgrade period. -func InvokeWatcher(log *logger.Logger, agentExecutable string) (*exec.Cmd, error) { +func InvokeWatcher(log *logger.Logger, agentExecutable string, rollbackWindow time.Duration) (*exec.Cmd, error) { if !IsUpgradeable() { log.Info("agent is not upgradable, not starting watcher") return nil, nil } - cmd := invokeCmd(agentExecutable) + cmd := makeOSWatchCmd(makeBaseWatchCmd(agentExecutable, rollbackWindow)) log.Infow("Starting upgrade watcher", "path", cmd.Path, "args", cmd.Args, "env", cmd.Env, "dir", cmd.Dir) if err := cmd.Start(); err != nil { return nil, fmt.Errorf("failed to start Upgrade Watcher: %w", err) @@ -238,3 +242,24 @@ func restartAgent(ctx context.Context, log *logger.Logger, c client.Client) erro close(signal) return nil } + +func makeBaseWatchCmd(agentExecutable string, rollbackWindow time.Duration) *exec.Cmd { + cmdArgs := []string{ + watcherSubcommand, + "--path.config", paths.Config(), + "--path.home", paths.Top(), + } + + if rollbackWindow > 0 { + cmdArgs = append(cmdArgs, "--rollback-window", fmt.Sprintf("%s", rollbackWindow.String())) + } + + // #nosec G204 -- user cannot inject any parameters to this command + return exec.Command(agentExecutable, cmdArgs...) +} + +// isRollbackWindowSupported checks if the rollback window feature is supported by the version of +// the target Agent, i.e the version being upgraded *to*. +func isRollbackWindowSupported(targetAgentVersion *version.ParsedSemVer) bool { + return !targetAgentVersion.Less(*rollbackWindowMinVersion) +} diff --git a/internal/pkg/agent/application/upgrade/rollback_darwin.go b/internal/pkg/agent/application/upgrade/rollback_darwin.go index 041abf11b40..5ac8069fd47 100644 --- a/internal/pkg/agent/application/upgrade/rollback_darwin.go +++ b/internal/pkg/agent/application/upgrade/rollback_darwin.go @@ -11,8 +11,6 @@ import ( "os/exec" "syscall" "time" - - "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" ) const ( @@ -21,13 +19,7 @@ const ( afterRestartDelay = 2 * time.Second ) -func invokeCmd(agentExecutable string) *exec.Cmd { - // #nosec G204 -- user cannot inject any parameters to this command - cmd := exec.Command(agentExecutable, watcherSubcommand, - "--path.config", paths.Config(), - "--path.home", paths.Top(), - ) - +func makeOSWatchCmd(baseWatchCmd *exec.Cmd) *exec.Cmd { var cred = &syscall.Credential{ Uid: uint32(os.Getuid()), Gid: uint32(os.Getgid()), @@ -38,6 +30,6 @@ func invokeCmd(agentExecutable string) *exec.Cmd { Credential: cred, Setsid: true, } - cmd.SysProcAttr = sysproc - return cmd + baseWatchCmd.SysProcAttr = sysproc + return baseWatchCmd } diff --git a/internal/pkg/agent/application/upgrade/rollback_linux.go b/internal/pkg/agent/application/upgrade/rollback_linux.go index bdaf918a2b6..ef9e46e6003 100644 --- a/internal/pkg/agent/application/upgrade/rollback_linux.go +++ b/internal/pkg/agent/application/upgrade/rollback_linux.go @@ -11,8 +11,6 @@ import ( "os/exec" "syscall" "time" - - "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" ) const ( @@ -21,13 +19,7 @@ const ( afterRestartDelay = 2 * time.Second ) -func invokeCmd(agentExecutable string) *exec.Cmd { - // #nosec G204 -- user cannot inject any parameters to this command - cmd := exec.Command(agentExecutable, watcherSubcommand, - "--path.config", paths.Config(), - "--path.home", paths.Top(), - ) - +func makeOSWatchCmd(baseWatchCmd *exec.Cmd) *exec.Cmd { var cred = &syscall.Credential{ Uid: uint32(os.Getuid()), Gid: uint32(os.Getgid()), @@ -40,6 +32,6 @@ func invokeCmd(agentExecutable string) *exec.Cmd { // propagate sigint instead of sigkill so we can ignore it Pdeathsig: syscall.SIGINT, } - cmd.SysProcAttr = sysproc - return cmd + baseWatchCmd.SysProcAttr = sysproc + return baseWatchCmd } diff --git a/internal/pkg/agent/application/upgrade/rollback_test.go b/internal/pkg/agent/application/upgrade/rollback_test.go index 3f9cc0a33ab..bdb75628b7b 100644 --- a/internal/pkg/agent/application/upgrade/rollback_test.go +++ b/internal/pkg/agent/application/upgrade/rollback_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "runtime" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -19,6 +20,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/pkg/core/logger" "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" + "github.com/elastic/elastic-agent/pkg/version" mocks "github.com/elastic/elastic-agent/testing/mocks/pkg/control/v2/client" ) @@ -328,6 +330,78 @@ func TestRollback(t *testing.T) { } } +func TestIsRollbackWindowSupported(t *testing.T) { + tests := map[string]struct { + version string + want bool + }{ + "supported_version": { + version: "9.2.0-SNAPSHOT", + want: true, + }, + "older_version": { + version: "9.0.0-SNAPSHOT", + want: false, + }, + "exactly_minimum_version": { + version: "9.1.0", + want: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + v, err := version.ParseVersion(tt.version) + require.NoError(t, err) + + got := isRollbackWindowSupported(v) + assert.Equal(t, tt.want, got, "isRollbackWindowSupported(%q)", tt.version) + }) + } +} + +func TestMakeBaseWatchCmd(t *testing.T) { + exec, err := os.Executable() + require.NoError(t, err) + exec, err = filepath.EvalSymlinks(exec) + require.NoError(t, err) + execDir := filepath.Dir(exec) + + agentExecutable := "elastic-agent" + t.Run("no_rollback_window", func(t *testing.T) { + cmd := makeBaseWatchCmd(agentExecutable, 0) + + // Expected command: + // elastic-agent watch --path.config /some/path --path.home /some/path + require.Equal(t, cmd.Args, []string{ + agentExecutable, + "watch", + "--path.config", + execDir, + "--path.home", + execDir, + }) + }) + + t.Run("with_rollback_window", func(t *testing.T) { + rollbackWindow := 2*time.Hour + 15*time.Minute + cmd := makeBaseWatchCmd(agentExecutable, rollbackWindow) + + // Expected command: + // elastic-agent watch --path.config /some/path --path.home /some/path --rollback-window 8100s + require.Equal(t, cmd.Args, []string{ + agentExecutable, + "watch", + "--path.config", + execDir, + "--path.home", + execDir, + "--rollback-window", + rollbackWindow.String(), + }) + }) +} + // checkFilesAfterCleanup is a convenience function to check the file structure within topDir. // *AgentHome paths must be the expected old and new agent paths relative to topDir (typically in the form of "data/elastic-agent-*") func checkFilesAfterCleanup(t *testing.T, topDir, newAgentHome string, oldAgentHomes ...string) { diff --git a/internal/pkg/agent/application/upgrade/rollback_windows.go b/internal/pkg/agent/application/upgrade/rollback_windows.go index b7c273c9385..e9d3ffc281a 100644 --- a/internal/pkg/agent/application/upgrade/rollback_windows.go +++ b/internal/pkg/agent/application/upgrade/rollback_windows.go @@ -9,8 +9,6 @@ package upgrade import ( "os/exec" "time" - - "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" ) const ( @@ -19,11 +17,6 @@ const ( afterRestartDelay = 20 * time.Second ) -func invokeCmd(agentExecutable string) *exec.Cmd { - // #nosec G204 -- user cannot inject any parameters to this command - cmd := exec.Command(agentExecutable, watcherSubcommand, - "--path.config", paths.Config(), - "--path.home", paths.Top(), - ) - return cmd +func makeOSWatchCmd(baseWatchCmd *exec.Cmd) *exec.Cmd { + return baseWatchCmd } diff --git a/internal/pkg/agent/application/upgrade/step_download.go b/internal/pkg/agent/application/upgrade/step_download.go index 58d56c81f52..7a2e4ff0afd 100644 --- a/internal/pkg/agent/application/upgrade/step_download.go +++ b/internal/pkg/agent/application/upgrade/step_download.go @@ -50,7 +50,7 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, parsedVersion *agtversi pgpBytes = u.appendFallbackPGP(parsedVersion, pgpBytes) // do not update source config - settings := *u.settings + settings := *u.downloadSettings var downloaderFunc downloader var factory downloaderFactory var verifier download.Verifier diff --git a/internal/pkg/agent/application/upgrade/step_download_test.go b/internal/pkg/agent/application/upgrade/step_download_test.go index f1e20427c25..2a5939ef6f1 100644 --- a/internal/pkg/agent/application/upgrade/step_download_test.go +++ b/internal/pkg/agent/application/upgrade/step_download_test.go @@ -19,6 +19,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" + "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/pkg/core/logger" "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" @@ -78,12 +79,13 @@ func TestDownloadWithRetries(t *testing.T) { expectedDownloadPath := "https://artifacts.elastic.co/downloads/beats/elastic-agent" testLogger, obs := loggertest.New("TestDownloadWithRetries") - settings := artifact.Config{ + downloadSettings := artifact.Config{ RetrySleepInitDuration: 20 * time.Millisecond, HTTPTransportSettings: httpcommon.HTTPTransportSettings{ Timeout: 2 * time.Second, }, } + upgradeSettings := configuration.UpgradeConfig{} // Successful immediately (no retries) t.Run("successful_immediately", func(t *testing.T) { @@ -91,16 +93,16 @@ func TestDownloadWithRetries(t *testing.T) { return &mockDownloader{expectedDownloadPath, nil}, nil } - u, err := NewUpgrader(testLogger, &settings, &info.AgentInfo{}) + u, err := NewUpgrader(testLogger, &downloadSettings, &upgradeSettings, &info.AgentInfo{}) require.NoError(t, err) parsedVersion, err := agtversion.ParseVersion("8.9.0") require.NoError(t, err) upgradeDetails, upgradeDetailsRetryUntil, upgradeDetailsRetryUntilWasUnset, upgradeDetailsRetryErrorMsg := mockUpgradeDetails(parsedVersion) - minRetryDeadline := time.Now().Add(settings.Timeout) + minRetryDeadline := time.Now().Add(downloadSettings.Timeout) - path, err := u.downloadWithRetries(context.Background(), mockDownloaderCtor, parsedVersion, &settings, upgradeDetails) + path, err := u.downloadWithRetries(context.Background(), mockDownloaderCtor, parsedVersion, &downloadSettings, upgradeDetails) require.NoError(t, err) require.Equal(t, expectedDownloadPath, path) @@ -141,16 +143,16 @@ func TestDownloadWithRetries(t *testing.T) { return nil, nil } - u, err := NewUpgrader(testLogger, &settings, &info.AgentInfo{}) + u, err := NewUpgrader(testLogger, &downloadSettings, &upgradeSettings, &info.AgentInfo{}) require.NoError(t, err) parsedVersion, err := agtversion.ParseVersion("8.9.0") require.NoError(t, err) upgradeDetails, upgradeDetailsRetryUntil, upgradeDetailsRetryUntilWasUnset, upgradeDetailsRetryErrorMsg := mockUpgradeDetails(parsedVersion) - minRetryDeadline := time.Now().Add(settings.Timeout) + minRetryDeadline := time.Now().Add(downloadSettings.Timeout) - path, err := u.downloadWithRetries(context.Background(), mockDownloaderCtor, parsedVersion, &settings, upgradeDetails) + path, err := u.downloadWithRetries(context.Background(), mockDownloaderCtor, parsedVersion, &downloadSettings, upgradeDetails) require.NoError(t, err) require.Equal(t, expectedDownloadPath, path) @@ -196,16 +198,16 @@ func TestDownloadWithRetries(t *testing.T) { return nil, nil } - u, err := NewUpgrader(testLogger, &settings, &info.AgentInfo{}) + u, err := NewUpgrader(testLogger, &downloadSettings, &upgradeSettings, &info.AgentInfo{}) require.NoError(t, err) parsedVersion, err := agtversion.ParseVersion("8.9.0") require.NoError(t, err) upgradeDetails, upgradeDetailsRetryUntil, upgradeDetailsRetryUntilWasUnset, upgradeDetailsRetryErrorMsg := mockUpgradeDetails(parsedVersion) - minRetryDeadline := time.Now().Add(settings.Timeout) + minRetryDeadline := time.Now().Add(downloadSettings.Timeout) - path, err := u.downloadWithRetries(context.Background(), mockDownloaderCtor, parsedVersion, &settings, upgradeDetails) + path, err := u.downloadWithRetries(context.Background(), mockDownloaderCtor, parsedVersion, &downloadSettings, upgradeDetails) require.NoError(t, err) require.Equal(t, expectedDownloadPath, path) @@ -231,7 +233,7 @@ func TestDownloadWithRetries(t *testing.T) { // Download timeout expired (before all retries are exhausted) t.Run("download_timeout_expired", func(t *testing.T) { - testCaseSettings := settings + testCaseSettings := downloadSettings testCaseSettings.Timeout = 500 * time.Millisecond testCaseSettings.RetrySleepInitDuration = 10 * time.Millisecond // exponential backoff with 10ms init and 500ms timeout should fit at least 3 attempts. @@ -241,7 +243,7 @@ func TestDownloadWithRetries(t *testing.T) { return &mockDownloader{"", errors.New("download failed")}, nil } - u, err := NewUpgrader(testLogger, &settings, &info.AgentInfo{}) + u, err := NewUpgrader(testLogger, &downloadSettings, &upgradeSettings, &info.AgentInfo{}) require.NoError(t, err) parsedVersion, err := agtversion.ParseVersion("8.9.0") diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go index 19d3b67cb2b..197239b8554 100644 --- a/internal/pkg/agent/application/upgrade/upgrade.go +++ b/internal/pkg/agent/application/upgrade/upgrade.go @@ -70,12 +70,13 @@ func init() { // Upgrader performs an upgrade type Upgrader struct { - log *logger.Logger - settings *artifact.Config - agentInfo info.Agent - upgradeable bool - fleetServerURI string - markerWatcher MarkerWatcher + log *logger.Logger + downloadSettings *artifact.Config + upgradeSettings *configuration.UpgradeConfig + agentInfo info.Agent + upgradeable bool + fleetServerURI string + markerWatcher MarkerWatcher } // IsUpgradeable when agent is installed and running as a service or flag was provided. @@ -86,13 +87,14 @@ func IsUpgradeable() bool { } // NewUpgrader creates an upgrader which is capable of performing upgrade operation -func NewUpgrader(log *logger.Logger, settings *artifact.Config, agentInfo info.Agent) (*Upgrader, error) { +func NewUpgrader(log *logger.Logger, downloadSettings *artifact.Config, upgradeSettings *configuration.UpgradeConfig, agentInfo info.Agent) (*Upgrader, error) { return &Upgrader{ - log: log, - settings: settings, - agentInfo: agentInfo, - upgradeable: IsUpgradeable(), - markerWatcher: newMarkerFileWatcher(markerFilePath(paths.Data()), log), + log: log, + downloadSettings: downloadSettings, + upgradeSettings: upgradeSettings, + agentInfo: agentInfo, + upgradeable: IsUpgradeable(), + markerWatcher: newMarkerFileWatcher(markerFilePath(paths.Data()), log), }, nil } @@ -107,7 +109,7 @@ func (u *Upgrader) SetClient(c fleetclient.Sender) { u.log.Debugf("Set client changed URI to %s", u.fleetServerURI) } -// Reload reloads the artifact configuration for the upgrader. +// Reload reloads the artifact download and upgrade configurations for the upgrader. // As of today, December 2023, fleet-server does not send most of the configuration // defined in artifact.Config, what will likely change in the near future. func (u *Upgrader) Reload(rawConfig *config.Config) error { @@ -133,17 +135,18 @@ func (u *Upgrader) Reload(rawConfig *config.Config) error { if cfg.Settings.DownloadConfig.SourceURI != "" { u.log.Infof("Source URI changed from %q to %q", - u.settings.SourceURI, + u.downloadSettings.SourceURI, cfg.Settings.DownloadConfig.SourceURI) } else { // source uri unset, reset to default u.log.Infof("Source URI reset from %q to %q", - u.settings.SourceURI, + u.downloadSettings.SourceURI, artifact.DefaultSourceURI) cfg.Settings.DownloadConfig.SourceURI = artifact.DefaultSourceURI } - u.settings = cfg.Settings.DownloadConfig + u.downloadSettings = cfg.Settings.DownloadConfig + u.upgradeSettings = cfg.Settings.Upgrade return nil } @@ -351,8 +354,18 @@ func (u *Upgrader) Upgrade(ctx context.Context, version string, sourceURI string watcherExecutable := selectWatcherExecutable(paths.Top(), previous, current) + // Check if the target Agent version supports the rollback window feature. If it doesn't, + // it won't know how to perform a deferred cleanup after a successful upgrade, so we need + // to rely on the Upgrade Watcher to do the cleanup immediately after it has deemed the upgrade + // to be successful. We ask the Upgrade Watcher to do this immediate cleanup by passing + // a rollback window of 0 seconds to the Upgrade Watcher. + rollbackWindow := u.upgradeSettings.Rollback.Window + if !isRollbackWindowSupported(parsedVersion) { + rollbackWindow = 0 + } + var watcherCmd *exec.Cmd - if watcherCmd, err = InvokeWatcher(u.log, watcherExecutable); err != nil { + if watcherCmd, err = InvokeWatcher(u.log, watcherExecutable, rollbackWindow); err != nil { u.log.Errorw("Rolling back: starting watcher failed", "error.message", err) rollbackErr := rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome) return nil, goerrors.Join(err, rollbackErr) @@ -377,6 +390,8 @@ func (u *Upgrader) Upgrade(ctx context.Context, version string, sourceURI string return cb, nil } +// selectWatcherExecutable returns the path to the watcher executable for the Agent that has the newer +// of the two Agent versions involved in the upgrade. func selectWatcherExecutable(topDir string, previous agentInstall, current agentInstall) string { // check if the upgraded version is less than the previous (currently installed) version if current.parsedVersion.Less(*previous.parsedVersion) { @@ -477,7 +492,7 @@ func (u *Upgrader) sourceURI(retrievedURI string) string { return retrievedURI } - return u.settings.SourceURI + return u.downloadSettings.SourceURI } func extractAgentVersion(metadata packageMetadata, upgradeVersion string) agentVersion { diff --git a/internal/pkg/agent/application/upgrade/upgrade_test.go b/internal/pkg/agent/application/upgrade/upgrade_test.go index 17d19252f6e..e08c13dbecd 100644 --- a/internal/pkg/agent/application/upgrade/upgrade_test.go +++ b/internal/pkg/agent/application/upgrade/upgrade_test.go @@ -271,21 +271,21 @@ func TestUpgraderReload(t *testing.T) { log, _ := loggertest.New("") u := Upgrader{ - log: log, - settings: artifact.DefaultConfig(), + log: log, + downloadSettings: artifact.DefaultConfig(), } err := u.Reload(config.MustNewConfigFrom(cfgyaml)) require.NoError(t, err, "error reloading config") - assert.Equal(t, &want, u.settings) + assert.Equal(t, &want, u.downloadSettings) } func TestUpgraderAckAction(t *testing.T) { log, _ := loggertest.New("") u := Upgrader{ - log: log, - settings: artifact.DefaultConfig(), + log: log, + downloadSettings: artifact.DefaultConfig(), } action := fleetapi.NewAction(fleetapi.ActionTypeUpgrade) @@ -574,8 +574,8 @@ agent.download: log, _ := loggertest.New("") u := Upgrader{ - log: log, - settings: artifact.DefaultConfig(), + log: log, + downloadSettings: artifact.DefaultConfig(), } cfg, err := config.NewConfigFrom(tc.cfg) @@ -584,11 +584,11 @@ agent.download: err = u.Reload(cfg) require.NoError(t, err, "error reloading config") - assert.Equal(t, tc.sourceURL, u.settings.SourceURI) + assert.Equal(t, tc.sourceURL, u.downloadSettings.SourceURI) if tc.proxyURL != "" { - require.NotNilf(t, u.settings.Proxy.URL, + require.NotNilf(t, u.downloadSettings.Proxy.URL, "ProxyURI should not be nil, want %s", tc.proxyURL) - assert.Equal(t, tc.proxyURL, u.settings.Proxy.URL.String()) + assert.Equal(t, tc.proxyURL, u.downloadSettings.Proxy.URL.String()) } }) } diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go index f91f2a37790..edb4254a622 100644 --- a/internal/pkg/agent/cmd/run.go +++ b/internal/pkg/agent/cmd/run.go @@ -256,7 +256,7 @@ func runElasticAgent(ctx context.Context, cancel context.CancelFunc, override ap } // initiate agent watcher - if _, err := upgrade.InvokeWatcher(l, paths.TopBinaryPath()); err != nil { + if _, err := upgrade.InvokeWatcher(l, paths.TopBinaryPath(), cfg.Settings.Upgrade.Rollback.Window); err != nil { // we should not fail because watcher is not working l.Error(errors.New(err, "failed to invoke rollback watcher")) } diff --git a/internal/pkg/agent/cmd/watch.go b/internal/pkg/agent/cmd/watch.go index f203c1814f7..f31736d4695 100644 --- a/internal/pkg/agent/cmd/watch.go +++ b/internal/pkg/agent/cmd/watch.go @@ -33,8 +33,9 @@ import ( ) const ( - watcherName = "elastic-agent-watcher" - watcherLockFile = "watcher.lock" + watcherName = "elastic-agent-watcher" + watcherLockFile = "watcher.lock" + flagRollbackWindow = "rollback-window" ) func newWatchCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command { @@ -42,7 +43,7 @@ func newWatchCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command Use: "watch", Short: "Watch the Elastic Agent for failures and initiate rollback", Long: `This command watches Elastic Agent for failures and initiates rollback if necessary.`, - Run: func(_ *cobra.Command, _ []string) { + Run: func(c *cobra.Command, _ []string) { cfg := getConfig(streams) log, err := configuredLogger(cfg, watcherName) if err != nil { @@ -53,7 +54,7 @@ func newWatchCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command // Make sure to flush any buffered logs before we're done. defer log.Sync() //nolint:errcheck // flushing buffered logs is best effort. - if err := watchCmd(log, paths.Top(), cfg.Settings.Upgrade.Watcher, new(upgradeAgentWatcher), new(upgradeInstallationModifier)); err != nil { + if err := watchCmd(c, log, paths.Top(), cfg.Settings.Upgrade.Watcher, new(upgradeAgentWatcher), new(upgradeInstallationModifier)); err != nil { log.Errorw("Watch command failed", "error.message", err) fmt.Fprintf(streams.Err, "Watch command failed: %v\n%s\n", err, troubleshootMessage()) os.Exit(4) @@ -61,6 +62,8 @@ func newWatchCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command }, } + cmd.Flags().DurationP(flagRollbackWindow, "", configuration.DefaultRollbackWindowDuration, "Duration in which Agent may be manually rolled back") + return cmd } @@ -73,7 +76,7 @@ type installationModifier interface { Rollback(ctx context.Context, log *logger.Logger, c client.Client, topDirPath, prevVersionedHome, prevHash string) error } -func watchCmd(log *logp.Logger, topDir string, cfg *configuration.UpgradeWatcherConfig, watcher agentWatcher, installModifier installationModifier) error { +func watchCmd(cmd *cobra.Command, log *logp.Logger, topDir string, cfg *configuration.UpgradeWatcherConfig, watcher agentWatcher, installModifier installationModifier) error { log.Infow("Upgrade Watcher started", "process.pid", os.Getpid(), "agent.version", version.GetAgentPackageVersion(), "config", cfg) dataDir := paths.DataFrom(topDir) marker, err := upgrade.LoadMarker(dataDir) @@ -102,6 +105,12 @@ func watchCmd(log *logp.Logger, topDir string, cfg *configuration.UpgradeWatcher _ = locker.Unlock() }() + rollbackWindow, err := cmd.Flags().GetDuration(flagRollbackWindow) + if err != nil { + return fmt.Errorf("failed to retrieve %s flag value while watching the agent upgrade: %w", flagRollbackWindow, err) + } + _ = rollbackWindow // TODO: use rollbackWindow when implementing https://github.com/elastic/elastic-agent/issues/6884 + isWithinGrace, tilGrace := gracePeriod(marker, cfg.GracePeriod) if isTerminalState(marker) || !isWithinGrace { stateString := "" diff --git a/internal/pkg/agent/cmd/watch_test.go b/internal/pkg/agent/cmd/watch_test.go index 9451c476543..3cf36f03d47 100644 --- a/internal/pkg/agent/cmd/watch_test.go +++ b/internal/pkg/agent/cmd/watch_test.go @@ -275,8 +275,9 @@ func Test_watchCmd(t *testing.T) { tmpDir := t.TempDir() mockWatcher := cmdmocks.NewAgentWatcher(t) mockInstallModifier := cmdmocks.NewInstallationModifier(t) + cmd := newWatchCommandWithArgs(nil, nil) tt.setupUpgradeMarker(t, tmpDir, mockWatcher, mockInstallModifier) - tt.wantErr(t, watchCmd(log, tmpDir, tt.args.cfg, mockWatcher, mockInstallModifier), fmt.Sprintf("watchCmd(%v, ...)", tt.args.cfg)) + tt.wantErr(t, watchCmd(cmd, log, tmpDir, tt.args.cfg, mockWatcher, mockInstallModifier), fmt.Sprintf("watchCmd(%v, ...)", tt.args.cfg)) t.Logf("watchCmd logs:\n%v", obs.All()) }) } diff --git a/internal/pkg/agent/configuration/upgrade.go b/internal/pkg/agent/configuration/upgrade.go index 405b405ec46..ed1f6f395b2 100644 --- a/internal/pkg/agent/configuration/upgrade.go +++ b/internal/pkg/agent/configuration/upgrade.go @@ -13,9 +13,9 @@ const ( // interval between checks for new (upgraded) Agent returning an error status. defaultStatusCheckInterval = 30 * time.Second - // period during which an upgraded Agent can be asked to rollback to the previous - // Agent version on disk. - defaultRollbackWindowDuration = 7 * 24 * time.Hour // 7 days + // DefaultRollbackWindowDuration is the period during which an upgraded Agent can be asked to + // rollback to the previous Agent version on disk. + DefaultRollbackWindowDuration = 7 * 24 * time.Hour // 7 days ) // UpgradeConfig is the configuration related to Agent upgrades. @@ -45,7 +45,7 @@ func DefaultUpgradeConfig() *UpgradeConfig { }, }, Rollback: &UpgradeRollbackConfig{ - Window: defaultRollbackWindowDuration, + Window: DefaultRollbackWindowDuration, }, } } diff --git a/internal/pkg/agent/configuration/upgrade_test.go b/internal/pkg/agent/configuration/upgrade_test.go index 0005e15ace8..0888662c0d6 100644 --- a/internal/pkg/agent/configuration/upgrade_test.go +++ b/internal/pkg/agent/configuration/upgrade_test.go @@ -28,7 +28,7 @@ func TestParseUpgradeConfig(t *testing.T) { }, }, Rollback: &UpgradeRollbackConfig{ - Window: defaultRollbackWindowDuration, + Window: DefaultRollbackWindowDuration, }, }, }, @@ -46,7 +46,7 @@ func TestParseUpgradeConfig(t *testing.T) { }, }, Rollback: &UpgradeRollbackConfig{ - Window: defaultRollbackWindowDuration, + Window: DefaultRollbackWindowDuration, }, }, }, @@ -66,7 +66,7 @@ func TestParseUpgradeConfig(t *testing.T) { }, }, Rollback: &UpgradeRollbackConfig{ - Window: defaultRollbackWindowDuration, + Window: DefaultRollbackWindowDuration, }, }, },