diff --git a/.mockery.yaml b/.mockery.yaml index ea5df1eebbe..2b09e2b4f57 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -30,10 +30,24 @@ packages: interfaces: Agent: github.com/elastic/elastic-agent/internal/pkg/agent/cmd: + config: + inpackage: True + with-expecter: True + dir: "{{.InterfaceDirRelative}}" + mockname: "{{.Mock}}{{.InterfaceName | firstUpper}}" + outpkg: "{{.PackageName}}" + filename: "{{.Mock | lower}}_{{.InterfaceName | lower}}_test.go" interfaces: agentWatcher: - config: - mockname: "AgentWatcher" installationModifier: - config: - mockname: "InstallationModifier" + github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade: + config: + inpackage: True + with-expecter: True + dir: "{{.InterfaceDirRelative}}" + mockname: "{{.Mock}}{{.InterfaceName | firstUpper}}" + outpkg: "{{.PackageName}}" + filename: "{{.Mock | lower}}_{{.InterfaceName | lower}}_test.go" + interfaces: + WatcherHelper: + watcherGrappler: diff --git a/_meta/config/common.reference.p2.yml.tmpl b/_meta/config/common.reference.p2.yml.tmpl index 8429db51431..29ef3cfb9d8 100644 --- a/_meta/config/common.reference.p2.yml.tmpl +++ b/_meta/config/common.reference.p2.yml.tmpl @@ -123,7 +123,7 @@ inputs: # # rollback settings # rollback: # # duration in which an upgraded Agent may be manually rolled back. -# window: 168h +# window: 0 # agent.process: # # timeout for creating new processes. when process is not successfully created by this timeout diff --git a/control_v2.proto b/control_v2.proto index 55a988a37ff..c5ec6ecb037 100644 --- a/control_v2.proto +++ b/control_v2.proto @@ -116,6 +116,9 @@ message UpgradeRequest { // // If provided Elastic Agent package embedded PGP key is not checked for signature during upgrade. bool skipDefaultPgp = 5; + + // If true it indicates that we wish to rollback the current/last upgrade + bool rollback = 6; } // A upgrade response message. diff --git a/elastic-agent.reference.yml b/elastic-agent.reference.yml index 41a4eb36150..7bc3a553867 100644 --- a/elastic-agent.reference.yml +++ b/elastic-agent.reference.yml @@ -129,7 +129,7 @@ inputs: # # rollback settings # rollback: # # duration in which an upgraded Agent may be manually rolled back. -# window: 168h +# window: 0 # agent.process: # # timeout for creating new processes. when process is not successfully created by this timeout diff --git a/internal/pkg/agent/application/actions/handlers/handler_action_upgrade.go b/internal/pkg/agent/application/actions/handlers/handler_action_upgrade.go index 1520677f06b..849957c0f6d 100644 --- a/internal/pkg/agent/application/actions/handlers/handler_action_upgrade.go +++ b/internal/pkg/agent/application/actions/handlers/handler_action_upgrade.go @@ -75,7 +75,7 @@ func (h *Upgrade) Handle(ctx context.Context, a fleetapi.Action, ack acker.Acker go func() { h.log.Infof("starting upgrade to version %s in background", action.Data.Version) - if err := h.coord.Upgrade(asyncCtx, action.Data.Version, action.Data.SourceURI, action, false, false); err != nil { + if err := h.coord.Upgrade(asyncCtx, action.Data.Version, false, action.Data.SourceURI, action, false, false); err != nil { h.log.Errorf("upgrade to version %s failed: %v", action.Data.Version, err) // If context is cancelled in getAsyncContext, the actions are acked there if !errors.Is(asyncCtx.Err(), context.Canceled) { diff --git a/internal/pkg/agent/application/actions/handlers/handler_action_upgrade_test.go b/internal/pkg/agent/application/actions/handlers/handler_action_upgrade_test.go index 10ac83f536e..ae18af1a576 100644 --- a/internal/pkg/agent/application/actions/handlers/handler_action_upgrade_test.go +++ b/internal/pkg/agent/application/actions/handlers/handler_action_upgrade_test.go @@ -48,15 +48,7 @@ func (u *mockUpgradeManager) Reload(rawConfig *config.Config) error { return nil } -func (u *mockUpgradeManager) Upgrade( - ctx context.Context, - version string, - sourceURI string, - action *fleetapi.ActionUpgrade, - details *details.Details, - skipVerifyOverride bool, - skipDefaultPgp bool, - pgpBytes ...string) (reexec.ShutdownCallbackFn, error) { +func (u *mockUpgradeManager) Upgrade(ctx context.Context, version string, rollback bool, sourceURI string, action *fleetapi.ActionUpgrade, details *details.Details, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) (reexec.ShutdownCallbackFn, error) { return u.UpgradeFn( ctx, diff --git a/internal/pkg/agent/application/actions/handlers/handler_helpers.go b/internal/pkg/agent/application/actions/handlers/handler_helpers.go index 39d5e48d7af..c83834bb584 100644 --- a/internal/pkg/agent/application/actions/handlers/handler_helpers.go +++ b/internal/pkg/agent/application/actions/handlers/handler_helpers.go @@ -28,7 +28,7 @@ type actionCoordinator interface { type upgradeCoordinator interface { actionCoordinator - Upgrade(ctx context.Context, version string, sourceURI string, action *fleetapi.ActionUpgrade, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) error + Upgrade(ctx context.Context, version string, rollback bool, sourceURI string, action *fleetapi.ActionUpgrade, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) error } type performActionFunc func(context.Context, component.Component, component.Unit, string, map[string]interface{}) (map[string]interface{}, error) diff --git a/internal/pkg/agent/application/application.go b/internal/pkg/agent/application/application.go index dcaa2ddc570..0cd825ad29a 100644 --- a/internal/pkg/agent/application/application.go +++ b/internal/pkg/agent/application/application.go @@ -120,7 +120,7 @@ func New( // monitoring is not supported in bootstrap mode https://github.com/elastic/elastic-agent/issues/1761 isMonitoringSupported := !disableMonitoring && cfg.Settings.V1MonitoringEnabled - upgrader, err := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig, agentInfo) + upgrader, err := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig, cfg.Settings.Upgrade, agentInfo, new(upgrade.AgentWatcherHelper)) if err != nil { return nil, nil, nil, fmt.Errorf("failed to create upgrader: %w", err) } diff --git a/internal/pkg/agent/application/coordinator/coordinator.go b/internal/pkg/agent/application/coordinator/coordinator.go index a21d8e7cb9f..3340435958c 100644 --- a/internal/pkg/agent/application/coordinator/coordinator.go +++ b/internal/pkg/agent/application/coordinator/coordinator.go @@ -85,7 +85,7 @@ type UpgradeManager interface { Reload(rawConfig *config.Config) error // Upgrade upgrades running agent. - Upgrade(ctx context.Context, version string, sourceURI string, action *fleetapi.ActionUpgrade, details *details.Details, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) (_ reexec.ShutdownCallbackFn, err error) + Upgrade(ctx context.Context, version string, rollback bool, sourceURI string, action *fleetapi.ActionUpgrade, details *details.Details, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) (_ reexec.ShutdownCallbackFn, err error) // Ack is used on startup to check if the agent has upgraded and needs to send an ack for the action Ack(ctx context.Context, acker acker.Acker) error @@ -695,7 +695,7 @@ func (c *Coordinator) Migrate(ctx context.Context, action *fleetapi.ActionMigrat // Upgrade runs the upgrade process. // Called from external goroutines. -func (c *Coordinator) Upgrade(ctx context.Context, version string, sourceURI string, action *fleetapi.ActionUpgrade, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) error { +func (c *Coordinator) Upgrade(ctx context.Context, version string, rollback bool, sourceURI string, action *fleetapi.ActionUpgrade, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) error { // early check outside of upgrader before overriding the state if !c.upgradeMgr.Upgradeable() { return ErrNotUpgradable @@ -735,7 +735,7 @@ func (c *Coordinator) Upgrade(ctx context.Context, version string, sourceURI str det := details.NewDetails(version, details.StateRequested, actionID) det.RegisterObserver(c.SetUpgradeDetails) - cb, err := c.upgradeMgr.Upgrade(ctx, version, sourceURI, action, det, skipVerifyOverride, skipDefaultPgp, pgpBytes...) + cb, err := c.upgradeMgr.Upgrade(ctx, version, rollback, sourceURI, action, det, skipVerifyOverride, skipDefaultPgp, pgpBytes...) if err != nil { c.ClearOverrideState() if errors.Is(err, upgrade.ErrUpgradeSameVersion) { diff --git a/internal/pkg/agent/application/coordinator/coordinator_test.go b/internal/pkg/agent/application/coordinator/coordinator_test.go index b2ba3bcb53c..34f810f42e0 100644 --- a/internal/pkg/agent/application/coordinator/coordinator_test.go +++ b/internal/pkg/agent/application/coordinator/coordinator_test.go @@ -535,7 +535,7 @@ func TestUpgradeSameErrorAcked(t *testing.T) { acker.On("Ack", mock.Anything, actionUpgrade).Return(nil) - require.NoError(t, coord.Upgrade(t.Context(), "9.0", "http://localhost", actionUpgrade, true, true)) + require.NoError(t, coord.Upgrade(t.Context(), "9.0", false, "http://localhost", actionUpgrade, true, true)) acker.AssertCalled(t, "Ack", mock.Anything, actionUpgrade) } @@ -917,7 +917,7 @@ func TestCoordinator_Upgrade(t *testing.T) { require.NoError(t, err) cfgMgr.Config(ctx, cfg) - err = coord.Upgrade(ctx, "9.0.0", "", nil, true, false) + err = coord.Upgrade(ctx, "9.0.0", false, "", nil, true, false) require.ErrorIs(t, err, ErrNotUpgradable) cancel() @@ -954,7 +954,7 @@ func TestCoordinator_UpgradeDetails(t *testing.T) { require.NoError(t, err) cfgMgr.Config(ctx, cfg) - err = coord.Upgrade(ctx, "9.0.0", "", nil, true, false) + err = coord.Upgrade(ctx, "9.0.0", false, "", nil, true, false) require.ErrorIs(t, expectedErr, err) cancel() @@ -1159,7 +1159,7 @@ func (f *fakeUpgradeManager) Reload(cfg *config.Config) error { return nil } -func (f *fakeUpgradeManager) Upgrade(ctx context.Context, version string, sourceURI string, action *fleetapi.ActionUpgrade, details *details.Details, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) (_ reexec.ShutdownCallbackFn, err error) { +func (f *fakeUpgradeManager) Upgrade(ctx context.Context, version string, rollback bool, sourceURI string, action *fleetapi.ActionUpgrade, details *details.Details, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) (_ reexec.ShutdownCallbackFn, err error) { f.upgradeCalled = true if f.upgradeErr != nil { return nil, f.upgradeErr diff --git a/internal/pkg/agent/application/coordinator/coordinator_unit_test.go b/internal/pkg/agent/application/coordinator/coordinator_unit_test.go index 59766a3226d..972c2d1eb7f 100644 --- a/internal/pkg/agent/application/coordinator/coordinator_unit_test.go +++ b/internal/pkg/agent/application/coordinator/coordinator_unit_test.go @@ -462,11 +462,7 @@ func TestCoordinatorReportsInvalidPolicy(t *testing.T) { } }() - upgradeMgr, err := upgrade.NewUpgrader( - log, - &artifact.Config{}, - &info.AgentInfo{}, - ) + upgradeMgr, err := upgrade.NewUpgrader(log, &artifact.Config{}, nil, &info.AgentInfo{}, new(upgrade.AgentWatcherHelper)) require.NoError(t, err, "errored when creating a new upgrader") // Channels have buffer length 1, so we don't have to run on multiple @@ -1526,7 +1522,7 @@ func TestCoordinatorInitiatesUpgrade(t *testing.T) { } // Call upgrade and make sure the upgrade manager receives an Upgrade call - err := coord.Upgrade(ctx, "1.2.3", "", nil, false, false) + err := coord.Upgrade(ctx, "1.2.3", false, "", nil, false, false) assert.True(t, upgradeMgr.upgradeCalled, "Coordinator Upgrade should call upgrade manager Upgrade") assert.Equal(t, upgradeMgr.upgradeErr, err, "Upgrade should report upgrade manager error") diff --git a/internal/pkg/agent/application/filelock/locker.go b/internal/pkg/agent/application/filelock/locker.go index 9de204c9731..316301ace26 100644 --- a/internal/pkg/agent/application/filelock/locker.go +++ b/internal/pkg/agent/application/filelock/locker.go @@ -39,6 +39,7 @@ func (a *AppLocker) TryLock() error { if !locked { return ErrAppAlreadyRunning } + return nil } diff --git a/internal/pkg/agent/application/filelock/testlocker/.gitignore b/internal/pkg/agent/application/filelock/testlocker/.gitignore new file mode 100644 index 00000000000..1afe2659727 --- /dev/null +++ b/internal/pkg/agent/application/filelock/testlocker/.gitignore @@ -0,0 +1,2 @@ +# Ignore test binary +testlocker \ No newline at end of file diff --git a/internal/pkg/agent/application/filelock/testlocker/main.go b/internal/pkg/agent/application/filelock/testlocker/main.go new file mode 100644 index 00000000000..4e95ed3c6e1 --- /dev/null +++ b/internal/pkg/agent/application/filelock/testlocker/main.go @@ -0,0 +1,64 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +// This is a simple program that will lock an applocker using a file passed using the -lockfile option, used for testing file lock works properly. +// os.Interrupt or signal.SIGTERM will make the program release the lock and exit +package main + +import ( + "flag" + "log" + "os" + "os/signal" + "path/filepath" + "syscall" + + "github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock" +) + +const AcquiredLockLogFmt = "Acquired lock on file %s\n" + +const lockFileFlagName = "lockfile" +const ignoreSignalFlagName = "ignoresignals" + +var lockFile = flag.String(lockFileFlagName, "", "path to lock file") +var ignoreSignals = flag.Bool(ignoreSignalFlagName, false, "ignore signals") + +func main() { + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + flag.Parse() + if *lockFile == "" { + log.Fatalf("No lockfile specified. Please run %s -%s ", os.Args[0], lockFileFlagName) + } + + appLocker := filelock.NewAppLocker(filepath.Dir(*lockFile), filepath.Base(*lockFile)) + + err := appLocker.TryLock() + if err != nil { + log.Fatalf("Error locking %s: %s", *lockFile, err.Error()) + } + + defer func(aLocker *filelock.AppLocker) { + + if unlockErr := aLocker.Unlock(); unlockErr != nil { + log.Printf("Error unlocking %s: %s", *lockFile, unlockErr.Error()) + } + }(appLocker) + + log.Printf(AcquiredLockLogFmt, *lockFile) + + for { + + s := <-signalChan + if *ignoreSignals { + log.Printf("Received signal %v , ignoring it...", s) + continue + } + + log.Printf("Received signal %v , exiting...", s) + break + } +} diff --git a/internal/pkg/agent/application/upgrade/details/details.go b/internal/pkg/agent/application/upgrade/details/details.go index cd83c61855d..b3fcf99069b 100644 --- a/internal/pkg/agent/application/upgrade/details/details.go +++ b/internal/pkg/agent/application/upgrade/details/details.go @@ -231,7 +231,8 @@ func (m Metadata) Equals(otherM Metadata) bool { m.DownloadPercent == otherM.DownloadPercent && m.DownloadRate == otherM.DownloadRate && equalTimePointers(m.RetryUntil, otherM.RetryUntil) && - m.RetryErrorMsg == otherM.RetryErrorMsg + m.RetryErrorMsg == otherM.RetryErrorMsg && + m.Reason == otherM.Reason } func equalTimePointers(t, otherT *time.Time) bool { diff --git a/internal/pkg/agent/application/upgrade/details/state.go b/internal/pkg/agent/application/upgrade/details/state.go index 41a04698cdb..bcb46b1569c 100644 --- a/internal/pkg/agent/application/upgrade/details/state.go +++ b/internal/pkg/agent/application/upgrade/details/state.go @@ -21,5 +21,6 @@ const ( StateFailed State = "UPG_FAILED" // List of well-known reasons for state transitions - ReasonWatchFailed = "watch failed" + ReasonWatchFailed = "watch failed" + ReasonManualRollback = "manual rollback requested" ) diff --git a/internal/pkg/agent/application/upgrade/marker_watcher_test.go b/internal/pkg/agent/application/upgrade/marker_watcher_test.go index 45db5bc4a6b..ec25d198035 100644 --- a/internal/pkg/agent/application/upgrade/marker_watcher_test.go +++ b/internal/pkg/agent/application/upgrade/marker_watcher_test.go @@ -126,6 +126,9 @@ details: expectedDetails: &details.Details{ TargetVersion: "8.9.2", State: details.StateRollback, + Metadata: details.Metadata{ + Reason: details.ReasonWatchFailed, + }, }, }, "same_version_with_details_some_state": { diff --git a/internal/pkg/agent/application/upgrade/mock_watchergrappler_test.go b/internal/pkg/agent/application/upgrade/mock_watchergrappler_test.go new file mode 100644 index 00000000000..b5e6a668262 --- /dev/null +++ b/internal/pkg/agent/application/upgrade/mock_watchergrappler_test.go @@ -0,0 +1,89 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package upgrade + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + logp "github.com/elastic/elastic-agent-libs/logp" +) + +// mockWatcherGrappler is an autogenerated mock type for the watcherGrappler type +type mockWatcherGrappler struct { + mock.Mock +} + +type mockWatcherGrappler_Expecter struct { + mock *mock.Mock +} + +func (_m *mockWatcherGrappler) EXPECT() *mockWatcherGrappler_Expecter { + return &mockWatcherGrappler_Expecter{mock: &_m.Mock} +} + +// TakeDownWatcher provides a mock function with given fields: ctx, log +func (_m *mockWatcherGrappler) TakeDownWatcher(ctx context.Context, log *logp.Logger) error { + ret := _m.Called(ctx, log) + + if len(ret) == 0 { + panic("no return value specified for TakeDownWatcher") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *logp.Logger) error); ok { + r0 = rf(ctx, log) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// mockWatcherGrappler_TakeDownWatcher_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TakeDownWatcher' +type mockWatcherGrappler_TakeDownWatcher_Call struct { + *mock.Call +} + +// TakeDownWatcher is a helper method to define mock.On call +// - ctx context.Context +// - log *logp.Logger +func (_e *mockWatcherGrappler_Expecter) TakeDownWatcher(ctx interface{}, log interface{}) *mockWatcherGrappler_TakeDownWatcher_Call { + return &mockWatcherGrappler_TakeDownWatcher_Call{Call: _e.mock.On("TakeDownWatcher", ctx, log)} +} + +func (_c *mockWatcherGrappler_TakeDownWatcher_Call) Run(run func(ctx context.Context, log *logp.Logger)) *mockWatcherGrappler_TakeDownWatcher_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*logp.Logger)) + }) + return _c +} + +func (_c *mockWatcherGrappler_TakeDownWatcher_Call) Return(_a0 error) *mockWatcherGrappler_TakeDownWatcher_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *mockWatcherGrappler_TakeDownWatcher_Call) RunAndReturn(run func(context.Context, *logp.Logger) error) *mockWatcherGrappler_TakeDownWatcher_Call { + _c.Call.Return(run) + return _c +} + +// newMockWatcherGrappler creates a new instance of mockWatcherGrappler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockWatcherGrappler(t interface { + mock.TestingT + Cleanup(func()) +}) *mockWatcherGrappler { + mock := &mockWatcherGrappler{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/pkg/agent/application/upgrade/mock_watcherhelper_test.go b/internal/pkg/agent/application/upgrade/mock_watcherhelper_test.go new file mode 100644 index 00000000000..43eb4002e60 --- /dev/null +++ b/internal/pkg/agent/application/upgrade/mock_watcherhelper_test.go @@ -0,0 +1,262 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package upgrade + +import ( + context "context" + exec "os/exec" + + logp "github.com/elastic/elastic-agent-libs/logp" + filelock "github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock" + + mock "github.com/stretchr/testify/mock" + + time "time" +) + +// MockWatcherHelper is an autogenerated mock type for the WatcherHelper type +type MockWatcherHelper struct { + mock.Mock +} + +type MockWatcherHelper_Expecter struct { + mock *mock.Mock +} + +func (_m *MockWatcherHelper) EXPECT() *MockWatcherHelper_Expecter { + return &MockWatcherHelper_Expecter{mock: &_m.Mock} +} + +// InvokeWatcher provides a mock function with given fields: log, agentExecutable +func (_m *MockWatcherHelper) InvokeWatcher(log *logp.Logger, agentExecutable string) (*exec.Cmd, error) { + ret := _m.Called(log, agentExecutable) + + if len(ret) == 0 { + panic("no return value specified for InvokeWatcher") + } + + var r0 *exec.Cmd + var r1 error + if rf, ok := ret.Get(0).(func(*logp.Logger, string) (*exec.Cmd, error)); ok { + return rf(log, agentExecutable) + } + if rf, ok := ret.Get(0).(func(*logp.Logger, string) *exec.Cmd); ok { + r0 = rf(log, agentExecutable) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*exec.Cmd) + } + } + + if rf, ok := ret.Get(1).(func(*logp.Logger, string) error); ok { + r1 = rf(log, agentExecutable) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockWatcherHelper_InvokeWatcher_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'InvokeWatcher' +type MockWatcherHelper_InvokeWatcher_Call struct { + *mock.Call +} + +// InvokeWatcher is a helper method to define mock.On call +// - log *logp.Logger +// - agentExecutable string +func (_e *MockWatcherHelper_Expecter) InvokeWatcher(log interface{}, agentExecutable interface{}) *MockWatcherHelper_InvokeWatcher_Call { + return &MockWatcherHelper_InvokeWatcher_Call{Call: _e.mock.On("InvokeWatcher", log, agentExecutable)} +} + +func (_c *MockWatcherHelper_InvokeWatcher_Call) Run(run func(log *logp.Logger, agentExecutable string)) *MockWatcherHelper_InvokeWatcher_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*logp.Logger), args[1].(string)) + }) + return _c +} + +func (_c *MockWatcherHelper_InvokeWatcher_Call) Return(_a0 *exec.Cmd, _a1 error) *MockWatcherHelper_InvokeWatcher_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockWatcherHelper_InvokeWatcher_Call) RunAndReturn(run func(*logp.Logger, string) (*exec.Cmd, error)) *MockWatcherHelper_InvokeWatcher_Call { + _c.Call.Return(run) + return _c +} + +// SelectWatcherExecutable provides a mock function with given fields: topDir, previous, current +func (_m *MockWatcherHelper) SelectWatcherExecutable(topDir string, previous agentInstall, current agentInstall) string { + ret := _m.Called(topDir, previous, current) + + if len(ret) == 0 { + panic("no return value specified for SelectWatcherExecutable") + } + + var r0 string + if rf, ok := ret.Get(0).(func(string, agentInstall, agentInstall) string); ok { + r0 = rf(topDir, previous, current) + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// MockWatcherHelper_SelectWatcherExecutable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SelectWatcherExecutable' +type MockWatcherHelper_SelectWatcherExecutable_Call struct { + *mock.Call +} + +// SelectWatcherExecutable is a helper method to define mock.On call +// - topDir string +// - previous agentInstall +// - current agentInstall +func (_e *MockWatcherHelper_Expecter) SelectWatcherExecutable(topDir interface{}, previous interface{}, current interface{}) *MockWatcherHelper_SelectWatcherExecutable_Call { + return &MockWatcherHelper_SelectWatcherExecutable_Call{Call: _e.mock.On("SelectWatcherExecutable", topDir, previous, current)} +} + +func (_c *MockWatcherHelper_SelectWatcherExecutable_Call) Run(run func(topDir string, previous agentInstall, current agentInstall)) *MockWatcherHelper_SelectWatcherExecutable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(agentInstall), args[2].(agentInstall)) + }) + return _c +} + +func (_c *MockWatcherHelper_SelectWatcherExecutable_Call) Return(_a0 string) *MockWatcherHelper_SelectWatcherExecutable_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockWatcherHelper_SelectWatcherExecutable_Call) RunAndReturn(run func(string, agentInstall, agentInstall) string) *MockWatcherHelper_SelectWatcherExecutable_Call { + _c.Call.Return(run) + return _c +} + +// TakeOverWatcher provides a mock function with given fields: ctx, log, topDir +func (_m *MockWatcherHelper) TakeOverWatcher(ctx context.Context, log *logp.Logger, topDir string) (*filelock.AppLocker, error) { + ret := _m.Called(ctx, log, topDir) + + if len(ret) == 0 { + panic("no return value specified for TakeOverWatcher") + } + + var r0 *filelock.AppLocker + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *logp.Logger, string) (*filelock.AppLocker, error)); ok { + return rf(ctx, log, topDir) + } + if rf, ok := ret.Get(0).(func(context.Context, *logp.Logger, string) *filelock.AppLocker); ok { + r0 = rf(ctx, log, topDir) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*filelock.AppLocker) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *logp.Logger, string) error); ok { + r1 = rf(ctx, log, topDir) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockWatcherHelper_TakeOverWatcher_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TakeOverWatcher' +type MockWatcherHelper_TakeOverWatcher_Call struct { + *mock.Call +} + +// TakeOverWatcher is a helper method to define mock.On call +// - ctx context.Context +// - log *logp.Logger +// - topDir string +func (_e *MockWatcherHelper_Expecter) TakeOverWatcher(ctx interface{}, log interface{}, topDir interface{}) *MockWatcherHelper_TakeOverWatcher_Call { + return &MockWatcherHelper_TakeOverWatcher_Call{Call: _e.mock.On("TakeOverWatcher", ctx, log, topDir)} +} + +func (_c *MockWatcherHelper_TakeOverWatcher_Call) Run(run func(ctx context.Context, log *logp.Logger, topDir string)) *MockWatcherHelper_TakeOverWatcher_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*logp.Logger), args[2].(string)) + }) + return _c +} + +func (_c *MockWatcherHelper_TakeOverWatcher_Call) Return(_a0 *filelock.AppLocker, _a1 error) *MockWatcherHelper_TakeOverWatcher_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockWatcherHelper_TakeOverWatcher_Call) RunAndReturn(run func(context.Context, *logp.Logger, string) (*filelock.AppLocker, error)) *MockWatcherHelper_TakeOverWatcher_Call { + _c.Call.Return(run) + return _c +} + +// WaitForWatcher provides a mock function with given fields: ctx, log, markerFilePath, waitTime +func (_m *MockWatcherHelper) WaitForWatcher(ctx context.Context, log *logp.Logger, markerFilePath string, waitTime time.Duration) error { + ret := _m.Called(ctx, log, markerFilePath, waitTime) + + if len(ret) == 0 { + panic("no return value specified for WaitForWatcher") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *logp.Logger, string, time.Duration) error); ok { + r0 = rf(ctx, log, markerFilePath, waitTime) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockWatcherHelper_WaitForWatcher_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'WaitForWatcher' +type MockWatcherHelper_WaitForWatcher_Call struct { + *mock.Call +} + +// WaitForWatcher is a helper method to define mock.On call +// - ctx context.Context +// - log *logp.Logger +// - markerFilePath string +// - waitTime time.Duration +func (_e *MockWatcherHelper_Expecter) WaitForWatcher(ctx interface{}, log interface{}, markerFilePath interface{}, waitTime interface{}) *MockWatcherHelper_WaitForWatcher_Call { + return &MockWatcherHelper_WaitForWatcher_Call{Call: _e.mock.On("WaitForWatcher", ctx, log, markerFilePath, waitTime)} +} + +func (_c *MockWatcherHelper_WaitForWatcher_Call) Run(run func(ctx context.Context, log *logp.Logger, markerFilePath string, waitTime time.Duration)) *MockWatcherHelper_WaitForWatcher_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*logp.Logger), args[2].(string), args[3].(time.Duration)) + }) + return _c +} + +func (_c *MockWatcherHelper_WaitForWatcher_Call) Return(_a0 error) *MockWatcherHelper_WaitForWatcher_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockWatcherHelper_WaitForWatcher_Call) RunAndReturn(run func(context.Context, *logp.Logger, string, time.Duration) error) *MockWatcherHelper_WaitForWatcher_Call { + _c.Call.Return(run) + return _c +} + +// NewMockWatcherHelper creates a new instance of MockWatcherHelper. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockWatcherHelper(t interface { + mock.TestingT + Cleanup(func()) +}) *MockWatcherHelper { + mock := &MockWatcherHelper{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/pkg/agent/application/upgrade/rollback.go b/internal/pkg/agent/application/upgrade/rollback.go index 90be1bbe2df..8f69f8fdcc5 100644 --- a/internal/pkg/agent/application/upgrade/rollback.go +++ b/internal/pkg/agent/application/upgrade/rollback.go @@ -35,6 +35,36 @@ const ( // Rollback rollbacks to previous version which was functioning before upgrade. func Rollback(ctx context.Context, log *logger.Logger, c client.Client, topDirPath, prevVersionedHome, prevHash string) error { + return RollbackWithOpts(ctx, log, c, topDirPath, prevVersionedHome, prevHash) +} + +var FatalRollbackError = errors.New("Fatal rollback error") + +type RollbackHook func(ctx context.Context, log *logger.Logger, topDirPath string) error +type rollbackSettings struct { + preRestartHook RollbackHook +} + +func newRollbackSettings(opts ...RollbackOpt) *rollbackSettings { + rs := new(rollbackSettings) + for _, opt := range opts { + opt(rs) + } + return rs +} + +type RollbackOpt func(*rollbackSettings) + +func WithPreRestartHook(h RollbackHook) RollbackOpt { + return func(s *rollbackSettings) { + s.preRestartHook = h + } +} + +func RollbackWithOpts(ctx context.Context, log *logger.Logger, c client.Client, topDirPath string, prevVersionedHome string, prevHash string, opts ...RollbackOpt) error { + + settings := newRollbackSettings(opts...) + symlinkPath := filepath.Join(topDirPath, agentName) var symlinkTarget string @@ -56,6 +86,18 @@ func Rollback(ctx context.Context, log *logger.Logger, c client.Client, topDirPa return err } + // Hook + if settings.preRestartHook != nil { + hookErr := settings.preRestartHook(ctx, log, topDirPath) + if hookErr != nil { + if errors.Is(hookErr, FatalRollbackError) { + return fmt.Errorf("pre-restart hook failed: %w", hookErr) + } else { + log.Warnf("pre-restart hook failed: %v", hookErr) + } + } + } + // Restart log.Info("Restarting the agent after rollback") if err := restartAgent(ctx, log, c); err != nil { @@ -149,28 +191,52 @@ func InvokeWatcher(log *logger.Logger, agentExecutable string) (*exec.Cmd, error log.Info("agent is not upgradable, not starting watcher") return nil, nil } - - cmd := invokeCmd(agentExecutable) - log.Infow("Starting upgrade watcher", "path", cmd.Path, "args", cmd.Args, "env", cmd.Env, "dir", cmd.Dir) - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("failed to start Upgrade Watcher: %w", err) + // invokeWatcherCmd and StartWatcherCmd are platform-specific functions dealing with process launching details. + cmd, err := StartWatcherCmd(log, func() *exec.Cmd { return invokeWatcherCmd(agentExecutable) }) + if err != nil { + return nil, fmt.Errorf("starting watcher process: %w", err) } upgradeWatcherPID := cmd.Process.Pid agentPID := os.Getpid() - - go func() { - if err := cmd.Wait(); err != nil { - log.Infow("Upgrade Watcher exited with error", "agent.upgrade.watcher.process.pid", "agent.process.pid", agentPID, upgradeWatcherPID, "error.message", err) - } - }() - log.Infow("Upgrade Watcher invoked", "agent.upgrade.watcher.process.pid", upgradeWatcherPID, "agent.process.pid", agentPID) return cmd, nil } +type WatcherInvocationOpt func(opts *watcherInvocationOptions) +type watcherHook func() + +type watcherInvocationOptions struct { + postWatchHook watcherHook +} + +func WithWatcherPostWaitHook(h watcherHook) WatcherInvocationOpt { + return func(opts *watcherInvocationOptions) { + opts.postWatchHook = h + } +} + +func applyWatcherInvocationOpts(opts ...WatcherInvocationOpt) *watcherInvocationOptions { + invocationOpts := new(watcherInvocationOptions) + for _, opt := range opts { + opt(invocationOpts) + } + return invocationOpts +} + +type cmdFactory func() *exec.Cmd + +func invokeWatcherCmd(agentExecutable string) *exec.Cmd { + return InvokeCmdWithArgs( + agentExecutable, + watcherSubcommand, + "--path.config", paths.Config(), + "--path.home", paths.Top(), + ) +} + func restartAgent(ctx context.Context, log *logger.Logger, c client.Client) error { restartViaDaemonFn := func(ctx context.Context) error { connectCtx, connectCancel := context.WithTimeout(ctx, 3*time.Second) diff --git a/internal/pkg/agent/application/upgrade/rollback_darwin.go b/internal/pkg/agent/application/upgrade/rollback_darwin.go index 041abf11b40..00b4b8cde56 100644 --- a/internal/pkg/agent/application/upgrade/rollback_darwin.go +++ b/internal/pkg/agent/application/upgrade/rollback_darwin.go @@ -11,8 +11,6 @@ import ( "os/exec" "syscall" "time" - - "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" ) const ( @@ -21,16 +19,13 @@ const ( afterRestartDelay = 2 * time.Second ) -func invokeCmd(agentExecutable string) *exec.Cmd { +func InvokeCmdWithArgs(executable string, args ...string) *exec.Cmd { // #nosec G204 -- user cannot inject any parameters to this command - cmd := exec.Command(agentExecutable, watcherSubcommand, - "--path.config", paths.Config(), - "--path.home", paths.Top(), - ) + cmd := exec.Command(executable, args...) var cred = &syscall.Credential{ - Uid: uint32(os.Getuid()), - Gid: uint32(os.Getgid()), + Uid: uint32(os.Getuid()), //nolint:gosec // int -> uint32 no overflow is possible since os.Getuid() should return a value compatible with uint32 + Gid: uint32(os.Getgid()), //nolint:gosec // int -> uint32 no overflow is possible since os.Getgid() should return a value compatible with uint32 Groups: nil, NoSetGroups: true, } diff --git a/internal/pkg/agent/application/upgrade/rollback_linux.go b/internal/pkg/agent/application/upgrade/rollback_linux.go index bdaf918a2b6..fb587dab842 100644 --- a/internal/pkg/agent/application/upgrade/rollback_linux.go +++ b/internal/pkg/agent/application/upgrade/rollback_linux.go @@ -11,8 +11,6 @@ import ( "os/exec" "syscall" "time" - - "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" ) const ( @@ -21,24 +19,21 @@ const ( afterRestartDelay = 2 * time.Second ) -func invokeCmd(agentExecutable string) *exec.Cmd { +func InvokeCmdWithArgs(executable string, args ...string) *exec.Cmd { // #nosec G204 -- user cannot inject any parameters to this command - cmd := exec.Command(agentExecutable, watcherSubcommand, - "--path.config", paths.Config(), - "--path.home", paths.Top(), - ) + cmd := exec.Command(executable, args...) var cred = &syscall.Credential{ - Uid: uint32(os.Getuid()), - Gid: uint32(os.Getgid()), + Uid: uint32(os.Getuid()), //nolint:gosec // int -> uint32 no overflow is possible since os.Getuid() should return a value compatible with uint32 + Gid: uint32(os.Getgid()), //nolint:gosec // int -> uint32 no overflow is possible since os.Getgid() should return a value compatible with uint32 Groups: nil, NoSetGroups: true, } var sysproc = &syscall.SysProcAttr{ Credential: cred, Setsid: true, - // propagate sigint instead of sigkill so we can ignore it - Pdeathsig: syscall.SIGINT, + // disable parent death signal for the watcher process + Pdeathsig: syscall.Signal(0x0), } cmd.SysProcAttr = sysproc return cmd diff --git a/internal/pkg/agent/application/upgrade/rollback_notwindows.go b/internal/pkg/agent/application/upgrade/rollback_notwindows.go new file mode 100644 index 00000000000..e73d85e818f --- /dev/null +++ b/internal/pkg/agent/application/upgrade/rollback_notwindows.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build !windows + +package upgrade + +import ( + "fmt" + "os" + "os/exec" + + "github.com/elastic/elastic-agent/pkg/core/logger" +) + +func StartWatcherCmd(log *logger.Logger, createCmd cmdFactory, opts ...WatcherInvocationOpt) (*exec.Cmd, error) { + + invocationOpts := applyWatcherInvocationOpts(opts...) + + cmd := createCmd() + log.Infow("Starting upgrade watcher", "path", cmd.Path, "args", cmd.Args, "env", cmd.Env, "dir", cmd.Dir) + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start Upgrade Watcher: %w", err) + } + + upgradeWatcherPID := cmd.Process.Pid + agentPID := os.Getpid() + + go func() { + if err := cmd.Wait(); err != nil { + log.Infow("Upgrade Watcher exited with error", "agent.upgrade.watcher.process.pid", agentPID, "agent.process.pid", upgradeWatcherPID, "error.message", err) + } + if invocationOpts.postWatchHook != nil { + invocationOpts.postWatchHook() + } + }() + return cmd, nil +} diff --git a/internal/pkg/agent/application/upgrade/rollback_test.go b/internal/pkg/agent/application/upgrade/rollback_test.go index 3f9cc0a33ab..93ca32278f3 100644 --- a/internal/pkg/agent/application/upgrade/rollback_test.go +++ b/internal/pkg/agent/application/upgrade/rollback_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "runtime" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -504,10 +505,6 @@ func createUpdateMarker(t *testing.T, log *logger.Logger, topDir, newAgentVersio hash: oldAgentHash, versionedHome: oldAgentVersionedHome, } - err := markUpgrade(log, - paths.DataFrom(topDir), - newAgentInstall, - oldAgentInstall, - nil, nil, OUTCOME_UPGRADE) + err := markUpgrade(log, paths.DataFrom(topDir), time.Now(), newAgentInstall, oldAgentInstall, nil, nil, OUTCOME_UPGRADE, 0) require.NoError(t, err, "error writing fake update marker") } diff --git a/internal/pkg/agent/application/upgrade/rollback_windows.go b/internal/pkg/agent/application/upgrade/rollback_windows.go index b7c273c9385..3ddfbac4690 100644 --- a/internal/pkg/agent/application/upgrade/rollback_windows.go +++ b/internal/pkg/agent/application/upgrade/rollback_windows.go @@ -7,10 +7,17 @@ package upgrade import ( + "errors" + "fmt" + "os" "os/exec" + "syscall" "time" + "unsafe" - "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" + "golang.org/x/sys/windows" + + "github.com/elastic/elastic-agent/pkg/core/logger" ) const ( @@ -19,11 +26,90 @@ const ( afterRestartDelay = 20 * time.Second ) -func invokeCmd(agentExecutable string) *exec.Cmd { +func InvokeCmdWithArgs(executable string, args ...string) *exec.Cmd { // #nosec G204 -- user cannot inject any parameters to this command - cmd := exec.Command(agentExecutable, watcherSubcommand, - "--path.config", paths.Config(), - "--path.home", paths.Top(), - ) + cmd := exec.Command(executable, args...) + + cmd.SysProcAttr = &syscall.SysProcAttr{ + // Signals are sent to process groups, and child process are part of the + // parent's process group. So to send a signal to a + // child process and not have it also affect ourselves + // (the parent process), the child needs to be created in a new + // process group. + // + // Creating a child with CREATE_NEW_PROCESS_GROUP disables CTLR_C_EVENT + // handling for the child, so the only way to gracefully stop it is with + // a CTRL_BREAK_EVENT signal. + // https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + // + // Watcher process will also need a console in order to receive CTRL_BREAK_EVENT on windows. + // Elastic Agent main process running as a service does not have a console allocated and the watcher process will also + // outlive its parent during an upgrade operation so we add the CREATE_NEW_CONSOLE flag. + CreationFlags: windows.CREATE_NEW_PROCESS_GROUP, + } return cmd } + +func StartWatcherCmd(log *logger.Logger, createCmd cmdFactory, opts ...WatcherInvocationOpt) (*exec.Cmd, error) { + + invocationOpts := applyWatcherInvocationOpts(opts...) + + // allocConsole + r1, _, consoleErr := allocConsoleProc.Call() + if r1 == 0 { + if !errors.Is(consoleErr, windows.ERROR_ACCESS_DENIED) { + return nil, fmt.Errorf("error allocating console: %w", consoleErr) + } else { + log.Warnf("Already possessing a console") + } + + } + cmd := createCmd() + log.Infow("Starting upgrade watcher", "path", cmd.Path, "args", cmd.Args, "env", cmd.Env, "dir", cmd.Dir) + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start Upgrade Watcher: %w", err) + } + list, consoleErr := getConsoleProcessList() + if consoleErr != nil { + log.Errorf("failed to get console process list: %v", consoleErr) + } else { + log.Infof("Found console processes %v", list) + } + // free console + r1, _, consoleErr = freeConsoleProc.Call() + if r1 == 0 { + return nil, fmt.Errorf("error freeing console: %w", consoleErr) + } + upgradeWatcherPID := cmd.Process.Pid + agentPID := os.Getpid() + + go func() { + if err := cmd.Wait(); err != nil { + log.Infow("Upgrade Watcher exited with error", "agent.upgrade.watcher.process.pid", agentPID, "agent.process.pid", upgradeWatcherPID, "error.message", err) + } + if invocationOpts.postWatchHook != nil { + invocationOpts.postWatchHook() + } + }() + + return cmd, nil +} + +// getConsoleProcessList retrieves the list of process IDs attached to the current console +func getConsoleProcessList() ([]uint32, error) { + // Allocate a buffer for PIDs + const maxProcs = 64 + pids := make([]uint32, maxProcs) + + r1, _, err := procGetConsoleProcessList.Call( + uintptr(unsafe.Pointer(&pids[0])), + uintptr(maxProcs), + ) + + count := uint32(r1) + if count == 0 { + return nil, err + } + + return pids[:count], nil +} diff --git a/internal/pkg/agent/application/upgrade/step_download_test.go b/internal/pkg/agent/application/upgrade/step_download_test.go index f1e20427c25..852e45d83e7 100644 --- a/internal/pkg/agent/application/upgrade/step_download_test.go +++ b/internal/pkg/agent/application/upgrade/step_download_test.go @@ -91,7 +91,7 @@ func TestDownloadWithRetries(t *testing.T) { return &mockDownloader{expectedDownloadPath, nil}, nil } - u, err := NewUpgrader(testLogger, &settings, &info.AgentInfo{}) + u, err := NewUpgrader(testLogger, &settings, nil, &info.AgentInfo{}, new(AgentWatcherHelper)) require.NoError(t, err) parsedVersion, err := agtversion.ParseVersion("8.9.0") @@ -141,7 +141,7 @@ func TestDownloadWithRetries(t *testing.T) { return nil, nil } - u, err := NewUpgrader(testLogger, &settings, &info.AgentInfo{}) + u, err := NewUpgrader(testLogger, &settings, nil, &info.AgentInfo{}, new(AgentWatcherHelper)) require.NoError(t, err) parsedVersion, err := agtversion.ParseVersion("8.9.0") @@ -196,7 +196,7 @@ func TestDownloadWithRetries(t *testing.T) { return nil, nil } - u, err := NewUpgrader(testLogger, &settings, &info.AgentInfo{}) + u, err := NewUpgrader(testLogger, &settings, nil, &info.AgentInfo{}, new(AgentWatcherHelper)) require.NoError(t, err) parsedVersion, err := agtversion.ParseVersion("8.9.0") @@ -241,7 +241,7 @@ func TestDownloadWithRetries(t *testing.T) { return &mockDownloader{"", errors.New("download failed")}, nil } - u, err := NewUpgrader(testLogger, &settings, &info.AgentInfo{}) + u, err := NewUpgrader(testLogger, &settings, nil, &info.AgentInfo{}, new(AgentWatcherHelper)) require.NoError(t, err) parsedVersion, err := agtversion.ParseVersion("8.9.0") diff --git a/internal/pkg/agent/application/upgrade/step_mark.go b/internal/pkg/agent/application/upgrade/step_mark.go index 65b4e878a40..b462e0b90c6 100644 --- a/internal/pkg/agent/application/upgrade/step_mark.go +++ b/internal/pkg/agent/application/upgrade/step_mark.go @@ -31,6 +31,13 @@ const ( OUTCOME_ROLLBACK ) +// RollbackAvailable identifies an elastic-agent install available for rollback +type RollbackAvailable struct { + Version string `json:"version" yaml:"version"` + Home string `json:"home" yaml:"home"` + ValidUntil time.Time `json:"valid_until" yaml:"valid_until"` +} + // UpdateMarker is a marker holding necessary information about ongoing upgrade. type UpdateMarker struct { // Version represents the version the agent is upgraded to @@ -57,6 +64,8 @@ type UpdateMarker struct { Details *details.Details `json:"details,omitempty" yaml:"details,omitempty"` DesiredOutcome UpgradeOutcome `json:"desired_outcome" yaml:"desired_outcome"` + + RollbacksAvailable []RollbackAvailable `json:"rollbacks_available,omitempty" yaml:"rollbacks_available,omitempty"` } // GetActionID returns the Fleet Action ID associated with the @@ -103,32 +112,34 @@ func convertToActionUpgrade(a *MarkerActionUpgrade) *fleetapi.ActionUpgrade { } type updateMarkerSerializer struct { - Version string `yaml:"version"` - Hash string `yaml:"hash"` - VersionedHome string `yaml:"versioned_home"` - UpdatedOn time.Time `yaml:"updated_on"` - PrevVersion string `yaml:"prev_version"` - PrevHash string `yaml:"prev_hash"` - PrevVersionedHome string `yaml:"prev_versioned_home"` - Acked bool `yaml:"acked"` - Action *MarkerActionUpgrade `yaml:"action"` - Details *details.Details `yaml:"details"` - DesiredOutcome UpgradeOutcome `yaml:"desired_outcome"` + Version string `yaml:"version"` + Hash string `yaml:"hash"` + VersionedHome string `yaml:"versioned_home"` + UpdatedOn time.Time `yaml:"updated_on"` + PrevVersion string `yaml:"prev_version"` + PrevHash string `yaml:"prev_hash"` + PrevVersionedHome string `yaml:"prev_versioned_home"` + Acked bool `yaml:"acked"` + Action *MarkerActionUpgrade `yaml:"action"` + Details *details.Details `yaml:"details"` + DesiredOutcome UpgradeOutcome `yaml:"desired_outcome"` + RollbacksAvailable []RollbackAvailable `yaml:"rollbacks_available,omitempty"` } func newMarkerSerializer(m *UpdateMarker) *updateMarkerSerializer { return &updateMarkerSerializer{ - Version: m.Version, - Hash: m.Hash, - VersionedHome: m.VersionedHome, - UpdatedOn: m.UpdatedOn, - PrevVersion: m.PrevVersion, - PrevHash: m.PrevHash, - PrevVersionedHome: m.PrevVersionedHome, - Acked: m.Acked, - Action: convertToMarkerAction(m.Action), - Details: m.Details, - DesiredOutcome: m.DesiredOutcome, + Version: m.Version, + Hash: m.Hash, + VersionedHome: m.VersionedHome, + UpdatedOn: m.UpdatedOn, + PrevVersion: m.PrevVersion, + PrevHash: m.PrevHash, + PrevVersionedHome: m.PrevVersionedHome, + Acked: m.Acked, + Action: convertToMarkerAction(m.Action), + Details: m.Details, + DesiredOutcome: m.DesiredOutcome, + RollbacksAvailable: m.RollbacksAvailable, } } @@ -197,7 +208,7 @@ type agentInstall struct { } // markUpgrade marks update happened so we can handle grace period -func markUpgrade(log *logger.Logger, dataDirPath string, agent, previousAgent agentInstall, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details, desiredOutcome UpgradeOutcome) error { +func markUpgrade(log *logger.Logger, dataDirPath string, updatedOn time.Time, agent, previousAgent agentInstall, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details, desiredOutcome UpgradeOutcome, rollbackWindow time.Duration) error { if len(previousAgent.hash) > hashLen { previousAgent.hash = previousAgent.hash[:hashLen] @@ -207,7 +218,7 @@ func markUpgrade(log *logger.Logger, dataDirPath string, agent, previousAgent ag Version: agent.version, Hash: agent.hash, VersionedHome: agent.versionedHome, - UpdatedOn: time.Now(), + UpdatedOn: updatedOn, PrevVersion: previousAgent.version, PrevHash: previousAgent.hash, PrevVersionedHome: previousAgent.versionedHome, @@ -216,13 +227,26 @@ func markUpgrade(log *logger.Logger, dataDirPath string, agent, previousAgent ag DesiredOutcome: desiredOutcome, } + if rollbackWindow > 0 && agent.parsedVersion != nil && !agent.parsedVersion.Less(*Version_9_2_0_SNAPSHOT) { + // if we have a not empty rollback window, write the prev version in the rollbacks_available field + // we also need to check the destination version because the manual rollback and delayed cleanup will be + // handled by that version of agent, so it needs to be recent enough + marker.RollbacksAvailable = []RollbackAvailable{ + { + Version: previousAgent.version, + Home: previousAgent.versionedHome, + ValidUntil: updatedOn.Add(rollbackWindow), + }, + } + } + markerBytes, err := yaml.Marshal(newMarkerSerializer(marker)) if err != nil { return errors.New(err, errors.TypeConfig, "failed to parse marker file") } markerPath := markerFilePath(dataDirPath) - log.Infow("Writing upgrade marker file", "file.path", markerPath, "hash", marker.Hash, "prev_hash", marker.PrevHash) + log.Infow("Writing upgrade marker file", "file.path", markerPath, "hash", marker.Hash, "prev_hash", marker.PrevHash, "content", string(markerBytes)) if err := os.WriteFile(markerPath, markerBytes, 0600); err != nil { return errors.New(err, errors.TypeFilesystem, "failed to create update marker file", errors.M(errors.MetaKeyPath, markerPath)) } @@ -278,17 +302,18 @@ func loadMarker(markerFile string) (*UpdateMarker, error) { } return &UpdateMarker{ - Version: marker.Version, - Hash: marker.Hash, - VersionedHome: marker.VersionedHome, - UpdatedOn: marker.UpdatedOn, - PrevVersion: marker.PrevVersion, - PrevHash: marker.PrevHash, - PrevVersionedHome: marker.PrevVersionedHome, - Acked: marker.Acked, - Action: convertToActionUpgrade(marker.Action), - Details: marker.Details, - DesiredOutcome: marker.DesiredOutcome, + Version: marker.Version, + Hash: marker.Hash, + VersionedHome: marker.VersionedHome, + UpdatedOn: marker.UpdatedOn, + PrevVersion: marker.PrevVersion, + PrevHash: marker.PrevHash, + PrevVersionedHome: marker.PrevVersionedHome, + Acked: marker.Acked, + Action: convertToActionUpgrade(marker.Action), + Details: marker.Details, + DesiredOutcome: marker.DesiredOutcome, + RollbacksAvailable: marker.RollbacksAvailable, }, nil } @@ -301,17 +326,18 @@ func SaveMarker(dataDirPath string, marker *UpdateMarker, shouldFsync bool) erro func saveMarkerToPath(marker *UpdateMarker, markerFile string, shouldFsync bool) error { makerSerializer := &updateMarkerSerializer{ - Version: marker.Version, - Hash: marker.Hash, - VersionedHome: marker.VersionedHome, - UpdatedOn: marker.UpdatedOn, - PrevVersion: marker.PrevVersion, - PrevHash: marker.PrevHash, - PrevVersionedHome: marker.PrevVersionedHome, - Acked: marker.Acked, - Action: convertToMarkerAction(marker.Action), - Details: marker.Details, - DesiredOutcome: marker.DesiredOutcome, + Version: marker.Version, + Hash: marker.Hash, + VersionedHome: marker.VersionedHome, + UpdatedOn: marker.UpdatedOn, + PrevVersion: marker.PrevVersion, + PrevHash: marker.PrevHash, + PrevVersionedHome: marker.PrevVersionedHome, + Acked: marker.Acked, + Action: convertToMarkerAction(marker.Action), + Details: marker.Details, + DesiredOutcome: marker.DesiredOutcome, + RollbacksAvailable: marker.RollbacksAvailable, } markerBytes, err := yaml.Marshal(makerSerializer) if err != nil { diff --git a/internal/pkg/agent/application/upgrade/step_mark_test.go b/internal/pkg/agent/application/upgrade/step_mark_test.go index fc1731e7b24..ba8736ce915 100644 --- a/internal/pkg/agent/application/upgrade/step_mark_test.go +++ b/internal/pkg/agent/application/upgrade/step_mark_test.go @@ -7,13 +7,17 @@ package upgrade import ( "os" "path/filepath" + "runtime" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" "github.com/elastic/elastic-agent/internal/pkg/fleetapi" + "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" + agtversion "github.com/elastic/elastic-agent/pkg/version" ) func TestSaveAndLoadMarker_NoLoss(t *testing.T) { @@ -260,3 +264,228 @@ desired_outcome: true }) } } + +func TestMarkUpgrade(t *testing.T) { + var parsed123SNAPSHOT = agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "") + var parsed456SNAPSHOT = agtversion.NewParsedSemVer(4, 5, 6, "SNAPSHOT", "") + var parsed920SNAPSHOT = agtversion.NewParsedSemVer(9, 2, 0, "SNAPSHOT", "") + // fix a timestamp (truncated to the second because of loss of precision during marshalling/unmarshalling) + updatedOnNow := time.Now().UTC().Truncate(time.Second) + + type args struct { + updatedOn time.Time + currentAgent agentInstall + previousAgent agentInstall + action *fleetapi.ActionUpgrade + details *details.Details + desiredOutcome UpgradeOutcome + rollbackWindow time.Duration + } + type workingDirHook func(t *testing.T, dataDir string) + + testcases := []struct { + name string + setupBeforeMark workingDirHook + args args + wantErr assert.ErrorAssertionFunc + assertAfterMark workingDirHook + }{ + { + name: "error writing update marker - check error", + setupBeforeMark: func(t *testing.T, dataDir string) { + + // read-only permissions on directories don't work on windows, skip + if runtime.GOOS == "windows" { + t.Skip("skipping test on windows since readonly permissions on directory don't work") + } + + err := os.Chmod(dataDir, 0555) + require.NoError(t, err, "error setting dataDir read-only") + }, + args: args{ + updatedOn: updatedOnNow, + currentAgent: agentInstall{ + parsedVersion: parsed456SNAPSHOT, + version: "4.5.6-SNAPSHOT", + hash: "curagt", + versionedHome: filepath.Join("data", "elastic-agent-4.5.6-SNAPSHOT-curagt"), + }, + previousAgent: agentInstall{ + parsedVersion: parsed123SNAPSHOT, + version: "1.2.3-SNAPSHOT", + hash: "prvagt", + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), + }, + action: nil, + details: details.NewDetails("4.5.6-SNAPSHOT", details.StateReplacing, ""), + desiredOutcome: OUTCOME_UPGRADE, + rollbackWindow: 0, + }, + wantErr: assert.Error, + }, + { + name: "no rollback window specified - no available rollbacks", + args: args{ + updatedOn: updatedOnNow, + currentAgent: agentInstall{ + parsedVersion: parsed456SNAPSHOT, + version: "4.5.6-SNAPSHOT", + hash: "curagt", + versionedHome: filepath.Join("data", "elastic-agent-4.5.6-SNAPSHOT-curagt"), + }, + previousAgent: agentInstall{ + parsedVersion: parsed123SNAPSHOT, + version: "1.2.3-SNAPSHOT", + hash: "prvagt", + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), + }, + action: nil, + details: details.NewDetails("4.5.6-SNAPSHOT", details.StateReplacing, ""), + desiredOutcome: OUTCOME_UPGRADE, + rollbackWindow: 0, + }, + wantErr: assert.NoError, + assertAfterMark: func(t *testing.T, dataDir string) { + actualMarker, err := LoadMarker(dataDir) + require.NoError(t, err, "error reading actualMarker content after writing") + + expectedMarker := &UpdateMarker{ + Version: "4.5.6-SNAPSHOT", + Hash: "curagt", + VersionedHome: filepath.Join("data", "elastic-agent-4.5.6-SNAPSHOT-curagt"), + UpdatedOn: updatedOnNow, + PrevVersion: "1.2.3-SNAPSHOT", + PrevHash: "prvagt", + PrevVersionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), + Acked: false, + Action: nil, + Details: &details.Details{ + TargetVersion: "4.5.6-SNAPSHOT", + State: "UPG_REPLACING", + ActionID: "", + Metadata: details.Metadata{}, + }, + DesiredOutcome: OUTCOME_UPGRADE, + } + assert.Equal(t, expectedMarker, actualMarker) + }, + }, + { + name: "rollback window specified but new version is too low - no rollbacks", + args: args{ + updatedOn: updatedOnNow, + currentAgent: agentInstall{ + parsedVersion: parsed456SNAPSHOT, + version: "4.5.6-SNAPSHOT", + hash: "curagt", + versionedHome: filepath.Join("data", "elastic-agent-4.5.6-SNAPSHOT-curagt"), + }, + previousAgent: agentInstall{ + parsedVersion: parsed123SNAPSHOT, + version: "1.2.3-SNAPSHOT", + hash: "prvagt", + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), + }, + action: nil, + details: details.NewDetails("4.5.6-SNAPSHOT", details.StateReplacing, ""), + desiredOutcome: OUTCOME_UPGRADE, + rollbackWindow: 7 * 24 * time.Hour, + }, + wantErr: assert.NoError, + assertAfterMark: func(t *testing.T, dataDir string) { + actualMarker, err := LoadMarker(dataDir) + require.NoError(t, err, "error reading actualMarker content after writing") + + expectedMarker := &UpdateMarker{ + Version: "4.5.6-SNAPSHOT", + Hash: "curagt", + VersionedHome: filepath.Join("data", "elastic-agent-4.5.6-SNAPSHOT-curagt"), + UpdatedOn: updatedOnNow, + PrevVersion: "1.2.3-SNAPSHOT", + PrevHash: "prvagt", + PrevVersionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), + Acked: false, + Action: nil, + Details: &details.Details{ + TargetVersion: "4.5.6-SNAPSHOT", + State: "UPG_REPLACING", + ActionID: "", + }, + DesiredOutcome: OUTCOME_UPGRADE, + } + assert.Equal(t, expectedMarker, actualMarker) + }, + }, + { + name: "rollback window specified and new version is at least 9.2.0-SNAPSHOT - available rollbacks must be present", + args: args{ + updatedOn: updatedOnNow, + currentAgent: agentInstall{ + parsedVersion: parsed920SNAPSHOT, + version: "9.2.0-SNAPSHOT", + hash: "newagt", + versionedHome: filepath.Join("data", "elastic-agent-9.2.0-SNAPSHOT-newagt"), + }, + previousAgent: agentInstall{ + parsedVersion: parsed123SNAPSHOT, + version: "1.2.3-SNAPSHOT", + hash: "prvagt", + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), + }, + action: nil, + details: details.NewDetails("9.2.0-SNAPSHOT", details.StateReplacing, ""), + desiredOutcome: OUTCOME_UPGRADE, + rollbackWindow: 7 * 24 * time.Hour, + }, + wantErr: assert.NoError, + assertAfterMark: func(t *testing.T, dataDir string) { + actualMarker, err := LoadMarker(dataDir) + require.NoError(t, err, "error reading actualMarker content after writing") + + expectedMarker := &UpdateMarker{ + Version: "9.2.0-SNAPSHOT", + Hash: "newagt", + VersionedHome: filepath.Join("data", "elastic-agent-9.2.0-SNAPSHOT-newagt"), + UpdatedOn: updatedOnNow, + PrevVersion: "1.2.3-SNAPSHOT", + PrevHash: "prvagt", + PrevVersionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), + Acked: false, + Action: nil, + Details: &details.Details{ + TargetVersion: "9.2.0-SNAPSHOT", + State: "UPG_REPLACING", + ActionID: "", + Metadata: details.Metadata{}, + }, + DesiredOutcome: OUTCOME_UPGRADE, + RollbacksAvailable: []RollbackAvailable{ + { + Version: "1.2.3-SNAPSHOT", + Home: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), + ValidUntil: updatedOnNow.Add(7 * 24 * time.Hour), + }, + }, + } + assert.Equal(t, expectedMarker, actualMarker) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + dataDir := t.TempDir() + log, _ := loggertest.New(t.Name()) + + if tc.setupBeforeMark != nil { + tc.setupBeforeMark(t, dataDir) + } + + err := markUpgrade(log, dataDir, tc.args.updatedOn, tc.args.currentAgent, tc.args.previousAgent, tc.args.action, tc.args.details, tc.args.desiredOutcome, tc.args.rollbackWindow) + tc.wantErr(t, err) + if tc.assertAfterMark != nil { + tc.assertAfterMark(t, dataDir) + } + }) + } +} diff --git a/internal/pkg/agent/application/upgrade/step_relink.go b/internal/pkg/agent/application/upgrade/step_relink.go index f9256d9980d..d6fc9a6b9c1 100644 --- a/internal/pkg/agent/application/upgrade/step_relink.go +++ b/internal/pkg/agent/application/upgrade/step_relink.go @@ -15,14 +15,14 @@ import ( ) const ( - windows = "windows" - exe = ".exe" + windowsOSName = "windows" + exe = ".exe" ) func changeSymlink(log *logger.Logger, topDirPath, symlinkPath, newTarget string) error { // handle windows suffixes - if runtime.GOOS == windows { + if runtime.GOOS == windowsOSName { symlinkPath += exe newTarget += exe } @@ -47,7 +47,7 @@ func prevSymlinkPath(topDirPath string) string { agentPrevName := agentName + ".prev" // handle windows suffixes - if runtime.GOOS == windows { + if runtime.GOOS == windowsOSName { agentPrevName = agentName + ".exe.prev" } diff --git a/internal/pkg/agent/application/upgrade/step_unpack.go b/internal/pkg/agent/application/upgrade/step_unpack.go index 830fd1b0663..9989cae434e 100644 --- a/internal/pkg/agent/application/upgrade/step_unpack.go +++ b/internal/pkg/agent/application/upgrade/step_unpack.go @@ -24,6 +24,7 @@ import ( v1 "github.com/elastic/elastic-agent/pkg/api/v1" "github.com/elastic/elastic-agent/pkg/component" "github.com/elastic/elastic-agent/pkg/core/logger" + manifestutils "github.com/elastic/elastic-agent/pkg/utils/manifest" ) // UnpackResult contains the location and hash of the unpacked agent files @@ -41,7 +42,7 @@ func (u *Upgrader) unpack(version, archivePath, dataDir string, flavor string) ( // or the extraction will be double nested var unpackRes UnpackResult var err error - if runtime.GOOS == windows { + if runtime.GOOS == windowsOSName { unpackRes, err = unzip(u.log, archivePath, dataDir, flavor) } else { unpackRes, err = untar(u.log, archivePath, dataDir, flavor) @@ -88,7 +89,7 @@ func unzip(log *logger.Logger, archivePath, dataDir string, flavor string) (Unpa fileNamePrefix := strings.TrimSuffix(filepath.Base(archivePath), ".zip") + "/" // omitting `elastic-agent-{version}-{os}-{arch}/` in filename - pm := pathMapper{} + var pm *manifestutils.PathMapper var versionedHome string metadata, err := getPackageMetadataFromZipReader(r, fileNamePrefix) @@ -99,10 +100,11 @@ func unzip(log *logger.Logger, archivePath, dataDir string, flavor string) (Unpa hash = metadata.hash[:hashLen] var registry map[string][]string if metadata.manifest != nil { - pm.mappings = metadata.manifest.Package.PathMappings + pm = manifestutils.NewPathMapper(metadata.manifest.Package.PathMappings) versionedHome = filepath.FromSlash(pm.Map(metadata.manifest.Package.VersionedHome)) registry = metadata.manifest.Package.Flavors } else { + pm = manifestutils.NewPathMapper(nil) // if at this point we didn't load the manifest, set the versioned to the backup value versionedHome = createVersionedHomeFromHash(hash) } @@ -319,7 +321,7 @@ func untar(log *logger.Logger, archivePath, dataDir string, flavor string) (Unpa var hash string // Look up manifest in the archive and prepare path mappings, if any - pm := pathMapper{} + var pm *manifestutils.PathMapper metadata, err := getPackageMetadataFromTar(archivePath) if err != nil { @@ -331,10 +333,11 @@ func untar(log *logger.Logger, archivePath, dataDir string, flavor string) (Unpa if metadata.manifest != nil { // set the path mappings - pm.mappings = metadata.manifest.Package.PathMappings + pm = manifestutils.NewPathMapper(metadata.manifest.Package.PathMappings) versionedHome = filepath.FromSlash(pm.Map(metadata.manifest.Package.VersionedHome)) registry = metadata.manifest.Package.Flavors } else { + pm = manifestutils.NewPathMapper(nil) // set default value of versioned home if it wasn't set by reading the manifest versionedHome = createVersionedHomeFromHash(metadata.hash) } @@ -610,21 +613,6 @@ func validFileName(p string) bool { return true } -type pathMapper struct { - mappings []map[string]string -} - -func (pm pathMapper) Map(packagePath string) string { - for _, mapping := range pm.mappings { - for pkgPath, mappedPath := range mapping { - if strings.HasPrefix(packagePath, pkgPath) { - return path.Join(mappedPath, packagePath[len(pkgPath):]) - } - } - } - return packagePath -} - type tarCloser struct { tarFile *os.File gzipReader *gzip.Reader diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go index 19d3b67cb2b..5a00d2abf61 100644 --- a/internal/pkg/agent/application/upgrade/upgrade.go +++ b/internal/pkg/agent/application/upgrade/upgrade.go @@ -19,6 +19,7 @@ import ( "github.com/otiai10/copy" "go.elastic.co/apm/v2" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock" "github.com/elastic/elastic-agent/internal/pkg/agent/application/info" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/application/reexec" @@ -56,10 +57,16 @@ var agentArtifact = artifact.Artifact{ } var ( - ErrWatcherNotStarted = errors.New("watcher did not start in time") - ErrUpgradeSameVersion = errors.New("upgrade did not occur because it is the same version") - ErrNonFipsToFips = errors.New("cannot switch to fips mode when upgrading") - ErrFipsToNonFips = errors.New("cannot switch to non-fips mode when upgrading") + ErrWatcherNotStarted = errors.New("watcher did not start in time") + ErrUpgradeSameVersion = errors.New("upgrade did not occur because it is the same version") + ErrNonFipsToFips = errors.New("cannot switch to fips mode when upgrading") + ErrFipsToNonFips = errors.New("cannot switch to non-fips mode when upgrading") + ErrNilUpdateMarker = errors.New("loaded a nil update marker") + ErrEmptyRollbackVersion = errors.New("rollback version is empty") + ErrNoRollbacksAvailable = errors.New("no rollbacks available") + + // Version_9_2_0_SNAPSHOT is the minimum version for manual rollback and rollback reason + Version_9_2_0_SNAPSHOT = agtversion.NewParsedSemVer(9, 2, 0, "SNAPSHOT", "") ) func init() { @@ -68,14 +75,33 @@ func init() { } } +// WatcherHelper is an abstraction of operations that Upgrader will trigger on elastic-agent watcher. +// This is defined to help with Upgrader testing and verify interactions with elastic-agent watcher +type WatcherHelper interface { + // InvokeWatcher invokes an elastic-agent watcher using the agentExecutable passed as argument + InvokeWatcher(log *logger.Logger, agentExecutable string) (*exec.Cmd, error) + // SelectWatcherExecutable will return the path to the newer elastic-agent executable that will be used to invoke the + // more recent watcher between the previous (the agent that started the upgrade) and current (the agent that will run after restart) + // agent installation + SelectWatcherExecutable(topDir string, previous agentInstall, current agentInstall) string + // WaitForWatcher will listen for changes to the update marker, waiting for the elastic-agent watcher to set UPG_WATCHING state + // in the upgrade details' metadata + WaitForWatcher(ctx context.Context, log *logger.Logger, markerFilePath string, waitTime time.Duration) error + // TakeOverWatcher will look for watcher processes and terminate them while at the same time trying to acquire the watcher AppLocker. + // It will return once it managed to get the AppLocker or with an error if the lock could not be acquired. + TakeOverWatcher(ctx context.Context, log *logger.Logger, topDir string) (*filelock.AppLocker, error) +} + // Upgrader performs an upgrade type Upgrader struct { - log *logger.Logger - settings *artifact.Config - agentInfo info.Agent - upgradeable bool - fleetServerURI string - markerWatcher MarkerWatcher + log *logger.Logger + settings *artifact.Config + upgradeSettings *configuration.UpgradeConfig + agentInfo info.Agent + upgradeable bool + fleetServerURI string + markerWatcher MarkerWatcher + watcherHelper WatcherHelper } // IsUpgradeable when agent is installed and running as a service or flag was provided. @@ -86,13 +112,15 @@ func IsUpgradeable() bool { } // NewUpgrader creates an upgrader which is capable of performing upgrade operation -func NewUpgrader(log *logger.Logger, settings *artifact.Config, agentInfo info.Agent) (*Upgrader, error) { +func NewUpgrader(log *logger.Logger, settings *artifact.Config, upgradeConfig *configuration.UpgradeConfig, agentInfo info.Agent, watcherHelper WatcherHelper) (*Upgrader, error) { return &Upgrader{ - log: log, - settings: settings, - agentInfo: agentInfo, - upgradeable: IsUpgradeable(), - markerWatcher: newMarkerFileWatcher(markerFilePath(paths.Data()), log), + log: log, + settings: settings, + upgradeSettings: upgradeConfig, + agentInfo: agentInfo, + upgradeable: IsUpgradeable(), + markerWatcher: newMarkerFileWatcher(markerFilePath(paths.Data()), log), + watcherHelper: watcherHelper, }, nil } @@ -144,6 +172,8 @@ func (u *Upgrader) Reload(rawConfig *config.Config) error { } u.settings = cfg.Settings.DownloadConfig + u.upgradeSettings = cfg.Settings.Upgrade + return nil } @@ -194,7 +224,12 @@ func checkUpgrade(log *logger.Logger, currentVersion, newVersion agentVersion, m } // Upgrade upgrades running agent, function returns shutdown callback that must be called by reexec. -func (u *Upgrader) Upgrade(ctx context.Context, version string, sourceURI string, action *fleetapi.ActionUpgrade, det *details.Details, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) (_ reexec.ShutdownCallbackFn, err error) { +func (u *Upgrader) Upgrade(ctx context.Context, version string, rollback bool, sourceURI string, action *fleetapi.ActionUpgrade, det *details.Details, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) (_ reexec.ShutdownCallbackFn, err error) { + + if rollback { + return u.rollbackToPreviousVersion(ctx, paths.Top(), time.Now(), version, action) + } + u.log.Infow("Upgrading agent", "version", version, "source_uri", sourceURI) currentVersion := agentVersion{ @@ -338,27 +373,26 @@ func (u *Upgrader) Upgrade(ctx context.Context, version string, sourceURI string hash: release.Commit(), versionedHome: currentVersionedHome, } - - if err := markUpgrade(u.log, - paths.Data(), // data dir to place the marker in - current, // new agent version data - previous, // old agent version data - action, det, OUTCOME_UPGRADE); err != nil { + rollbackWindow := time.Duration(0) + if u.upgradeSettings != nil && u.upgradeSettings.Rollback != nil { + rollbackWindow = u.upgradeSettings.Rollback.Window + } + if err := markUpgrade(u.log, paths.Data(), time.Now(), current, previous, action, det, OUTCOME_UPGRADE, rollbackWindow); err != nil { u.log.Errorw("Rolling back: marking upgrade failed", "error.message", err) rollbackErr := rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome) return nil, goerrors.Join(err, rollbackErr) } - watcherExecutable := selectWatcherExecutable(paths.Top(), previous, current) + watcherExecutable := u.watcherHelper.SelectWatcherExecutable(paths.Top(), previous, current) var watcherCmd *exec.Cmd - if watcherCmd, err = InvokeWatcher(u.log, watcherExecutable); err != nil { + if watcherCmd, err = u.watcherHelper.InvokeWatcher(u.log, watcherExecutable); err != nil { u.log.Errorw("Rolling back: starting watcher failed", "error.message", err) rollbackErr := rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome) return nil, goerrors.Join(err, rollbackErr) } - watcherWaitErr := waitForWatcher(ctx, u.log, markerFilePath(paths.Data()), watcherMaxWaitTime) + watcherWaitErr := u.watcherHelper.WaitForWatcher(ctx, u.log, markerFilePath(paths.Data()), watcherMaxWaitTime) if watcherWaitErr != nil { killWatcherErr := watcherCmd.Process.Kill() rollbackErr := rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome) @@ -377,50 +411,132 @@ func (u *Upgrader) Upgrade(ctx context.Context, version string, sourceURI string return cb, nil } -func selectWatcherExecutable(topDir string, previous agentInstall, current agentInstall) string { - // check if the upgraded version is less than the previous (currently installed) version - if current.parsedVersion.Less(*previous.parsedVersion) { - // use the current agent executable for watch, if downgrading the old agent doesn't understand the current agent's path structure. - return paths.BinaryPath(filepath.Join(topDir, previous.versionedHome), agentName) - } else { - // use the new agent executable as it should be able to parse the new update marker - return paths.BinaryPath(filepath.Join(topDir, current.versionedHome), agentName) +func (u *Upgrader) rollbackToPreviousVersion(ctx context.Context, topDir string, now time.Time, version string, action *fleetapi.ActionUpgrade) (reexec.ShutdownCallbackFn, error) { + if version == "" { + return nil, ErrEmptyRollbackVersion } -} -func waitForWatcher(ctx context.Context, log *logger.Logger, markerFilePath string, waitTime time.Duration) error { - return waitForWatcherWithTimeoutCreationFunc(ctx, log, markerFilePath, waitTime, context.WithTimeout) -} + // check that the upgrade marker exists and is accessible + updateMarkerPath := markerFilePath(paths.DataFrom(topDir)) + _, err := os.Stat(updateMarkerPath) + if err != nil { + return nil, fmt.Errorf("stat() on upgrade marker %q failed: %w", updateMarkerPath, err) + } -type createContextWithTimeout func(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) + // read the upgrade marker + updateMarker, err := LoadMarker(paths.DataFrom(topDir)) + if err != nil { + return nil, fmt.Errorf("loading marker: %w", err) + } -func waitForWatcherWithTimeoutCreationFunc(ctx context.Context, log *logger.Logger, markerFilePath string, waitTime time.Duration, createTimeoutContext createContextWithTimeout) error { - // Wait for the watcher to be up and running - watcherContext, cancel := createTimeoutContext(ctx, waitTime) - defer cancel() + if updateMarker == nil { + return nil, ErrNilUpdateMarker + } - markerWatcher := newMarkerFileWatcher(markerFilePath, log) - err := markerWatcher.Run(watcherContext) + // extract the agent installs involved in the upgrade and select the most appropriate watcher executable + previous, current, err := extractAgentInstallsFromMarker(updateMarker) if err != nil { - return fmt.Errorf("error starting update marker watcher: %w", err) + return nil, fmt.Errorf("extracting current and previous install details: %w", err) } + watcherExecutable := u.watcherHelper.SelectWatcherExecutable(topDir, previous, current) - log.Infof("waiting up to %s for upgrade watcher to set %s state in upgrade marker", waitTime, details.StateWatching) + err = withTakeOverWatcher(ctx, u.log, topDir, u.watcherHelper, func() error { + // read the upgrade marker + updateMarker, err = LoadMarker(paths.DataFrom(topDir)) + if err != nil { + return fmt.Errorf("loading marker: %w", err) + } - for { - select { - case updMarker := <-markerWatcher.Watch(): - if updMarker.Details != nil && updMarker.Details.State == details.StateWatching { - // watcher started and it is watching, all good - log.Infof("upgrade watcher set %s state in upgrade marker: exiting wait loop", details.StateWatching) - return nil + if updateMarker == nil { + return ErrNilUpdateMarker + } + + if len(updateMarker.RollbacksAvailable) == 0 { + return ErrNoRollbacksAvailable + } + var selectedRollback *RollbackAvailable + for _, rollback := range updateMarker.RollbacksAvailable { + if rollback.Version == version && now.Before(rollback.ValidUntil) { + selectedRollback = &rollback + break } + } + if selectedRollback == nil { + return fmt.Errorf("version %q not listed among the available rollbacks: %w", version, ErrNoRollbacksAvailable) + } + + // write the desired outcome of the upgrade + err = u.persistManualRollback(topDir, updateMarker) + if err != nil { + return fmt.Errorf("persisting rollback in update marker: %w", err) + } + + return nil + }) + + // Invoke watcher again (now that we released the watcher applocks) + _, invokeWatcherErr := u.watcherHelper.InvokeWatcher(u.log, watcherExecutable) + if invokeWatcherErr != nil { + return nil, goerrors.Join(err, fmt.Errorf("invoking watcher: %w", invokeWatcherErr)) + } + + if err != nil { + return nil, err + } + + return nil, nil - case <-watcherContext.Done(): - log.Errorf("upgrade watcher did not start watching within %s or context has expired", waitTime) - return goerrors.Join(ErrWatcherNotStarted, watcherContext.Err()) +} + +func withTakeOverWatcher(ctx context.Context, log *logger.Logger, topDir string, watcherHelper WatcherHelper, f func() error) error { + watcherApplock, err := watcherHelper.TakeOverWatcher(ctx, log, topDir) + if err != nil { + return fmt.Errorf("taking over watcher processes: %w", err) + } + defer func(watcherApplock *filelock.AppLocker) { + releaseWatcherAppLockerErr := watcherApplock.Unlock() + if releaseWatcherAppLockerErr != nil { + log.Warnw("error releasing watcher applock", "error", releaseWatcherAppLockerErr) } + }(watcherApplock) + + return f() +} + +func extractAgentInstallsFromMarker(updateMarker *UpdateMarker) (previous agentInstall, current agentInstall, err error) { + previousParsedVersion, err := agtversion.ParseVersion(updateMarker.PrevVersion) + if err != nil { + return previous, current, fmt.Errorf("parsing previous version %q: %w", updateMarker.PrevVersion, err) + } + previous = agentInstall{ + parsedVersion: previousParsedVersion, + version: updateMarker.PrevVersion, + hash: updateMarker.PrevHash, + versionedHome: updateMarker.PrevVersionedHome, + } + + currentParsedVersion, err := agtversion.ParseVersion(updateMarker.Version) + if err != nil { + return previous, current, fmt.Errorf("parsing current version %q: %w", updateMarker.Version, err) + } + current = agentInstall{ + parsedVersion: currentParsedVersion, + version: updateMarker.Version, + hash: updateMarker.Hash, + versionedHome: updateMarker.VersionedHome, } + + return previous, current, nil +} + +func (u *Upgrader) persistManualRollback(topDir string, updateMarker *UpdateMarker) error { + updateMarker.DesiredOutcome = OUTCOME_ROLLBACK + err := SaveMarker(paths.DataFrom(topDir), updateMarker, true) + if err != nil { + return fmt.Errorf("saving marker: %w", err) + } + + return nil } // Ack acks last upgrade action diff --git a/internal/pkg/agent/application/upgrade/upgrade_test.go b/internal/pkg/agent/application/upgrade/upgrade_test.go index 17d19252f6e..4030c266d2c 100644 --- a/internal/pkg/agent/application/upgrade/upgrade_test.go +++ b/internal/pkg/agent/application/upgrade/upgrade_test.go @@ -8,10 +8,11 @@ import ( "context" "crypto/tls" "fmt" + "io/fs" "os" + "os/exec" "path/filepath" "runtime" - "sync" "testing" "time" @@ -19,17 +20,16 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" "github.com/elastic/elastic-agent-libs/transport/httpcommon" "github.com/elastic/elastic-agent-libs/transport/tlscommon" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact" - "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" + "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/internal/pkg/config" "github.com/elastic/elastic-agent/internal/pkg/fleetapi" - "github.com/elastic/elastic-agent/internal/pkg/fleetapi/acker" "github.com/elastic/elastic-agent/internal/pkg/release" v1 "github.com/elastic/elastic-agent/pkg/api/v1" "github.com/elastic/elastic-agent/pkg/control/v2/client" @@ -37,7 +37,9 @@ import ( "github.com/elastic/elastic-agent/pkg/core/logger" "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" agtversion "github.com/elastic/elastic-agent/pkg/version" - mocks "github.com/elastic/elastic-agent/testing/mocks/pkg/control/v2/client" + infomocks "github.com/elastic/elastic-agent/testing/mocks/internal_/pkg/agent/application/info" + ackermocks "github.com/elastic/elastic-agent/testing/mocks/internal_/pkg/fleetapi/acker" + clientmocks "github.com/elastic/elastic-agent/testing/mocks/pkg/control/v2/client" ) func Test_CopyFile(t *testing.T) { @@ -239,7 +241,7 @@ func TestIsInProgress(t *testing.T) { t.Run(name, func(t *testing.T) { // Expect client.State() call to be made only if no Upgrade Watcher PIDs // are returned (i.e. no Upgrade Watcher is found to be running). - mc := mocks.NewClient(t) + mc := clientmocks.NewClient(t) if test.watcherPIDsFetcher != nil { pids, _ := test.watcherPIDsFetcher() if len(pids) == 0 { @@ -293,37 +295,31 @@ func TestUpgraderAckAction(t *testing.T) { require.Nil(t, u.AckAction(t.Context(), nil, action)) }) t.Run("AckAction with acker", func(t *testing.T) { - acker := &fakeAcker{} - acker.On("Ack", mock.Anything, action).Return(nil) - acker.On("Commit", mock.Anything).Return(nil) + mockAcker := ackermocks.NewAcker(t) + mockAcker.EXPECT().Ack(mock.Anything, action).Return(nil) + mockAcker.EXPECT().Commit(mock.Anything).Return(nil) - require.Nil(t, u.AckAction(t.Context(), acker, action)) - acker.AssertCalled(t, "Ack", mock.Anything, action) - acker.AssertCalled(t, "Commit", mock.Anything) + require.Nil(t, u.AckAction(t.Context(), mockAcker, action)) }) t.Run("AckAction with acker - failing commit", func(t *testing.T) { - acker := &fakeAcker{} + mockAcker := ackermocks.NewAcker(t) errCommit := errors.New("failed commit") - acker.On("Ack", mock.Anything, action).Return(nil) - acker.On("Commit", mock.Anything).Return(errCommit) + mockAcker.EXPECT().Ack(mock.Anything, action).Return(nil) + mockAcker.EXPECT().Commit(mock.Anything).Return(errCommit) - require.ErrorIs(t, u.AckAction(t.Context(), acker, action), errCommit) - acker.AssertCalled(t, "Ack", mock.Anything, action) - acker.AssertCalled(t, "Commit", mock.Anything) + require.ErrorIs(t, u.AckAction(t.Context(), mockAcker, action), errCommit) }) t.Run("AckAction with acker - failed ack", func(t *testing.T) { - acker := &fakeAcker{} + mockAcker := ackermocks.NewAcker(t) errAck := errors.New("ack error") - acker.On("Ack", mock.Anything, action).Return(errAck) - acker.On("Commit", mock.Anything).Return(nil) + mockAcker.EXPECT().Ack(mock.Anything, action).Return(errAck) + // no expectation on Commit() since it shouldn't be called after an error during Ack() - require.ErrorIs(t, u.AckAction(t.Context(), acker, action), errAck) - acker.AssertCalled(t, "Ack", mock.Anything, action) - acker.AssertNotCalled(t, "Commit", mock.Anything) + require.ErrorIs(t, u.AckAction(t.Context(), mockAcker, action), errAck) }) } @@ -959,242 +955,6 @@ func TestCheckUpgrade(t *testing.T) { } } -func TestWaitForWatcher(t *testing.T) { - wantErrWatcherNotStarted := func(t assert.TestingT, err error, i ...interface{}) bool { - return assert.ErrorIs(t, err, ErrWatcherNotStarted, i) - } - - tests := []struct { - name string - states []details.State - stateChangeInterval time.Duration - cancelWaitContext bool - wantErr assert.ErrorAssertionFunc - }{ - { - name: "Happy path: watcher is watching already", - states: []details.State{details.StateWatching}, - stateChangeInterval: 1 * time.Millisecond, - wantErr: assert.NoError, - }, - { - name: "Sad path: watcher is never starting", - states: []details.State{details.StateReplacing}, - stateChangeInterval: 1 * time.Millisecond, - cancelWaitContext: true, - wantErr: wantErrWatcherNotStarted, - }, - { - name: "Runaround path: marker is jumping around and landing on watching", - states: []details.State{ - details.StateRequested, - details.StateScheduled, - details.StateDownloading, - details.StateExtracting, - details.StateReplacing, - details.StateRestarting, - details.StateWatching, - }, - stateChangeInterval: 1 * time.Millisecond, - wantErr: assert.NoError, - }, - { - name: "Timeout: marker is never created", - states: nil, - stateChangeInterval: 1 * time.Millisecond, - cancelWaitContext: true, - wantErr: wantErrWatcherNotStarted, - }, - { - name: "Timeout2: state doesn't get there in time", - states: []details.State{ - details.StateRequested, - details.StateScheduled, - details.StateDownloading, - details.StateExtracting, - details.StateReplacing, - details.StateRestarting, - }, - - stateChangeInterval: 1 * time.Millisecond, - cancelWaitContext: true, - wantErr: wantErrWatcherNotStarted, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - deadline, ok := t.Deadline() - if !ok { - deadline = time.Now().Add(5 * time.Second) - } - testCtx, testCancel := context.WithDeadline(context.Background(), deadline) - defer testCancel() - - tmpDir := t.TempDir() - updMarkerFilePath := filepath.Join(tmpDir, markerFilename) - - waitContext, waitCancel := context.WithCancel(testCtx) - defer waitCancel() - - fakeTimeout := 30 * time.Second - - // in order to take timing out of the equation provide a context that we can cancel manually - // still assert that the parent context and timeout passed are correct - var createContextFunc createContextWithTimeout = func(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { - assert.Same(t, testCtx, ctx, "parent context should be the same as the waitForWatcherCall") - assert.Equal(t, fakeTimeout, timeout, "timeout used in new context should be the same as testcase") - - return waitContext, waitCancel - } - - if len(tt.states) > 0 { - initialState := tt.states[0] - writeState(t, updMarkerFilePath, initialState) - } - - wg := new(sync.WaitGroup) - - var furtherStates []details.State - if len(tt.states) > 1 { - // we have more states to produce - furtherStates = tt.states[1:] - } - - wg.Add(1) - - // worker goroutine: writes out additional states while the test is blocked on waitOnWatcher() call and expires - // the wait context if cancelWaitContext is set to true. Timing of the goroutine is driven by stateChangeInterval. - go func() { - defer wg.Done() - tick := time.NewTicker(tt.stateChangeInterval) - defer tick.Stop() - for _, state := range furtherStates { - select { - case <-testCtx.Done(): - return - case <-tick.C: - writeState(t, updMarkerFilePath, state) - } - } - if tt.cancelWaitContext { - <-tick.C - waitCancel() - } - }() - - log, _ := loggertest.New(tt.name) - - tt.wantErr(t, waitForWatcherWithTimeoutCreationFunc(testCtx, log, updMarkerFilePath, fakeTimeout, createContextFunc), fmt.Sprintf("waitForWatcher %s, %v, %s, %s)", updMarkerFilePath, tt.states, tt.stateChangeInterval, fakeTimeout)) - - // wait for goroutines to finish - wg.Wait() - }) - } -} - -func writeState(t *testing.T, path string, state details.State) { - ms := newMarkerSerializer(&UpdateMarker{ - Version: "version", - Hash: "hash", - VersionedHome: "versionedHome", - UpdatedOn: time.Now(), - PrevVersion: "prev_version", - PrevHash: "prev_hash", - PrevVersionedHome: "prev_versionedhome", - Acked: false, - Action: nil, - Details: &details.Details{ - TargetVersion: "version", - State: state, - ActionID: "", - Metadata: details.Metadata{}, - }, - }) - - bytes, err := yaml.Marshal(ms) - if assert.NoError(t, err, "error marshaling the test upgrade marker") { - err = os.WriteFile(path, bytes, 0770) - assert.NoError(t, err, "error writing out the test upgrade marker") - } -} - -func Test_selectWatcherExecutable(t *testing.T) { - type args struct { - previous agentInstall - current agentInstall - } - tests := []struct { - name string - args args - want string - }{ - { - name: "Simple upgrade, we should launch the new (current) watcher", - args: args{ - previous: agentInstall{ - parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "", ""), - versionedHome: filepath.Join("data", "elastic-agent-1.2.3-somehash"), - }, - current: agentInstall{ - parsedVersion: agtversion.NewParsedSemVer(4, 5, 6, "", ""), - versionedHome: filepath.Join("data", "elastic-agent-4.5.6-someotherhash"), - }, - }, - want: filepath.Join("data", "elastic-agent-4.5.6-someotherhash"), - }, - { - name: "Simple downgrade, we should launch the currently installed (previous) watcher", - args: args{ - previous: agentInstall{ - parsedVersion: agtversion.NewParsedSemVer(4, 5, 6, "", ""), - versionedHome: filepath.Join("data", "elastic-agent-4.5.6-someotherhash"), - }, - current: agentInstall{ - parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "", ""), - versionedHome: filepath.Join("data", "elastic-agent-1.2.3-somehash"), - }, - }, - want: filepath.Join("data", "elastic-agent-4.5.6-someotherhash"), - }, - { - name: "Upgrade from snapshot to released version, we should launch the new (current) watcher", - args: args{ - previous: agentInstall{ - parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", ""), - versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-somehash"), - }, - current: agentInstall{ - parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "", ""), - versionedHome: filepath.Join("data", "elastic-agent-1.2.3-someotherhash"), - }, - }, - want: filepath.Join("data", "elastic-agent-1.2.3-someotherhash"), - }, - { - name: "Downgrade from released version to SNAPSHOT, we should launch the currently installed (previous) watcher", - args: args{ - previous: agentInstall{ - parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "", ""), - versionedHome: filepath.Join("data", "elastic-agent-1.2.3-somehash"), - }, - current: agentInstall{ - parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", ""), - versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-someotherhash"), - }, - }, - - want: filepath.Join("data", "elastic-agent-1.2.3-somehash"), - }, - } - // Just need a top dir path. This test does not make any operation on the filesystem, so a temp dir path is as good as any - fakeTopDir := filepath.Join(t.TempDir(), "Elastic", "Agent") - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, paths.BinaryPath(filepath.Join(fakeTopDir, tt.want), agentName), selectWatcherExecutable(fakeTopDir, tt.args.previous, tt.args.current), "selectWatcherExecutable(%v, %v)", tt.args.previous, tt.args.current) - }) - } -} - func TestIsSameReleaseVersion(t *testing.T) { tests := []struct { name string @@ -1277,18 +1037,270 @@ func TestIsSameReleaseVersion(t *testing.T) { } } -var _ acker.Acker = &fakeAcker{} +func TestManualRollback(t *testing.T) { + const updatemarkerwatching456NoRollbackAvailable = ` + version: 4.5.6 + hash: newver + versioned_home: data/elastic-agent-4.5.6-newver + updated_on: 2025-07-11T10:11:12.131415Z + prev_version: 1.2.3 + prev_hash: oldver + prev_versioned_home: data/elastic-agent-1.2.3-oldver + acked: false + action: null + details: + target_version: 4.5.6 + state: UPG_WATCHING + metadata: + retry_until: null + desired_outcome: UPGRADE + ` + const updatemarkerwatching456 = ` + version: 4.5.6 + hash: newver + versioned_home: data/elastic-agent-4.5.6-newver + updated_on: 2025-07-11T10:11:12.131415Z + prev_version: 1.2.3 + prev_hash: oldver + prev_versioned_home: data/elastic-agent-1.2.3-oldver + acked: false + action: null + details: + target_version: 4.5.6 + state: UPG_WATCHING + metadata: + retry_until: null + desired_outcome: UPGRADE + rollbacks_available: + - version: 1.2.3 + home: data/elastic-agent-1.2.3-oldver + valid_until: 2025-07-18T10:11:12.131415Z + ` + + parsed123Version, err := agtversion.ParseVersion("1.2.3") + require.NoError(t, err) + parsed456Version, err := agtversion.ParseVersion("4.5.6") + require.NoError(t, err) + + agentInstall123 := agentInstall{ + parsedVersion: parsed123Version, + version: "1.2.3", + hash: "oldver", + versionedHome: "data/elastic-agent-1.2.3-oldver", + } -type fakeAcker struct { - mock.Mock -} + agentInstall456 := agentInstall{ + parsedVersion: parsed456Version, + version: "4.5.6", + hash: "newver", + versionedHome: "data/elastic-agent-4.5.6-newver", + } -func (f *fakeAcker) Ack(ctx context.Context, action fleetapi.Action) error { - args := f.Called(ctx, action) - return args.Error(0) -} + // this is the updated_on timestamp in the example + nowBeforeTTL, err := time.Parse(time.RFC3339, `2025-07-11T10:11:12Z`) + require.NoError(t, err, "error parsing nowBeforeTTL") -func (f *fakeAcker) Commit(ctx context.Context) error { - args := f.Called(ctx) - return args.Error(0) + // the update marker yaml assume 7d TLL for rollbacks, let's make an extra day pass + nowAfterTTL := nowBeforeTTL.Add(8 * 24 * time.Hour) + + type setupF func(t *testing.T, topDir string, agent *infomocks.Agent, watcherHelper *MockWatcherHelper) + type postRollbackAssertionsF func(t *testing.T, topDir string) + type testcase struct { + name string + setup setupF + artifactSettings *artifact.Config + upgradeSettings *configuration.UpgradeConfig + now time.Time + version string + wantErr assert.ErrorAssertionFunc + additionalAsserts postRollbackAssertionsF + } + + testcases := []testcase{ + { + name: "no rollback version - rollback fails", + setup: func(t *testing.T, topDir string, agent *infomocks.Agent, watcherHelper *MockWatcherHelper) { + //do not setup anything here, let the rollback fail + }, + artifactSettings: artifact.DefaultConfig(), + upgradeSettings: configuration.DefaultUpgradeConfig(), + version: "", + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorIs(t, err, ErrEmptyRollbackVersion) + }, + additionalAsserts: nil, + }, + { + name: "no update marker - rollback fails", + setup: func(t *testing.T, topDir string, agent *infomocks.Agent, watcherHelper *MockWatcherHelper) { + //do not setup anything here, let the rollback fail + }, + artifactSettings: artifact.DefaultConfig(), + upgradeSettings: configuration.DefaultUpgradeConfig(), + version: "1.2.3", + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorIs(t, err, fs.ErrNotExist) + }, + additionalAsserts: nil, + }, + { + name: "update marker is malformed - rollback fails", + setup: func(t *testing.T, topDir string, agent *infomocks.Agent, watcherHelper *MockWatcherHelper) { + err := os.WriteFile(markerFilePath(paths.DataFrom(topDir)), []byte("this is not a proper YAML file"), 0600) + require.NoError(t, err, "error setting up update marker") + locker := filelock.NewAppLocker(topDir, "watcher.lock") + err = locker.TryLock() + require.NoError(t, err, "error locking initial watcher AppLocker") + // there's no takeover watcher so no expectation on that or InvokeWatcher + t.Cleanup(func() { + unlockErr := locker.Unlock() + assert.NoError(t, unlockErr, "error unlocking initial watcher AppLocker") + }) + }, + artifactSettings: artifact.DefaultConfig(), + upgradeSettings: configuration.DefaultUpgradeConfig(), + version: "1.2.3", + wantErr: assert.Error, + additionalAsserts: nil, + }, + { + name: "update marker ok but rollback available is empty - error", + setup: func(t *testing.T, topDir string, agent *infomocks.Agent, watcherHelper *MockWatcherHelper) { + err := os.WriteFile(markerFilePath(paths.DataFrom(topDir)), []byte(updatemarkerwatching456NoRollbackAvailable), 0600) + require.NoError(t, err, "error setting up update marker") + locker := filelock.NewAppLocker(topDir, "watcher.lock") + err = locker.TryLock() + require.NoError(t, err, "error locking initial watcher AppLocker") + watcherHelper.EXPECT().TakeOverWatcher(t.Context(), mock.Anything, topDir).Return(locker, nil) + newerWatcherExecutable := filepath.Join(topDir, "data", "elastic-agent-4.5.6-newver", "elastic-agent") + watcherHelper.EXPECT().SelectWatcherExecutable(topDir, agentInstall123, agentInstall456).Return(newerWatcherExecutable) + watcherHelper.EXPECT().InvokeWatcher(mock.Anything, newerWatcherExecutable).Return(&exec.Cmd{Path: newerWatcherExecutable, Args: []string{"watch", "for realsies"}}, nil) + }, + artifactSettings: artifact.DefaultConfig(), + upgradeSettings: configuration.DefaultUpgradeConfig(), + version: "2.3.4-unknown", + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorIs(t, err, ErrNoRollbacksAvailable) + }, + additionalAsserts: func(t *testing.T, topDir string) { + // marker should be untouched + filePath := markerFilePath(paths.DataFrom(topDir)) + require.FileExists(t, filePath) + markerFileBytes, readMarkerErr := os.ReadFile(filePath) + require.NoError(t, readMarkerErr) + + assert.YAMLEq(t, updatemarkerwatching456NoRollbackAvailable, string(markerFileBytes), "update marker should be untouched") + }, + }, + { + name: "update marker ok but version is not available for rollback - error", + setup: func(t *testing.T, topDir string, agent *infomocks.Agent, watcherHelper *MockWatcherHelper) { + err := os.WriteFile(markerFilePath(paths.DataFrom(topDir)), []byte(updatemarkerwatching456), 0600) + require.NoError(t, err, "error setting up update marker") + locker := filelock.NewAppLocker(topDir, "watcher.lock") + err = locker.TryLock() + require.NoError(t, err, "error locking initial watcher AppLocker") + watcherHelper.EXPECT().TakeOverWatcher(t.Context(), mock.Anything, topDir).Return(locker, nil) + newerWatcherExecutable := filepath.Join(topDir, "data", "elastic-agent-4.5.6-newver", "elastic-agent") + watcherHelper.EXPECT().SelectWatcherExecutable(topDir, agentInstall123, agentInstall456).Return(newerWatcherExecutable) + watcherHelper.EXPECT().InvokeWatcher(mock.Anything, newerWatcherExecutable).Return(&exec.Cmd{Path: newerWatcherExecutable, Args: []string{"watch", "for realsies"}}, nil) + }, + artifactSettings: artifact.DefaultConfig(), + upgradeSettings: configuration.DefaultUpgradeConfig(), + version: "2.3.4-unknown", + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorIs(t, err, ErrNoRollbacksAvailable) + }, + additionalAsserts: func(t *testing.T, topDir string) { + // marker should be untouched + filePath := markerFilePath(paths.DataFrom(topDir)) + require.FileExists(t, filePath) + markerFileBytes, readMarkerErr := os.ReadFile(filePath) + require.NoError(t, readMarkerErr) + + assert.YAMLEq(t, updatemarkerwatching456, string(markerFileBytes), "update marker should be untouched") + }, + }, + { + name: "update marker ok but rollback is expired - error", + setup: func(t *testing.T, topDir string, agent *infomocks.Agent, watcherHelper *MockWatcherHelper) { + err := os.WriteFile(markerFilePath(paths.DataFrom(topDir)), []byte(updatemarkerwatching456), 0600) + require.NoError(t, err, "error setting up update marker") + locker := filelock.NewAppLocker(topDir, "watcher.lock") + err = locker.TryLock() + require.NoError(t, err, "error locking initial watcher AppLocker") + watcherHelper.EXPECT().TakeOverWatcher(t.Context(), mock.Anything, topDir).Return(locker, nil) + newerWatcherExecutable := filepath.Join(topDir, "data", "elastic-agent-4.5.6-newver", "elastic-agent") + watcherHelper.EXPECT().SelectWatcherExecutable(topDir, agentInstall123, agentInstall456).Return(newerWatcherExecutable) + watcherHelper.EXPECT().InvokeWatcher(mock.Anything, newerWatcherExecutable).Return(&exec.Cmd{Path: newerWatcherExecutable, Args: []string{"watch", "for realsies"}}, nil) + }, + artifactSettings: artifact.DefaultConfig(), + upgradeSettings: configuration.DefaultUpgradeConfig(), + now: nowAfterTTL, + version: "1.2.3", + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorIs(t, err, ErrNoRollbacksAvailable) + }, + additionalAsserts: func(t *testing.T, topDir string) { + // marker should be untouched + filePath := markerFilePath(paths.DataFrom(topDir)) + require.FileExists(t, filePath) + markerFileBytes, readMarkerErr := os.ReadFile(filePath) + require.NoError(t, readMarkerErr) + + assert.YAMLEq(t, updatemarkerwatching456, string(markerFileBytes), "update marker should be untouched") + }, + }, + { + name: "update marker ok - takeover watcher, persist rollback and restart most recent watcher", + setup: func(t *testing.T, topDir string, agent *infomocks.Agent, watcherHelper *MockWatcherHelper) { + err := os.WriteFile(markerFilePath(paths.DataFrom(topDir)), []byte(updatemarkerwatching456), 0600) + require.NoError(t, err, "error setting up update marker") + locker := filelock.NewAppLocker(topDir, "watcher.lock") + err = locker.TryLock() + require.NoError(t, err, "error locking initial watcher AppLocker") + watcherHelper.EXPECT().TakeOverWatcher(t.Context(), mock.Anything, topDir).Return(locker, nil) + newerWatcherExecutable := filepath.Join(topDir, "data", "elastic-agent-4.5.6-newver", "elastic-agent") + watcherHelper.EXPECT().SelectWatcherExecutable(topDir, agentInstall123, agentInstall456).Return(newerWatcherExecutable) + watcherHelper.EXPECT().InvokeWatcher(mock.Anything, newerWatcherExecutable).Return(&exec.Cmd{Path: newerWatcherExecutable, Args: []string{"watch", "for realsies"}}, nil) + }, + artifactSettings: artifact.DefaultConfig(), + upgradeSettings: configuration.DefaultUpgradeConfig(), + now: nowBeforeTTL, + version: "1.2.3", + wantErr: assert.NoError, + additionalAsserts: func(t *testing.T, topDir string) { + marker, loadMarkerErr := LoadMarker(paths.DataFrom(topDir)) + require.NoError(t, loadMarkerErr, "error loading marker") + require.NotNil(t, marker, "marker is nil") + + assert.Equal(t, OUTCOME_ROLLBACK, marker.DesiredOutcome) + require.NotNil(t, marker.Details) + assert.NotEmpty(t, marker.RollbacksAvailable) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + log, _ := loggertest.New(t.Name()) + mockAgentInfo := infomocks.NewAgent(t) + mockWatcherHelper := NewMockWatcherHelper(t) + topDir := t.TempDir() + err := os.MkdirAll(paths.DataFrom(topDir), 0777) + require.NoError(t, err, "error creating data directory in topDir %q", topDir) + + if tc.setup != nil { + tc.setup(t, topDir, mockAgentInfo, mockWatcherHelper) + } + + upgrader, err := NewUpgrader(log, tc.artifactSettings, tc.upgradeSettings, mockAgentInfo, mockWatcherHelper) + require.NoError(t, err, "error instantiating upgrader") + _, err = upgrader.rollbackToPreviousVersion(t.Context(), topDir, tc.now, tc.version, nil) + tc.wantErr(t, err, "unexpected error returned by rollbackToPreviousVersion()") + if tc.additionalAsserts != nil { + tc.additionalAsserts(t, topDir) + } + }) + } } diff --git a/internal/pkg/agent/application/upgrade/watcher.go b/internal/pkg/agent/application/upgrade/watcher.go index df7ee03df70..b90cab647f9 100644 --- a/internal/pkg/agent/application/upgrade/watcher.go +++ b/internal/pkg/agent/application/upgrade/watcher.go @@ -8,11 +8,15 @@ import ( "context" "errors" "fmt" + "os/exec" + "path/filepath" "time" "google.golang.org/grpc" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" "github.com/elastic/elastic-agent/pkg/control/v2/client" "github.com/elastic/elastic-agent/pkg/core/logger" ) @@ -21,6 +25,8 @@ const ( statusCheckMissesAllowed = 4 // enable 2 minute start (30 second periods) statusLossesAllowed = 2 // enable connection lost to agent twice statusFailureFlipFlopsAllowed = 3 // no more than three failure flip-flops allowed + + watcherApplockerFileName = "watcher.lock" ) var ( @@ -257,3 +263,131 @@ func (ch *AgentWatcher) checkFailures() bool { } return false } + +// Ensure that AgentWatcherHelper implements the WatcherHelper interface +var _ WatcherHelper = &AgentWatcherHelper{} + +type AgentWatcherHelper struct { +} + +func (a AgentWatcherHelper) InvokeWatcher(log *logger.Logger, agentExecutable string) (*exec.Cmd, error) { + return InvokeWatcher(log, agentExecutable) +} + +func (a AgentWatcherHelper) SelectWatcherExecutable(topDir string, previous agentInstall, current agentInstall) string { + return selectWatcherExecutable(topDir, previous, current) +} + +func (a AgentWatcherHelper) WaitForWatcher(ctx context.Context, log *logger.Logger, markerFilePath string, waitTime time.Duration) error { + return waitForWatcher(ctx, log, markerFilePath, waitTime) +} + +func (a AgentWatcherHelper) TakeOverWatcher(ctx context.Context, log *logger.Logger, topDir string) (*filelock.AppLocker, error) { + return takeOverWatcher(ctx, log, new(commandWatcherGrappler), topDir, 30*time.Second, 500*time.Millisecond, 100*time.Millisecond) +} + +// watcherGrappler is an abstraction over the way elastic-agent main process should take down (stop, gracefully if possible) a watcher process +type watcherGrappler interface { + TakeDownWatcher(ctx context.Context, log *logger.Logger) error +} + +type commandWatcherGrappler struct{} + +func (c commandWatcherGrappler) TakeDownWatcher(ctx context.Context, log *logger.Logger) error { + cmd := createTakeDownWatcherCommand(ctx) + log.Debugf("launching takedown with %v", cmd.Args) + output, err := cmd.CombinedOutput() + log.Debugf("takedown output: %s", string(output)) + if err != nil { + return fmt.Errorf("watcher command takedown failed: %w", err) + } + return nil +} + +// Private functions providing implementation of AgentWatcherHelper +func takeOverWatcher(ctx context.Context, log *logger.Logger, watcherGrappler watcherGrappler, topDir string, timeout time.Duration, watcherSweepInterval time.Duration, takeOverInterval time.Duration) (*filelock.AppLocker, error) { + takeoverCtx, takeoverCancel := context.WithTimeout(ctx, timeout) + defer takeoverCancel() + + go func() { + sweepTicker := time.NewTicker(watcherSweepInterval) + defer sweepTicker.Stop() + for { + select { + case <-takeoverCtx.Done(): + return + case <-sweepTicker.C: + err := watcherGrappler.TakeDownWatcher(takeoverCtx, log) + if err != nil { + log.Errorf("error taking down watcher: %s", err) + continue + } + + } + } + }() + + // we should retry to take over the AppLocker for 30s, but AppLocker interface is limited + takeOverTicker := time.NewTicker(takeOverInterval) + defer takeOverTicker.Stop() + for { + select { + case <-takeoverCtx.Done(): + return nil, fmt.Errorf("timed out taking over watcher applocker") + case <-takeOverTicker.C: + locker := filelock.NewAppLocker(topDir, watcherApplockerFileName) + err := locker.TryLock() + if err != nil { + log.Errorf("error locking watcher applocker: %s", err) + continue + } + return locker, nil + } + } +} + +func selectWatcherExecutable(topDir string, previous agentInstall, current agentInstall) string { + // check if the upgraded version is less than the previous (currently installed) version + if current.parsedVersion.Less(*previous.parsedVersion) { + // use the current agent executable for watch, if downgrading the old agent doesn't understand the current agent's path structure. + return paths.BinaryPath(filepath.Join(topDir, previous.versionedHome), agentName) + } else { + // use the new agent executable as it should be able to parse the new update marker + return paths.BinaryPath(filepath.Join(topDir, current.versionedHome), agentName) + } +} + +func waitForWatcher(ctx context.Context, log *logger.Logger, markerFilePath string, waitTime time.Duration) error { + return waitForWatcherWithTimeoutCreationFunc(ctx, log, markerFilePath, waitTime, context.WithTimeout) +} + +type createContextWithTimeout func(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) + +func waitForWatcherWithTimeoutCreationFunc(ctx context.Context, log *logger.Logger, markerFilePath string, waitTime time.Duration, createTimeoutContext createContextWithTimeout) error { + // Wait for the watcher to be up and running + watcherContext, cancel := createTimeoutContext(ctx, waitTime) + defer cancel() + + markerWatcher := newMarkerFileWatcher(markerFilePath, log) + err := markerWatcher.Run(watcherContext) + if err != nil { + return fmt.Errorf("error starting update marker watcher: %w", err) + } + + log.Infof("waiting up to %s for upgrade watcher to set %s state in upgrade marker", waitTime, details.StateWatching) + + for { + select { + case updMarker := <-markerWatcher.Watch(): + if updMarker.Details != nil && updMarker.Details.State == details.StateWatching { + // watcher started and it is watching, all good + log.Infof("upgrade watcher set %s state in upgrade marker: exiting wait loop", details.StateWatching) + return nil + } + + case <-watcherContext.Done(): + log.Errorf("upgrade watcher did not start watching within %s or context has expired", waitTime) + return errors.Join(ErrWatcherNotStarted, watcherContext.Err()) + } + } +} diff --git a/internal/pkg/agent/application/upgrade/watcher_notwindows.go b/internal/pkg/agent/application/upgrade/watcher_notwindows.go new file mode 100644 index 00000000000..8c5e8726108 --- /dev/null +++ b/internal/pkg/agent/application/upgrade/watcher_notwindows.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build !windows + +package upgrade + +import ( + "context" + "os" + "os/exec" + + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" +) + +func createTakeDownWatcherCommand(ctx context.Context) *exec.Cmd { + executable, _ := os.Executable() + + // #nosec G204 -- user cannot inject any parameters to this command + cmd := exec.CommandContext(ctx, executable, watcherSubcommand, + "--path.config", paths.Config(), + "--path.home", paths.Top(), + "--takedown", + ) + return cmd +} diff --git a/internal/pkg/agent/application/upgrade/watcher_test.go b/internal/pkg/agent/application/upgrade/watcher_test.go index b639df0b2f4..12a7c744c61 100644 --- a/internal/pkg/agent/application/upgrade/watcher_test.go +++ b/internal/pkg/agent/application/upgrade/watcher_test.go @@ -8,16 +8,26 @@ import ( "context" "fmt" "net" + "os" + "path/filepath" + "sync" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "gopkg.in/yaml.v3" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" "github.com/elastic/elastic-agent/pkg/control/v2/client" "github.com/elastic/elastic-agent/pkg/control/v2/cproto" "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" + agtversion "github.com/elastic/elastic-agent/pkg/version" ) func TestWatcher_CannotConnect(t *testing.T) { @@ -623,3 +633,360 @@ func (s *mockDaemon) Client() client.Client { func (s *mockDaemon) StateWatch(_ *cproto.Empty, srv cproto.ElasticAgentControl_StateWatchServer) error { return s.watch(srv) } + +func Test_selectWatcherExecutable(t *testing.T) { + type args struct { + previous agentInstall + current agentInstall + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Simple upgrade, we should launch the new (current) watcher", + args: args{ + previous: agentInstall{ + parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "", ""), + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-somehash"), + }, + current: agentInstall{ + parsedVersion: agtversion.NewParsedSemVer(4, 5, 6, "", ""), + versionedHome: filepath.Join("data", "elastic-agent-4.5.6-someotherhash"), + }, + }, + want: filepath.Join("data", "elastic-agent-4.5.6-someotherhash"), + }, + { + name: "Simple downgrade, we should launch the currently installed (previous) watcher", + args: args{ + previous: agentInstall{ + parsedVersion: agtversion.NewParsedSemVer(4, 5, 6, "", ""), + versionedHome: filepath.Join("data", "elastic-agent-4.5.6-someotherhash"), + }, + current: agentInstall{ + parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "", ""), + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-somehash"), + }, + }, + want: filepath.Join("data", "elastic-agent-4.5.6-someotherhash"), + }, + { + name: "Upgrade from snapshot to released version, we should launch the new (current) watcher", + args: args{ + previous: agentInstall{ + parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", ""), + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-somehash"), + }, + current: agentInstall{ + parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "", ""), + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-someotherhash"), + }, + }, + want: filepath.Join("data", "elastic-agent-1.2.3-someotherhash"), + }, + { + name: "Downgrade from released version to SNAPSHOT, we should launch the currently installed (previous) watcher", + args: args{ + previous: agentInstall{ + parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "", ""), + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-somehash"), + }, + current: agentInstall{ + parsedVersion: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", ""), + versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-someotherhash"), + }, + }, + + want: filepath.Join("data", "elastic-agent-1.2.3-somehash"), + }, + } + // Just need a top dir path. This test does not make any operation on the filesystem, so a temp dir path is as good as any + fakeTopDir := filepath.Join(t.TempDir(), "Elastic", "Agent") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, paths.BinaryPath(filepath.Join(fakeTopDir, tt.want), agentName), selectWatcherExecutable(fakeTopDir, tt.args.previous, tt.args.current), "selectWatcherExecutable(%v, %v)", tt.args.previous, tt.args.current) + }) + } +} + +func TestWaitForWatcher(t *testing.T) { + wantErrWatcherNotStarted := func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorIs(t, err, ErrWatcherNotStarted, i) + } + + tests := []struct { + name string + states []details.State + stateChangeInterval time.Duration + cancelWaitContext bool + wantErr assert.ErrorAssertionFunc + }{ + { + name: "Happy path: watcher is watching already", + states: []details.State{details.StateWatching}, + stateChangeInterval: 1 * time.Millisecond, + wantErr: assert.NoError, + }, + { + name: "Sad path: watcher is never starting", + states: []details.State{details.StateReplacing}, + stateChangeInterval: 1 * time.Millisecond, + cancelWaitContext: true, + wantErr: wantErrWatcherNotStarted, + }, + { + name: "Runaround path: marker is jumping around and landing on watching", + states: []details.State{ + details.StateRequested, + details.StateScheduled, + details.StateDownloading, + details.StateExtracting, + details.StateReplacing, + details.StateRestarting, + details.StateWatching, + }, + stateChangeInterval: 1 * time.Millisecond, + wantErr: assert.NoError, + }, + { + name: "Timeout: marker is never created", + states: nil, + stateChangeInterval: 1 * time.Millisecond, + cancelWaitContext: true, + wantErr: wantErrWatcherNotStarted, + }, + { + name: "Timeout2: state doesn't get there in time", + states: []details.State{ + details.StateRequested, + details.StateScheduled, + details.StateDownloading, + details.StateExtracting, + details.StateReplacing, + details.StateRestarting, + }, + + stateChangeInterval: 1 * time.Millisecond, + cancelWaitContext: true, + wantErr: wantErrWatcherNotStarted, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + deadline, ok := t.Deadline() + if !ok { + deadline = time.Now().Add(5 * time.Second) + } + testCtx, testCancel := context.WithDeadline(context.Background(), deadline) + defer testCancel() + + tmpDir := t.TempDir() + updMarkerFilePath := filepath.Join(tmpDir, markerFilename) + + waitContext, waitCancel := context.WithCancel(testCtx) + defer waitCancel() + + fakeTimeout := 30 * time.Second + + // in order to take timing out of the equation provide a context that we can cancel manually + // still assert that the parent context and timeout passed are correct + var createContextFunc createContextWithTimeout = func(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + assert.Same(t, testCtx, ctx, "parent context should be the same as the waitForWatcherCall") + assert.Equal(t, fakeTimeout, timeout, "timeout used in new context should be the same as testcase") + + return waitContext, waitCancel + } + + if len(tt.states) > 0 { + initialState := tt.states[0] + writeState(t, updMarkerFilePath, initialState) + } + + wg := new(sync.WaitGroup) + + var furtherStates []details.State + if len(tt.states) > 1 { + // we have more states to produce + furtherStates = tt.states[1:] + } + + wg.Add(1) + + // worker goroutine: writes out additional states while the test is blocked on waitOnWatcher() call and expires + // the wait context if cancelWaitContext is set to true. Timing of the goroutine is driven by stateChangeInterval. + go func() { + defer wg.Done() + tick := time.NewTicker(tt.stateChangeInterval) + defer tick.Stop() + for _, state := range furtherStates { + select { + case <-testCtx.Done(): + return + case <-tick.C: + writeState(t, updMarkerFilePath, state) + } + } + if tt.cancelWaitContext { + <-tick.C + waitCancel() + } + }() + + log, _ := loggertest.New(tt.name) + + tt.wantErr(t, waitForWatcherWithTimeoutCreationFunc(testCtx, log, updMarkerFilePath, fakeTimeout, createContextFunc), fmt.Sprintf("waitForWatcher %s, %v, %s, %s)", updMarkerFilePath, tt.states, tt.stateChangeInterval, fakeTimeout)) + + // wait for goroutines to finish + wg.Wait() + }) + } +} + +func writeState(t *testing.T, path string, state details.State) { + ms := newMarkerSerializer(&UpdateMarker{ + Version: "version", + Hash: "hash", + VersionedHome: "versionedHome", + UpdatedOn: time.Now(), + PrevVersion: "prev_version", + PrevHash: "prev_hash", + PrevVersionedHome: "prev_versionedhome", + Acked: false, + Action: nil, + Details: &details.Details{ + TargetVersion: "version", + State: state, + ActionID: "", + Metadata: details.Metadata{}, + }, + }) + + bytes, err := yaml.Marshal(ms) + if assert.NoError(t, err, "error marshaling the test upgrade marker") { + err = os.WriteFile(path, bytes, 0770) + assert.NoError(t, err, "error writing out the test upgrade marker") + } +} + +// TestTakeOverWatcher verifies that takeOverWatcher behaves within expectations. +// This test cannot run in parallel because it deals with launching test processes and verifying their state. +// In case of aggressive PID reuse along with parallel execution, this test could kill "innocent" processes +func TestTakeOverWatcher(t *testing.T) { + + type setupFunc func(t *testing.T, workdir string, mockWatcherGrappler *mockWatcherGrappler) + type assertFunc func(t *testing.T, workdir string, appLocker *filelock.AppLocker) + + testcases := []struct { + name string + setup setupFunc + wantErr assert.ErrorAssertionFunc + assertPostTakeover assertFunc + }{ + { + name: "no contention for watcher applocker", + setup: func(t *testing.T, workdir string, mockWatcherGrappler *mockWatcherGrappler) { + // nothing to do here + }, + wantErr: assert.NoError, + assertPostTakeover: func(t *testing.T, workdir string, appLocker *filelock.AppLocker) { + assert.NotNil(t, appLocker, "appLocker should not be nil") + assert.FileExists(t, filepath.Join(workdir, watcherApplockerFileName)) + }, + }, + { + name: "contention with a process that can be taken down: no error", + setup: func(t *testing.T, workdir string, mockWatcherGrappler *mockWatcherGrappler) { + // create and lock an applocker + locker := filelock.NewAppLocker(workdir, watcherApplockerFileName) + err := locker.TryLock() + require.NoError(t, err, "error setting up the applocker") + mockWatcherGrappler.EXPECT().TakeDownWatcher(mock.Anything, mock.Anything).Run(func(_ context.Context, _ *logp.Logger) { + unlockErr := locker.Unlock() + assert.NoError(t, unlockErr, "error unlocking the applocker") + }).Return(nil) + + // add a cleanup to unlock the applocker at the end of the test anyway in case of failures + t.Cleanup(func() { + _ = locker.Unlock() + }) + }, + wantErr: assert.NoError, + assertPostTakeover: func(t *testing.T, workdir string, appLocker *filelock.AppLocker) { + assert.NotNil(t, appLocker, "appLocker should not be nil") + assert.FileExists(t, filepath.Join(workdir, watcherApplockerFileName)) + }, + }, + { + name: "contention with a process that can be taken down with multiple attempts: no error", + setup: func(t *testing.T, workdir string, mockWatcherGrappler *mockWatcherGrappler) { + // create and lock an applocker + locker := filelock.NewAppLocker(workdir, watcherApplockerFileName) + err := locker.TryLock() + require.NoError(t, err, "error setting up the applocker") + mockWatcherGrappler.EXPECT().TakeDownWatcher(mock.Anything, mock.Anything).Return(fmt.Errorf("some takedown error")).Once() + mockWatcherGrappler.EXPECT().TakeDownWatcher(mock.Anything, mock.Anything).Run(func(_ context.Context, _ *logp.Logger) { + unlockErr := locker.Unlock() + assert.NoError(t, unlockErr, "error unlocking the applocker") + }).Return(nil) + + // add a cleanup to unlock the applocker at the end of the test anyway in case of failures + t.Cleanup(func() { + _ = locker.Unlock() + }) + }, + wantErr: assert.NoError, + assertPostTakeover: func(t *testing.T, workdir string, appLocker *filelock.AppLocker) { + assert.NotNil(t, appLocker, "appLocker should not be nil") + assert.FileExists(t, filepath.Join(workdir, watcherApplockerFileName)) + }, + }, + { + name: "contention with a process that cannot be taken down: error is returned by takeOverWatcher", + setup: func(t *testing.T, workdir string, mockWatcherGrappler *mockWatcherGrappler) { + // create and lock an applocker + locker := filelock.NewAppLocker(workdir, watcherApplockerFileName) + err := locker.TryLock() + require.NoError(t, err, "error setting up the applocker") + + // Expect the calls to applocker but do not release the lock + mockWatcherGrappler.EXPECT().TakeDownWatcher(mock.Anything, mock.Anything).Return(nil) + + // add a cleanup to unlock the applocker at the end of the test anyway + t.Cleanup(func() { + _ = locker.Unlock() + }) + }, + wantErr: assert.Error, + assertPostTakeover: func(t *testing.T, workdir string, appLocker *filelock.AppLocker) { + assert.Nil(t, appLocker, "appLocker should be nil") + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + workDir := t.TempDir() + logger, logs := loggertest.New(t.Name()) + + mockGrappler := newMockWatcherGrappler(t) + tc.setup(t, workDir, mockGrappler) + + appLocker, err := takeOverWatcher(t.Context(), logger, mockGrappler, workDir, 10*time.Second, 500*time.Millisecond, 100*time.Millisecond) + loggertest.PrintObservedLogs(logs.TakeAll(), t.Log) + + tc.wantErr(t, err) + if appLocker != nil { + defer func(appLocker *filelock.AppLocker) { + unlockErr := appLocker.Unlock() + assert.NoError(t, unlockErr, "error unlocking the app locker") + }(appLocker) + } + if tc.assertPostTakeover != nil { + tc.assertPostTakeover(t, workDir, appLocker) + } + }) + } + +} diff --git a/internal/pkg/agent/application/upgrade/watcher_windows.go b/internal/pkg/agent/application/upgrade/watcher_windows.go new file mode 100644 index 00000000000..9f973826681 --- /dev/null +++ b/internal/pkg/agent/application/upgrade/watcher_windows.go @@ -0,0 +1,42 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build windows + +package upgrade + +import ( + "context" + "os" + "os/exec" + "syscall" + + "golang.org/x/sys/windows" + + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" +) + +var ( + kernel32API = windows.NewLazySystemDLL("kernel32.dll") + + freeConsoleProc = kernel32API.NewProc("FreeConsole") + procGetConsoleProcessList = kernel32API.NewProc("GetConsoleProcessList") + allocConsoleProc = kernel32API.NewProc("AllocConsole") +) + +func createTakeDownWatcherCommand(ctx context.Context) *exec.Cmd { + executable, _ := os.Executable() + + // #nosec G204 -- user cannot inject any parameters to this command + cmd := exec.CommandContext(ctx, executable, watcherSubcommand, + "--path.config", paths.Config(), + "--path.home", paths.Top(), + "--takedown", + ) + cmd.SysProcAttr = &syscall.SysProcAttr{ + // https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + CreationFlags: windows.DETACHED_PROCESS, + } + return cmd +} diff --git a/testing/mocks/internal_/pkg/agent/cmd/agent_watcher_mock.go b/internal/pkg/agent/cmd/mock_agentwatcher_test.go similarity index 50% rename from testing/mocks/internal_/pkg/agent/cmd/agent_watcher_mock.go rename to internal/pkg/agent/cmd/mock_agentwatcher_test.go index 4541e5d9aa0..e3685a86fb5 100644 --- a/testing/mocks/internal_/pkg/agent/cmd/agent_watcher_mock.go +++ b/internal/pkg/agent/cmd/mock_agentwatcher_test.go @@ -16,21 +16,21 @@ import ( time "time" ) -// AgentWatcher is an autogenerated mock type for the agentWatcher type -type AgentWatcher struct { +// mockAgentWatcher is an autogenerated mock type for the agentWatcher type +type mockAgentWatcher struct { mock.Mock } -type AgentWatcher_Expecter struct { +type mockAgentWatcher_Expecter struct { mock *mock.Mock } -func (_m *AgentWatcher) EXPECT() *AgentWatcher_Expecter { - return &AgentWatcher_Expecter{mock: &_m.Mock} +func (_m *mockAgentWatcher) EXPECT() *mockAgentWatcher_Expecter { + return &mockAgentWatcher_Expecter{mock: &_m.Mock} } // Watch provides a mock function with given fields: ctx, tilGrace, errorCheckInterval, log -func (_m *AgentWatcher) Watch(ctx context.Context, tilGrace time.Duration, errorCheckInterval time.Duration, log *logp.Logger) error { +func (_m *mockAgentWatcher) Watch(ctx context.Context, tilGrace time.Duration, errorCheckInterval time.Duration, log *logp.Logger) error { ret := _m.Called(ctx, tilGrace, errorCheckInterval, log) if len(ret) == 0 { @@ -47,8 +47,8 @@ func (_m *AgentWatcher) Watch(ctx context.Context, tilGrace time.Duration, error return r0 } -// AgentWatcher_Watch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Watch' -type AgentWatcher_Watch_Call struct { +// mockAgentWatcher_Watch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Watch' +type mockAgentWatcher_Watch_Call struct { *mock.Call } @@ -57,34 +57,34 @@ type AgentWatcher_Watch_Call struct { // - tilGrace time.Duration // - errorCheckInterval time.Duration // - log *logp.Logger -func (_e *AgentWatcher_Expecter) Watch(ctx interface{}, tilGrace interface{}, errorCheckInterval interface{}, log interface{}) *AgentWatcher_Watch_Call { - return &AgentWatcher_Watch_Call{Call: _e.mock.On("Watch", ctx, tilGrace, errorCheckInterval, log)} +func (_e *mockAgentWatcher_Expecter) Watch(ctx interface{}, tilGrace interface{}, errorCheckInterval interface{}, log interface{}) *mockAgentWatcher_Watch_Call { + return &mockAgentWatcher_Watch_Call{Call: _e.mock.On("Watch", ctx, tilGrace, errorCheckInterval, log)} } -func (_c *AgentWatcher_Watch_Call) Run(run func(ctx context.Context, tilGrace time.Duration, errorCheckInterval time.Duration, log *logp.Logger)) *AgentWatcher_Watch_Call { +func (_c *mockAgentWatcher_Watch_Call) Run(run func(ctx context.Context, tilGrace time.Duration, errorCheckInterval time.Duration, log *logp.Logger)) *mockAgentWatcher_Watch_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(time.Duration), args[2].(time.Duration), args[3].(*logp.Logger)) }) return _c } -func (_c *AgentWatcher_Watch_Call) Return(_a0 error) *AgentWatcher_Watch_Call { +func (_c *mockAgentWatcher_Watch_Call) Return(_a0 error) *mockAgentWatcher_Watch_Call { _c.Call.Return(_a0) return _c } -func (_c *AgentWatcher_Watch_Call) RunAndReturn(run func(context.Context, time.Duration, time.Duration, *logp.Logger) error) *AgentWatcher_Watch_Call { +func (_c *mockAgentWatcher_Watch_Call) RunAndReturn(run func(context.Context, time.Duration, time.Duration, *logp.Logger) error) *mockAgentWatcher_Watch_Call { _c.Call.Return(run) return _c } -// NewAgentWatcher creates a new instance of AgentWatcher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// newMockAgentWatcher creates a new instance of mockAgentWatcher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func NewAgentWatcher(t interface { +func newMockAgentWatcher(t interface { mock.TestingT Cleanup(func()) -}) *AgentWatcher { - mock := &AgentWatcher{} +}) *mockAgentWatcher { + mock := &mockAgentWatcher{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/internal/pkg/agent/cmd/mock_installationmodifier_test.go b/internal/pkg/agent/cmd/mock_installationmodifier_test.go new file mode 100644 index 00000000000..7581a521000 --- /dev/null +++ b/internal/pkg/agent/cmd/mock_installationmodifier_test.go @@ -0,0 +1,147 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package cmd + +import ( + context "context" + + client "github.com/elastic/elastic-agent/pkg/control/v2/client" + + logp "github.com/elastic/elastic-agent-libs/logp" + + mock "github.com/stretchr/testify/mock" +) + +// mockInstallationModifier is an autogenerated mock type for the installationModifier type +type mockInstallationModifier struct { + mock.Mock +} + +type mockInstallationModifier_Expecter struct { + mock *mock.Mock +} + +func (_m *mockInstallationModifier) EXPECT() *mockInstallationModifier_Expecter { + return &mockInstallationModifier_Expecter{mock: &_m.Mock} +} + +// Cleanup provides a mock function with given fields: log, topDirPath, currentVersionedHome, currentHash, removeMarker, keepLogs +func (_m *mockInstallationModifier) Cleanup(log *logp.Logger, topDirPath string, currentVersionedHome string, currentHash string, removeMarker bool, keepLogs bool) error { + ret := _m.Called(log, topDirPath, currentVersionedHome, currentHash, removeMarker, keepLogs) + + if len(ret) == 0 { + panic("no return value specified for Cleanup") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*logp.Logger, string, string, string, bool, bool) error); ok { + r0 = rf(log, topDirPath, currentVersionedHome, currentHash, removeMarker, keepLogs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// mockInstallationModifier_Cleanup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Cleanup' +type mockInstallationModifier_Cleanup_Call struct { + *mock.Call +} + +// Cleanup is a helper method to define mock.On call +// - log *logp.Logger +// - topDirPath string +// - currentVersionedHome string +// - currentHash string +// - removeMarker bool +// - keepLogs bool +func (_e *mockInstallationModifier_Expecter) Cleanup(log interface{}, topDirPath interface{}, currentVersionedHome interface{}, currentHash interface{}, removeMarker interface{}, keepLogs interface{}) *mockInstallationModifier_Cleanup_Call { + return &mockInstallationModifier_Cleanup_Call{Call: _e.mock.On("Cleanup", log, topDirPath, currentVersionedHome, currentHash, removeMarker, keepLogs)} +} + +func (_c *mockInstallationModifier_Cleanup_Call) Run(run func(log *logp.Logger, topDirPath string, currentVersionedHome string, currentHash string, removeMarker bool, keepLogs bool)) *mockInstallationModifier_Cleanup_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*logp.Logger), args[1].(string), args[2].(string), args[3].(string), args[4].(bool), args[5].(bool)) + }) + return _c +} + +func (_c *mockInstallationModifier_Cleanup_Call) Return(_a0 error) *mockInstallationModifier_Cleanup_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *mockInstallationModifier_Cleanup_Call) RunAndReturn(run func(*logp.Logger, string, string, string, bool, bool) error) *mockInstallationModifier_Cleanup_Call { + _c.Call.Return(run) + return _c +} + +// Rollback provides a mock function with given fields: ctx, log, c, topDirPath, prevVersionedHome, prevHash, preRestart +func (_m *mockInstallationModifier) Rollback(ctx context.Context, log *logp.Logger, c client.Client, topDirPath string, prevVersionedHome string, prevHash string, preRestart rollbackHook) error { + ret := _m.Called(ctx, log, c, topDirPath, prevVersionedHome, prevHash, preRestart) + + if len(ret) == 0 { + panic("no return value specified for Rollback") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *logp.Logger, client.Client, string, string, string, rollbackHook) error); ok { + r0 = rf(ctx, log, c, topDirPath, prevVersionedHome, prevHash, preRestart) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// mockInstallationModifier_Rollback_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Rollback' +type mockInstallationModifier_Rollback_Call struct { + *mock.Call +} + +// Rollback is a helper method to define mock.On call +// - ctx context.Context +// - log *logp.Logger +// - c client.Client +// - topDirPath string +// - prevVersionedHome string +// - prevHash string +// - preRestart rollbackHook +func (_e *mockInstallationModifier_Expecter) Rollback(ctx interface{}, log interface{}, c interface{}, topDirPath interface{}, prevVersionedHome interface{}, prevHash interface{}, preRestart interface{}) *mockInstallationModifier_Rollback_Call { + return &mockInstallationModifier_Rollback_Call{Call: _e.mock.On("Rollback", ctx, log, c, topDirPath, prevVersionedHome, prevHash, preRestart)} +} + +func (_c *mockInstallationModifier_Rollback_Call) Run(run func(ctx context.Context, log *logp.Logger, c client.Client, topDirPath string, prevVersionedHome string, prevHash string, preRestart rollbackHook)) *mockInstallationModifier_Rollback_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*logp.Logger), args[2].(client.Client), args[3].(string), args[4].(string), args[5].(string), args[6].(rollbackHook)) + }) + return _c +} + +func (_c *mockInstallationModifier_Rollback_Call) Return(_a0 error) *mockInstallationModifier_Rollback_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *mockInstallationModifier_Rollback_Call) RunAndReturn(run func(context.Context, *logp.Logger, client.Client, string, string, string, rollbackHook) error) *mockInstallationModifier_Rollback_Call { + _c.Call.Return(run) + return _c +} + +// newMockInstallationModifier creates a new instance of mockInstallationModifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockInstallationModifier(t interface { + mock.TestingT + Cleanup(func()) +}) *mockInstallationModifier { + mock := &mockInstallationModifier{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go index 281d5a8870b..e32e1ae3423 100644 --- a/internal/pkg/agent/cmd/run.go +++ b/internal/pkg/agent/cmd/run.go @@ -718,7 +718,7 @@ func ensureInstallMarkerPresent() error { if err != nil { return fmt.Errorf("failed to get current file owner: %w", err) } - if err := install.CreateInstallMarker(paths.Top(), ownership); err != nil { + if err := install.CreateInstallMarker(paths.Top(), ownership, paths.Home(), version.GetAgentPackageVersion()); err != nil { return fmt.Errorf("unable to create installation marker file during upgrade: %w", err) } diff --git a/internal/pkg/agent/cmd/upgrade.go b/internal/pkg/agent/cmd/upgrade.go index c944133e288..400293f1616 100644 --- a/internal/pkg/agent/cmd/upgrade.go +++ b/internal/pkg/agent/cmd/upgrade.go @@ -33,6 +33,7 @@ const ( flagPGPBytesPath = "pgp-path" flagPGPBytesURI = "pgp-uri" flagForce = "force" + flagRollback = "rollback" ) var ( @@ -64,6 +65,7 @@ func newUpgradeCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Comman cmd.Flags().String(flagPGPBytesURI, "", "Path to a web location containing PGP to use for package verification") cmd.Flags().String(flagPGPBytesPath, "", "Path to a file containing PGP to use for package verification") cmd.Flags().BoolP(flagForce, "", false, "Advanced option to force an upgrade on a fleet managed agent") + cmd.Flags().BoolP(flagRollback, "", false, "Roll back an upgrade") err := cmd.Flags().MarkHidden(flagForce) if err != nil { fmt.Fprintf(streams.Err, "error while setting upgrade force flag attributes: %s", err.Error()) @@ -162,6 +164,11 @@ func upgradeCmdWithClient(input *upgradeInput) error { return fmt.Errorf("failed to retrieve command flag information while trying to upgrade the agent: %w", err) } + rollback, err := cmd.Flags().GetBool(flagRollback) + if err != nil { + return fmt.Errorf("failed to retrieve command flag information %q while trying to upgrade the agent: %w", flagRollback, err) + } + skipVerification, err := cmd.Flags().GetBool(flagSkipVerify) if err != nil { return fmt.Errorf("failed to retrieve %s flag information while upgrading the agent: %w", flagSkipVerify, err) @@ -181,7 +188,7 @@ func upgradeCmdWithClient(input *upgradeInput) error { if err != nil { return fmt.Errorf("failed to check if upgrade is already in progress: %w", err) } - if isBeingUpgraded { + if isBeingUpgraded && !rollback { return errors.New("an upgrade is already in progress; please try again later.") } @@ -215,7 +222,7 @@ func upgradeCmdWithClient(input *upgradeInput) error { } } skipDefaultPgp, _ := cmd.Flags().GetBool(flagSkipDefaultPgp) - version, err = c.Upgrade(context.Background(), version, sourceURI, skipVerification, skipDefaultPgp, pgpChecks...) + version, err = c.Upgrade(context.Background(), version, rollback, sourceURI, skipVerification, skipDefaultPgp, pgpChecks...) if err != nil { s, ok := status.FromError(err) // Sometimes the gRPC server shuts down before replying to the command which is expected diff --git a/internal/pkg/agent/cmd/upgrade_test.go b/internal/pkg/agent/cmd/upgrade_test.go index f9b98e5ec89..54f0510c25a 100644 --- a/internal/pkg/agent/cmd/upgrade_test.go +++ b/internal/pkg/agent/cmd/upgrade_test.go @@ -143,7 +143,7 @@ func TestUpgradeCmd(t *testing.T) { t.Run("proceed with upgrade if fleet managed, privileged, --force is set", func(t *testing.T) { mockClient := clientmocks.NewClient(t) mockClient.EXPECT().State(mock.Anything).Return(&client.AgentState{State: cproto.State_HEALTHY}, nil) - mockClient.EXPECT().Upgrade(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("mockVersion", nil) + mockClient.EXPECT().Upgrade(mock.Anything, mock.Anything, false, mock.Anything, mock.Anything, mock.Anything).Return("mockVersion", nil) args := []string{"8.13.0"} // Version argument streams := cli.NewIOStreams() @@ -231,7 +231,7 @@ func TestUpgradeCmd(t *testing.T) { t.Run("proceed with upgrade if agent is standalone, user is privileged and skip-verify flag is set", func(t *testing.T) { mockClient := clientmocks.NewClient(t) mockClient.EXPECT().State(mock.Anything).Return(&client.AgentState{State: cproto.State_HEALTHY}, nil) - mockClient.EXPECT().Upgrade(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("mockVersion", nil) + mockClient.EXPECT().Upgrade(mock.Anything, mock.Anything, false, mock.Anything, mock.Anything, mock.Anything).Return("mockVersion", nil) args := []string{"8.13.0"} // Version argument streams := cli.NewIOStreams() diff --git a/internal/pkg/agent/cmd/watch.go b/internal/pkg/agent/cmd/watch.go index f203c1814f7..5ed49672262 100644 --- a/internal/pkg/agent/cmd/watch.go +++ b/internal/pkg/agent/cmd/watch.go @@ -8,9 +8,7 @@ import ( "context" "fmt" "os" - "os/signal" "runtime" - "syscall" "time" "github.com/spf13/cobra" @@ -18,6 +16,7 @@ import ( "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/configure" "github.com/elastic/elastic-agent/pkg/control/v2/client" + "github.com/elastic/elastic-agent/pkg/utils" "github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" @@ -35,14 +34,22 @@ import ( const ( watcherName = "elastic-agent-watcher" watcherLockFile = "watcher.lock" + + errorSettingParentSignalsExitCode = 6 ) +// watcherPIDsFetcher defines the type of function responsible for fetching watcher PIDs. +// This will allow for easier testing of takeOverWatcher using fake binaries +type watcherPIDsFetcher func() ([]int, error) + +var ErrWatchCancelled = errors.New("watch cancelled") + func newWatchCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command { cmd := &cobra.Command{ Use: "watch", Short: "Watch the Elastic Agent for failures and initiate rollback", Long: `This command watches Elastic Agent for failures and initiates rollback if necessary.`, - Run: func(_ *cobra.Command, _ []string) { + Run: func(c *cobra.Command, _ []string) { cfg := getConfig(streams) log, err := configuredLogger(cfg, watcherName) if err != nil { @@ -53,6 +60,22 @@ func newWatchCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command // Make sure to flush any buffered logs before we're done. defer log.Sync() //nolint:errcheck // flushing buffered logs is best effort. + err = setupParentProcessSignals() + if err != nil { + fmt.Fprintf(streams.Err, "Error setting parent process signals: %v\n", err) + os.Exit(errorSettingParentSignalsExitCode) + } + + takedown, _ := c.Flags().GetBool("takedown") + if takedown { + err = takedownWatcher(context.Background(), log, utils.GetWatcherPIDs) + if err != nil { + log.Errorf("error taking down watcher: %v", err) + os.Exit(5) + } + return + } + if err := watchCmd(log, paths.Top(), cfg.Settings.Upgrade.Watcher, new(upgradeAgentWatcher), new(upgradeInstallationModifier)); err != nil { log.Errorw("Watch command failed", "error.message", err) fmt.Fprintf(streams.Err, "Watch command failed: %v\n%s\n", err, troubleshootMessage()) @@ -60,7 +83,8 @@ func newWatchCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command } }, } - + cmd.Flags().BoolP("takedown", "t", false, "Take down the running watcher") + cmd.Flags().MarkHidden("takedown") //nolint:errcheck // not required return cmd } @@ -68,9 +92,11 @@ type agentWatcher interface { Watch(ctx context.Context, tilGrace, errorCheckInterval time.Duration, log *logp.Logger) error } +type rollbackHook func(ctx context.Context, log *logger.Logger, topDirPath string) error + type installationModifier interface { Cleanup(log *logger.Logger, topDirPath, currentVersionedHome, currentHash string, removeMarker, keepLogs bool) error - Rollback(ctx context.Context, log *logger.Logger, c client.Client, topDirPath, prevVersionedHome, prevHash string) error + Rollback(ctx context.Context, log *logger.Logger, c client.Client, topDirPath, prevVersionedHome, prevHash string, preRestart rollbackHook) error } func watchCmd(log *logp.Logger, topDir string, cfg *configuration.UpgradeWatcherConfig, watcher agentWatcher, installModifier installationModifier) error { @@ -102,6 +128,34 @@ func watchCmd(log *logp.Logger, topDir string, cfg *configuration.UpgradeWatcher _ = locker.Unlock() }() + if marker.DesiredOutcome == upgrade.OUTCOME_ROLLBACK && marker.Details != nil && marker.Details.State != details.StateRollback { + // TODO: there should be some sanity check in rollback functions like the installation we are going back to should exist and work + log.Infof("rolling back because of DesiredOutcome=%s", marker.DesiredOutcome.String()) + + updateMarkerAndDetails := func(_ context.Context, _ *logger.Logger, _ string) error { + if marker.Details == nil { + actionID := "" + if marker.Action != nil { + actionID = marker.Action.ActionID + } + marker.Details = details.NewDetails(marker.Version, details.StateRollback, actionID) + } + marker.Details.SetStateWithReason(details.StateRollback, details.ReasonManualRollback) + err = upgrade.SaveMarker(dataDir, marker, true) + if err != nil { + return fmt.Errorf("saving marker after rolling back: %w", err) + } + return nil + } + + err = installModifier.Rollback(context.Background(), log, client.New(), paths.Top(), marker.PrevVersionedHome, marker.PrevHash, updateMarkerAndDetails) + if err != nil { + return fmt.Errorf("rolling back: %w", err) + } + + return nil + } + isWithinGrace, tilGrace := gracePeriod(marker, cfg.GracePeriod) if isTerminalState(marker) || !isWithinGrace { stateString := "" @@ -130,10 +184,15 @@ func watchCmd(log *logp.Logger, topDir string, cfg *configuration.UpgradeWatcher errorCheckInterval := cfg.ErrorCheck.Interval ctx := context.Background() if err := watcher.Watch(ctx, tilGrace, errorCheckInterval, log); err != nil { + if errors.Is(err, ErrWatchCancelled) { + // the watch has been cancelled prematurely, don't clean or rollback just yet + return nil + } + log.Error("Error detected, proceeding to rollback: %v", err) upgradeDetails.SetStateWithReason(details.StateRollback, details.ReasonWatchFailed) - err = installModifier.Rollback(ctx, log, client.New(), paths.Top(), marker.PrevVersionedHome, marker.PrevHash) + err = installModifier.Rollback(ctx, log, client.New(), paths.Top(), marker.PrevVersionedHome, marker.PrevHash, nil) if err != nil { log.Error("rollback failed", err) upgradeDetails.Fail(err) @@ -180,48 +239,6 @@ func isWindows() bool { return runtime.GOOS == "windows" } -func watch(ctx context.Context, tilGrace time.Duration, errorCheckInterval time.Duration, log *logger.Logger) error { - errChan := make(chan error) - - ctx, cancel := context.WithCancel(ctx) - - //cleanup - defer func() { - cancel() - close(errChan) - }() - - agentWatcher := upgrade.NewAgentWatcher(errChan, log, errorCheckInterval) - go agentWatcher.Run(ctx) - - signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP) - - t := time.NewTimer(tilGrace) - defer t.Stop() - -WATCHLOOP: - for { - select { - case <-signals: - // ignore - continue - case <-ctx.Done(): - break WATCHLOOP - // grace period passed, agent is considered stable - case <-t.C: - log.Info("Grace period passed, not watching") - break WATCHLOOP - // Agent in degraded state. - case err := <-errChan: - log.Errorf("Agent Error detected: %s", err.Error()) - return err - } - } - - return nil -} - // gracePeriod returns true if it is within grace period and time until grace period ends. // otherwise it returns false and 0 func gracePeriod(marker *upgrade.UpdateMarker, gracePeriodDuration time.Duration) (bool, time.Duration) { diff --git a/internal/pkg/agent/cmd/watch_impl.go b/internal/pkg/agent/cmd/watch_impl.go index 92e3118435c..8fac2ac315e 100644 --- a/internal/pkg/agent/cmd/watch_impl.go +++ b/internal/pkg/agent/cmd/watch_impl.go @@ -6,6 +6,9 @@ package cmd import ( "context" + "os" + "os/signal" + "syscall" "time" "github.com/elastic/elastic-agent-libs/logp" @@ -26,6 +29,59 @@ func (a upgradeInstallationModifier) Cleanup(log *logger.Logger, topDirPath, cur return upgrade.Cleanup(log, topDirPath, currentVersionedHome, currentHash, removeMarker, keepLogs) } -func (a upgradeInstallationModifier) Rollback(ctx context.Context, log *logger.Logger, c client.Client, topDirPath, prevVersionedHome, prevHash string) error { - return upgrade.Rollback(ctx, log, c, topDirPath, prevVersionedHome, prevHash) +func (a upgradeInstallationModifier) Rollback(ctx context.Context, log *logger.Logger, c client.Client, topDirPath, prevVersionedHome, prevHash string, preRestart rollbackHook) error { + var opts []upgrade.RollbackOpt + if preRestart != nil { + opts = append(opts, upgrade.WithPreRestartHook(upgrade.RollbackHook(preRestart))) + } + return upgrade.RollbackWithOpts(ctx, log, c, topDirPath, prevVersionedHome, prevHash, opts...) +} + +func watch(ctx context.Context, tilGrace time.Duration, errorCheckInterval time.Duration, log *logger.Logger) error { + errChan := make(chan error) + + ctx, cancel := context.WithCancel(ctx) + + //cleanup + defer func() { + cancel() + close(errChan) + }() + + agtWatcher := upgrade.NewAgentWatcher(errChan, log, errorCheckInterval) + go agtWatcher.Run(ctx) + + // Allow for signals to interrupt the watch + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP) + defer signal.Stop(signals) + + graceTimer := time.NewTimer(tilGrace) + defer graceTimer.Stop() + + return watchLoop(ctx, log, signals, errChan, graceTimer.C) +} + +func watchLoop(ctx context.Context, log *logger.Logger, signals <-chan os.Signal, errChan <-chan error, graceTimer <-chan time.Time) error { + for { + select { + case s := <-signals: + log.Infof("received signal: (%d): %v during watch", s, s) + if s == syscall.SIGINT || s == syscall.SIGTERM { + log.Infof("received signal: (%d): %v. Exiting watch", s, s) + return ErrWatchCancelled + } + continue + case <-ctx.Done(): + return nil + // grace period passed, agent is considered stable + case <-graceTimer: + log.Info("Grace period passed, not watching") + return nil + // Agent in degraded state. + case err := <-errChan: + log.Errorf("Agent Error detected: %s", err.Error()) + return err + } + } } diff --git a/internal/pkg/agent/cmd/watch_impl_test.go b/internal/pkg/agent/cmd/watch_impl_test.go new file mode 100644 index 00000000000..d9537b58c92 --- /dev/null +++ b/internal/pkg/agent/cmd/watch_impl_test.go @@ -0,0 +1,73 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package cmd + +import ( + "context" + "fmt" + "os" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" +) + +func Test_watchLoop(t *testing.T) { + + t.Run("watchloop returns when context expires - no error", func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + log, _ := loggertest.New(t.Name()) + signals := make(chan os.Signal, 1) + errChan := make(chan error, 1) + graceTimer := make(chan time.Time, 1) + err := watchLoop(ctx, log, signals, errChan, graceTimer) + require.NoError(t, err) + }) + + t.Run("watchloop returns when grace timer triggers - no error", func(t *testing.T) { + log, _ := loggertest.New(t.Name()) + signals := make(chan os.Signal, 1) + errChan := make(chan error, 1) + graceTimer := make(chan time.Time, 1) + graceTimer <- time.Now() + err := watchLoop(t.Context(), log, signals, errChan, graceTimer) + require.NoError(t, err) + }) + + t.Run("watchloop returns when error from AgentWatcher is received - error", func(t *testing.T) { + log, _ := loggertest.New(t.Name()) + signals := make(chan os.Signal, 1) + errChan := make(chan error, 1) + graceTimer := make(chan time.Time, 1) + agentWatcherError := fmt.Errorf("some error") + errChan <- agentWatcherError + err := watchLoop(t.Context(), log, signals, errChan, graceTimer) + require.ErrorIs(t, err, agentWatcherError) + }) + + t.Run("watchloop returns when receiving signals - error", func(t *testing.T) { + testSignals := []syscall.Signal{ + syscall.SIGTERM, + syscall.SIGINT, + } + + for _, signal := range testSignals { + t.Run(signal.String(), func(t *testing.T) { + log, _ := loggertest.New(t.Name()) + signals := make(chan os.Signal, 1) + errChan := make(chan error, 1) + graceTimer := make(chan time.Time, 1) + signals <- signal + err := watchLoop(t.Context(), log, signals, errChan, graceTimer) + assert.ErrorIs(t, err, ErrWatchCancelled) + }) + } + }) +} diff --git a/internal/pkg/agent/cmd/watch_notwindows.go b/internal/pkg/agent/cmd/watch_notwindows.go new file mode 100644 index 00000000000..ece82dc020c --- /dev/null +++ b/internal/pkg/agent/cmd/watch_notwindows.go @@ -0,0 +1,62 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build !windows + +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "syscall" + + "github.com/elastic/elastic-agent/pkg/core/logger" +) + +func takedownWatcher(ctx context.Context, log *logger.Logger, pidFetchFunc watcherPIDsFetcher) error { + pids, err := pidFetchFunc() + if err != nil { + return fmt.Errorf("error listing watcher processes: %w", err) + } + + ownPID := os.Getpid() + var accumulatedSignalingErrors error + for _, pid := range pids { + + if ctx.Err() != nil { + return ctx.Err() + } + + if pid == ownPID { + continue + } + + log.Debugf("attempting to terminate watcher process with PID: %d", pid) + + process, err := os.FindProcess(pid) + if err != nil { + accumulatedSignalingErrors = errors.Join(accumulatedSignalingErrors, fmt.Errorf("error finding watcher process with PID: %d: %w", pid, err)) + continue + } + + err = process.Signal(syscall.SIGTERM) + if err != nil { + accumulatedSignalingErrors = errors.Join(accumulatedSignalingErrors, fmt.Errorf("error killing watcher process with PID: %d: %w", pid, err)) + continue + } + + } + return accumulatedSignalingErrors +} + +func isProcessLive(process *os.Process) (bool, error) { + signalErr := process.Signal(syscall.Signal(0)) + if signalErr != nil { + return false, nil //nolint:nilerr // if we receive an error it means that the process is not running, so the check completed without errors + } else { + return true, nil + } +} diff --git a/internal/pkg/agent/cmd/watch_signals_linux.go b/internal/pkg/agent/cmd/watch_signals_linux.go new file mode 100644 index 00000000000..97d5a329d61 --- /dev/null +++ b/internal/pkg/agent/cmd/watch_signals_linux.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build linux + +package cmd + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +func setupParentProcessSignals() error { + // Perform prctl(PR_SET_PDEATHSIG, 0) to clear the parent death signal + err := unix.Prctl(unix.PR_SET_PDEATHSIG, 0, 0, 0, 0) + if err != nil { + return fmt.Errorf("clearing parent death signal: %w", err) + } + + return nil +} diff --git a/internal/pkg/agent/cmd/watch_signals_notlinux.go b/internal/pkg/agent/cmd/watch_signals_notlinux.go new file mode 100644 index 00000000000..ab565f95d39 --- /dev/null +++ b/internal/pkg/agent/cmd/watch_signals_notlinux.go @@ -0,0 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build !linux + +package cmd + +func setupParentProcessSignals() error { + // nothing to do here + + return nil +} diff --git a/internal/pkg/agent/cmd/watch_test.go b/internal/pkg/agent/cmd/watch_test.go index 9451c476543..91b2a1899c8 100644 --- a/internal/pkg/agent/cmd/watch_test.go +++ b/internal/pkg/agent/cmd/watch_test.go @@ -7,7 +7,10 @@ package cmd import ( "fmt" "os" + "os/exec" + "path/filepath" "runtime" + "strings" "testing" "time" @@ -17,16 +20,17 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/internal/pkg/fleetapi" "github.com/elastic/elastic-agent/internal/pkg/release" + "github.com/elastic/elastic-agent/pkg/core/logger" "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade" - cmdmocks "github.com/elastic/elastic-agent/testing/mocks/internal_/pkg/agent/cmd" ) func TestInitUpgradeDetails(t *testing.T) { @@ -91,13 +95,13 @@ func Test_watchCmd(t *testing.T) { } tests := []struct { name string - setupUpgradeMarker func(t *testing.T, tmpDir string, watcher *cmdmocks.AgentWatcher, installModifier *cmdmocks.InstallationModifier) + setupUpgradeMarker func(t *testing.T, tmpDir string, watcher *mockAgentWatcher, installModifier *mockInstallationModifier) args args wantErr assert.ErrorAssertionFunc }{ { name: "no upgrade marker, no party", - setupUpgradeMarker: func(t *testing.T, topDir string, watcher *cmdmocks.AgentWatcher, installModifier *cmdmocks.InstallationModifier) { + setupUpgradeMarker: func(t *testing.T, topDir string, watcher *mockAgentWatcher, installModifier *mockInstallationModifier) { dataDirPath := paths.DataFrom(topDir) err := os.MkdirAll(dataDirPath, 0755) require.NoError(t, err) @@ -109,7 +113,7 @@ func Test_watchCmd(t *testing.T) { }, { name: "happy path: no error watching, cleanup prev install", - setupUpgradeMarker: func(t *testing.T, topDir string, watcher *cmdmocks.AgentWatcher, installModifier *cmdmocks.InstallationModifier) { + setupUpgradeMarker: func(t *testing.T, topDir string, watcher *mockAgentWatcher, installModifier *mockInstallationModifier) { dataDirPath := paths.DataFrom(topDir) err := os.MkdirAll(dataDirPath, 0755) require.NoError(t, err) @@ -150,7 +154,7 @@ func Test_watchCmd(t *testing.T) { }, { name: "unhappy path: error watching, rollback to previous install", - setupUpgradeMarker: func(t *testing.T, topDir string, watcher *cmdmocks.AgentWatcher, installModifier *cmdmocks.InstallationModifier) { + setupUpgradeMarker: func(t *testing.T, topDir string, watcher *mockAgentWatcher, installModifier *mockInstallationModifier) { dataDirPath := paths.DataFrom(topDir) err := os.MkdirAll(dataDirPath, 0755) require.NoError(t, err) @@ -177,7 +181,7 @@ func Test_watchCmd(t *testing.T) { Watch(mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(errors.New("some watch error due to agent misbehaving")) installModifier.EXPECT(). - Rollback(mock.Anything, mock.Anything, mock.Anything, paths.Top(), "elastic-agent-prvver", "prvver"). + Rollback(mock.Anything, mock.Anything, mock.Anything, paths.Top(), "elastic-agent-prvver", "prvver", mock.MatchedBy(func(hook rollbackHook) bool { return hook == nil })). Return(nil) }, args: args{ @@ -187,7 +191,7 @@ func Test_watchCmd(t *testing.T) { }, { name: "upgrade rolled back: no watching, cleanup must be called", - setupUpgradeMarker: func(t *testing.T, topDir string, watcher *cmdmocks.AgentWatcher, installModifier *cmdmocks.InstallationModifier) { + setupUpgradeMarker: func(t *testing.T, topDir string, watcher *mockAgentWatcher, installModifier *mockInstallationModifier) { dataDirPath := paths.DataFrom(topDir) err := os.MkdirAll(dataDirPath, 0755) require.NoError(t, err) @@ -228,7 +232,7 @@ func Test_watchCmd(t *testing.T) { }, { name: "after grace period: no watching, cleanup must be called", - setupUpgradeMarker: func(t *testing.T, topDir string, watcher *cmdmocks.AgentWatcher, installModifier *cmdmocks.InstallationModifier) { + setupUpgradeMarker: func(t *testing.T, topDir string, watcher *mockAgentWatcher, installModifier *mockInstallationModifier) { dataDirPath := paths.DataFrom(topDir) err := os.MkdirAll(dataDirPath, 0755) require.NoError(t, err) @@ -268,16 +272,330 @@ func Test_watchCmd(t *testing.T) { }, wantErr: assert.NoError, }, + { + name: "Desired outcome is rollback, rollback immediately", + setupUpgradeMarker: func(t *testing.T, tmpDir string, watcher *mockAgentWatcher, installModifier *mockInstallationModifier) { + dataDirPath := paths.DataFrom(tmpDir) + err := os.MkdirAll(dataDirPath, 0755) + require.NoError(t, err) + // upgrade started yesterday ;) + updatedOn := time.Now().Add(-1 * 24 * time.Hour) + err = upgrade.SaveMarker( + dataDirPath, + &upgrade.UpdateMarker{ + Version: "4.5.6", + Hash: "newver", + VersionedHome: "elastic-agent-4.5.6-newver", + UpdatedOn: updatedOn, + PrevVersion: "1.2.3", + PrevHash: "prvver", + PrevVersionedHome: "elastic-agent-prvver", + Acked: false, + Action: nil, + Details: &details.Details{ + TargetVersion: "4.5.6", + State: details.StateWatching, + ActionID: "", + Metadata: details.Metadata{}, + }, + DesiredOutcome: upgrade.OUTCOME_ROLLBACK, + }, + true, + ) + require.NoError(t, err) + + installModifier.EXPECT(). + Rollback(mock.Anything, mock.Anything, mock.Anything, paths.Top(), "elastic-agent-prvver", "prvver", mock.Anything). + Return(nil) + }, + args: args{ + cfg: configuration.DefaultUpgradeConfig().Watcher, + }, + wantErr: assert.NoError, + }, + { + name: "Desired outcome is rollback no upgrade details, no rollback and simple cleanup", + setupUpgradeMarker: func(t *testing.T, tmpDir string, watcher *mockAgentWatcher, installModifier *mockInstallationModifier) { + dataDirPath := paths.DataFrom(tmpDir) + err := os.MkdirAll(dataDirPath, 0755) + require.NoError(t, err) + // upgrade started yesterday ;) + updatedOn := time.Now().Add(-1 * 24 * time.Hour) + err = upgrade.SaveMarker( + dataDirPath, + &upgrade.UpdateMarker{ + Version: "4.5.6", + Hash: "newver", + VersionedHome: "elastic-agent-4.5.6-newver", + UpdatedOn: updatedOn, + PrevVersion: "1.2.3", + PrevHash: "prvver", + PrevVersionedHome: "elastic-agent-prvver", + Acked: false, + Action: &fleetapi.ActionUpgrade{ + ActionID: "action-id", + ActionType: fleetapi.ActionTypeUpgrade, + Data: fleetapi.ActionUpgradeData{Version: "4.5.6"}, + }, + Details: nil, + DesiredOutcome: upgrade.OUTCOME_ROLLBACK, + }, + true, + ) + require.NoError(t, err) + + installModifier.EXPECT(). + Cleanup(mock.Anything, paths.Top(), paths.VersionedHome(tmpDir), release.ShortCommit(), true, false). + Return(nil) + }, + args: args{ + cfg: configuration.DefaultUpgradeConfig().Watcher, + }, + wantErr: assert.NoError, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { log, obs := loggertest.New(t.Name()) tmpDir := t.TempDir() - mockWatcher := cmdmocks.NewAgentWatcher(t) - mockInstallModifier := cmdmocks.NewInstallationModifier(t) + mockWatcher := newMockAgentWatcher(t) + mockInstallModifier := newMockInstallationModifier(t) tt.setupUpgradeMarker(t, tmpDir, mockWatcher, mockInstallModifier) tt.wantErr(t, watchCmd(log, tmpDir, tt.args.cfg, mockWatcher, mockInstallModifier), fmt.Sprintf("watchCmd(%v, ...)", tt.args.cfg)) - t.Logf("watchCmd logs:\n%v", obs.All()) + t.Log("watchCmd logs:\n") + for _, osbLog := range obs.All() { + t.Logf("\t%s - %s - %v\n", osbLog.Level, osbLog.Message, osbLog.Context) + } }) } } + +func Test_takedownWatcher(t *testing.T) { + + const applockerFileName = "mocklocker.lock" + + testExecutablePath := filepath.Join("..", "application", "filelock", "testlocker", "testlocker") + if runtime.GOOS == "windows" { + testExecutablePath += ".exe" + } + testExecutableAbsolutePath, err := filepath.Abs(testExecutablePath) + require.NoError(t, err, "error calculating absolute test executable part") + + require.FileExists(t, testExecutableAbsolutePath, + "testlocker binary not found.\n"+ + "Check that:\n"+ + "- test binaries have been built with mage build:testbinaries\n"+ + "- the path of the executable is correct") + + returnCmdPIDsFetcher := func(cmds ...*exec.Cmd) watcherPIDsFetcher { + return func() ([]int, error) { + pids := make([]int, 0, len(cmds)) + for _, c := range cmds { + if c.Process != nil { + pids = append(pids, c.Process.Pid) + } + } + + return pids, nil + } + } + + // create a struct with a *exec.Cmd and a channel that will be closed when Wait() returns for the exec.Cmd + // this should keep the data race detector happy. + type testProcess struct { + cmd *exec.Cmd + waitChan chan struct{} + } + + type setupFunc func(t *testing.T, log *logger.Logger, workdir string) (watcherPIDsFetcher, []testProcess) + type assertFunc func(t *testing.T, workdir string, cmds []testProcess) + + tests := []struct { + name string + setup setupFunc + wantErr assert.ErrorAssertionFunc + assertPostTakedown assertFunc + }{ + { + name: "no contention for watcher applocker", + setup: func(_ *testing.T, _ *logger.Logger, _ string) (watcherPIDsFetcher, []testProcess) { + // nothing to do here, always return and empty list of pids + return func() ([]int, error) { + return nil, nil + }, nil + }, + wantErr: assert.NoError, + assertPostTakedown: func(t *testing.T, workdir string, _ []testProcess) { + // we should be able to lock, no problem + locker := filelock.NewAppLocker(workdir, applockerFileName) + lockError := locker.TryLock() + t.Cleanup(func() { + _ = locker.Unlock() + }) + + assert.NoError(t, lockError) + + }, + }, + { + name: "contention with test binary listening to signals: test binary is terminated gracefully", + setup: func(t *testing.T, log *logger.Logger, workdir string) (watcherPIDsFetcher, []testProcess) { + cmd, testChan := createTestlockerCommand(t, log.Named("testlocker"), applockerFileName, testExecutableAbsolutePath, workdir, false) + require.NoError(t, err, "error starting testlocker binary") + + // wait for test binary to acquire lock + require.EventuallyWithT(t, func(collect *assert.CollectT) { + assert.FileExists(collect, filepath.Join(workdir, applockerFileName), "watcher applocker should have been created by the test binary") + }, 10*time.Second, 100*time.Millisecond) + require.NotNil(t, cmd.Process, "process details for testlocker should not be nil") + + t.Logf("started testlocker process with PID %d", cmd.Process.Pid) + + return returnCmdPIDsFetcher(cmd), []testProcess{{cmd: cmd, waitChan: testChan}} + }, + wantErr: assert.NoError, + assertPostTakedown: func(t *testing.T, workdir string, cmds []testProcess) { + + assert.Len(t, cmds, 1) + testlockerProcess := cmds[0] + require.NotNil(t, testlockerProcess, "test locker process info should have a not nil cmd") + + require.Eventually(t, func() bool { + running, checkErr := isProcessRunning(t, testlockerProcess.cmd) + if checkErr != nil { + t.Logf("error checking for testlocker process running: %s", checkErr.Error()) + return false + } + return !running + }, 30*time.Second, 100*time.Millisecond, "test locker process should have exited") + + <-testlockerProcess.waitChan + + assert.True(t, testlockerProcess.cmd.ProcessState.Exited(), "test locker process should have terminated") + assert.Equal(t, 0, testlockerProcess.cmd.ProcessState.ExitCode(), "test locker process should have a successful exit status") + + assert.FileExists(t, filepath.Join(workdir, applockerFileName)) + testApplocker := filelock.NewAppLocker(workdir, applockerFileName) + testApplockerError := testApplocker.TryLock() + t.Cleanup(func() { + _ = testApplocker.Unlock() + }) + assert.NoError(t, testApplockerError, "error locking applocker") + }, + }, + { + name: "contention with test binary not listening to signals: test binary is not terminated", + setup: func(t *testing.T, log *logger.Logger, workdir string) (watcherPIDsFetcher, []testProcess) { + cmd, waitChan := createTestlockerCommand(t, log.Named("testlocker"), applockerFileName, testExecutableAbsolutePath, workdir, true) + require.NoError(t, err, "error starting testlocker binary") + + // wait for test binary to acquire lock + require.EventuallyWithT(t, func(collect *assert.CollectT) { + assert.FileExists(collect, filepath.Join(workdir, applockerFileName), "watcher applocker should have been created by the test binary") + }, 10*time.Second, 100*time.Millisecond) + require.NotNil(t, cmd.Process, "process details for testlocker should not be nil") + + t.Logf("started testlocker process with PID %d", cmd.Process.Pid) + + return returnCmdPIDsFetcher(cmd), []testProcess{{cmd: cmd, waitChan: waitChan}} + }, + wantErr: assert.NoError, + assertPostTakedown: func(t *testing.T, workdir string, cmds []testProcess) { + + assert.Len(t, cmds, 1) + testlockerProcess := cmds[0] + require.NotNil(t, testlockerProcess, "test locker process info should have exec.Cmd set") + + // check that the process is still running for a time + assert.Never(t, func() bool { + running, checkErr := isProcessRunning(t, testlockerProcess.cmd) + if checkErr != nil { + t.Logf("error checking for testlocker process running: %s", checkErr.Error()) + return false + } + return !running + }, 1*time.Second, 100*time.Millisecond, "test locker process should still be running for some time") + + // Kill the process explicitly + err = testlockerProcess.cmd.Process.Kill() + assert.NoError(t, err, "error killing testlocker process") + + <-testlockerProcess.waitChan + + if assert.NotNil(t, testlockerProcess.cmd.ProcessState, "test locker process should have been terminated") { + assert.NotEqual(t, 0, testlockerProcess.cmd.ProcessState.ExitCode(), "test locker process should not return a successful exit code") + } + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + workDir := t.TempDir() + log, obsLogs := loggertest.New(t.Name()) + t.Cleanup(func() { + // however it ends, try to print out the logs of takedownWatcher + loggertest.PrintObservedLogs(obsLogs.All(), t.Log) + }) + pidFetcher, processInfos := tc.setup(t, log, workDir) + tc.wantErr(t, takedownWatcher(t.Context(), log.Named("takedownWatcher"), pidFetcher)) + if tc.assertPostTakedown != nil { + tc.assertPostTakedown(t, workDir, processInfos) + } + }) + } +} + +func createTestlockerCommand(t *testing.T, log *logger.Logger, applockerFileName string, testExecutablePath string, workdir string, ignoreSignals bool) (*exec.Cmd, chan struct{}) { + + watchTerminated := make(chan struct{}) + + args := []string{"-lockfile", filepath.Join(workdir, applockerFileName)} + if ignoreSignals { + args = append(args, "-ignoresignals") + } + + // use the same invoke as the one used to launch a watcher + watcherCmd, err := upgrade.StartWatcherCmd(log, func() *exec.Cmd { + cmd := upgrade.InvokeCmdWithArgs(testExecutablePath, args...) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd + }, + upgrade.WithWatcherPostWaitHook(func() { + close(watchTerminated) + }), + ) + + require.NoError(t, err, "error starting testlocker binary") + return watcherCmd, watchTerminated +} + +func isProcessRunning(t *testing.T, cmd *exec.Cmd) (bool, error) { + if cmd.Process == nil { + return false, nil + } + t.Logf("checking if pid %d is still running", cmd.Process.Pid) + // search for the pid on the running processes + process, err := os.FindProcess(cmd.Process.Pid) + if err != nil { + t.Logf("error string: %q", err.Error()) + if runtime.GOOS == "windows" && strings.Contains(err.Error(), "The parameter is incorrect") { + // in windows, noone can hear you scream + // invalid parameter means that the process object cannot be found + t.Logf("pid %d is not running because on windows we got an incorrect parameter error", cmd.Process.Pid) + return false, nil + } + + t.Logf("error finding process: %T %v", err, err) + return false, err + } + + if process == nil { + t.Logf("pid %d is not running because os.GetProcess returned a nil process", cmd.Process.Pid) + return false, nil + } + + return isProcessLive(cmd.Process) +} diff --git a/internal/pkg/agent/cmd/watch_windows.go b/internal/pkg/agent/cmd/watch_windows.go new file mode 100644 index 00000000000..90ef342ff16 --- /dev/null +++ b/internal/pkg/agent/cmd/watch_windows.go @@ -0,0 +1,136 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build windows + +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "unsafe" + + "golang.org/x/sys/windows" + + "github.com/elastic/elastic-agent/pkg/core/logger" +) + +var ( + kernel32API = windows.NewLazySystemDLL("kernel32.dll") + + freeConsoleProc = kernel32API.NewProc("FreeConsole") + attachConsoleProc = kernel32API.NewProc("AttachConsole") + procGetConsoleProcessList = kernel32API.NewProc("GetConsoleProcessList") +) + +func takedownWatcher(ctx context.Context, log *logger.Logger, pidFetchFunc watcherPIDsFetcher) error { + pids, err := pidFetchFunc() + if err != nil { + return fmt.Errorf("error listing watcher processes: %w", err) + } + + ownPID := os.Getpid() + + var accumulatedSignalingErrors error + for _, pid := range pids { + + if ctx.Err() != nil { + return ctx.Err() + } + + if pid == ownPID { + continue + } + + log.Debugf("attempting to terminate watcher process with PID: %d", pid) + accumulatedSignalingErrors = errors.Join(accumulatedSignalingErrors, signalPID(log, pid)) + } + + return accumulatedSignalingErrors +} + +// GetConsoleProcessList retrieves the list of process IDs attached to the current console +func GetConsoleProcessList() ([]uint32, error) { + // Allocate a buffer for PIDs + const maxProcs = 64 + pids := make([]uint32, maxProcs) + + r1, _, err := procGetConsoleProcessList.Call( + uintptr(unsafe.Pointer(&pids[0])), + uintptr(maxProcs), + ) + + count := uint32(r1) + if count == 0 { + return nil, err + } + + return pids[:count], nil +} + +// signalPID takes care of signaling a given PID. It also leverages defer() for freeing console and other housekeeping +func signalPID(log *logger.Logger, pid int) error { + r1, _, consoleErr := freeConsoleProc.Call() + if r1 == 0 { + log.Warnf("error preemptively detaching from console: %s", consoleErr) + } + + r1, _, consoleErr = attachConsoleProc.Call(uintptr(pid)) + if r1 == 0 { + return fmt.Errorf("error attaching console to watcher process with PID %d: %w", pid, consoleErr) + } + log.Infof("successfully attached console with PID: %d", pid) + + defer func() { + r1, _, consoleErr = freeConsoleProc.Call() + if r1 == 0 { + log.Errorf("error detaching from console: %s", consoleErr) + } else { + log.Infof("successfully detached from console of PID: %d", pid) + } + }() + + if list, consoleProcessListErr := GetConsoleProcessList(); consoleProcessListErr != nil { + log.Errorf("error listing console processes: %s", consoleProcessListErr) + } else { + log.Infof("Own PID: %d, Watcher pid %d, Process list on console: %v", os.Getpid(), pid, list) + } + + // Normally we would want to send the Ctrl+Break event only to the watcher process but due to the fact that + // the parent process of the watcher has already terminated, we have to hug it tightly and take it down with us + // by specifying processGroupID=0 + //nolint:gosec // int -> uint32 no overflow is possible since windows PID is a DWORD (uint32) (see https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getprocessid and https://learn.microsoft.com/en-us/windows/win32/winprog/windows-data-types) + killProcErr := windows.GenerateConsoleCtrlEvent(windows.CTRL_BREAK_EVENT, uint32(pid)) + + if killProcErr != nil { + return fmt.Errorf("error signaling process with PID: %d: %w", pid, killProcErr) + } + + return nil +} + +func isProcessLive(process *os.Process) (bool, error) { + //exitCodeStillActive according to https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getexitcodeprocess + const exitCodeStillActive = 259 + // Open the process with PROCESS_QUERY_LIMITED_INFORMATION access + //nolint:gosec // int -> uint32 no overflow is possible since windows PID is a DWORD (uint32) (see https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getprocessid and https://learn.microsoft.com/en-us/windows/win32/winprog/windows-data-types) + handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(process.Pid)) + if err != nil { + return false, fmt.Errorf("OpenProcess failed: %w", err) + } + + defer func(handle windows.Handle) { + _ = windows.CloseHandle(handle) + }(handle) + + var exitCode uint32 + err = windows.GetExitCodeProcess(handle, &exitCode) + if err != nil { + return false, fmt.Errorf("getting process exit code: %w", err) + } + + return exitCode == exitCodeStillActive, nil +} diff --git a/internal/pkg/agent/configuration/upgrade.go b/internal/pkg/agent/configuration/upgrade.go index 405b405ec46..bd79fd3f94c 100644 --- a/internal/pkg/agent/configuration/upgrade.go +++ b/internal/pkg/agent/configuration/upgrade.go @@ -15,7 +15,9 @@ const ( // period during which an upgraded Agent can be asked to rollback to the previous // Agent version on disk. - defaultRollbackWindowDuration = 7 * 24 * time.Hour // 7 days + // this is temporarily set to 0 to disable the rollback window until manual rollback functionality is complete. + // defaultRollbackWindowDuration = 7 * 24 * time.Hour // 7 days + defaultRollbackWindowDuration = 0 ) // UpgradeConfig is the configuration related to Agent upgrades. diff --git a/internal/pkg/agent/install/install.go b/internal/pkg/agent/install/install.go index 79f62f88833..70015865c4e 100644 --- a/internal/pkg/agent/install/install.go +++ b/internal/pkg/agent/install/install.go @@ -17,6 +17,7 @@ import ( "github.com/kardianos/service" "github.com/otiai10/copy" "github.com/schollz/progressbar/v3" + "gopkg.in/yaml.v3" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" @@ -25,6 +26,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/cli" v1 "github.com/elastic/elastic-agent/pkg/api/v1" "github.com/elastic/elastic-agent/pkg/utils" + manifestutils "github.com/elastic/elastic-agent/pkg/utils/manifest" ) const ( @@ -61,17 +63,20 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p } } - err = setupInstallPath(topPath, ownership) - if err != nil { - return utils.FileOwner{}, fmt.Errorf("error setting up install path: %w", err) - } - manifest, err := readPackageManifest(dir) if err != nil { return utils.FileOwner{}, fmt.Errorf("reading package manifest: %w", err) } pathMappings := manifest.Package.PathMappings + pathMapper := manifestutils.NewPathMapper(pathMappings) + + targetVersionedHome := filepath.FromSlash(pathMapper.Map(manifest.Package.VersionedHome)) + + err = setupInstallPath(topPath, ownership, targetVersionedHome, manifest.Package.Version) + if err != nil { + return utils.FileOwner{}, fmt.Errorf("error setting up install path: %w", err) + } pt.Describe("Copying install files") copyConcurrency := calculateCopyConcurrency(streams) @@ -184,7 +189,7 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p } // setup the basic topPath, and the .installed file -func setupInstallPath(topPath string, ownership utils.FileOwner) error { +func setupInstallPath(topPath string, ownership utils.FileOwner, versionedHome string, version string) error { // ensure parent directory exists err := os.MkdirAll(filepath.Dir(topPath), 0755) if err != nil { @@ -198,7 +203,7 @@ func setupInstallPath(topPath string, ownership utils.FileOwner) error { } // create the install marker - if err := CreateInstallMarker(topPath, ownership); err != nil { + if err := CreateInstallMarker(topPath, ownership, versionedHome, version); err != nil { return fmt.Errorf("failed to create install marker: %w", err) } return nil @@ -516,16 +521,32 @@ func hasAllSSDs(block ghw.BlockInfo) bool { // CreateInstallMarker creates a `.installed` file at the given install path, // and then calls fixInstallMarkerPermissions to set the ownership provided by `ownership` -func CreateInstallMarker(topPath string, ownership utils.FileOwner) error { +func CreateInstallMarker(topPath string, ownership utils.FileOwner, home string, version string) error { markerFilePath := filepath.Join(topPath, paths.MarkerFileName) - handle, err := os.Create(markerFilePath) + err := createInstallMarkerFile(markerFilePath, version, home) if err != nil { - return err + return fmt.Errorf("creating install marker: %w", err) } - _ = handle.Close() return fixInstallMarkerPermissions(markerFilePath, ownership) } +func createInstallMarkerFile(markerFilePath string, version string, home string) error { + handle, err := os.Create(markerFilePath) + if err != nil { + return fmt.Errorf("creating destination file %q : %w", markerFilePath, err) + } + defer func() { + _ = handle.Close() + }() + installDescriptor := v1.NewInstallDescriptor() + installDescriptor.AgentInstalls = []v1.AgentInstallDesc{{Version: version, VersionedHome: home}} + err = yaml.NewEncoder(handle).Encode(installDescriptor) + if err != nil { + return fmt.Errorf("writing install descriptor: %w", err) + } + return nil +} + func UnprivilegedUser(username, password string) (string, string) { if username != "" { return username, password diff --git a/internal/pkg/agent/install/install_test.go b/internal/pkg/agent/install/install_test.go index f2716e493f6..98aea5cce10 100644 --- a/internal/pkg/agent/install/install_test.go +++ b/internal/pkg/agent/install/install_test.go @@ -224,7 +224,19 @@ func TestSetupInstallPath(t *testing.T) { tmpdir := t.TempDir() ownership, err := utils.CurrentFileOwner() require.NoError(t, err) - err = setupInstallPath(tmpdir, ownership) + err = setupInstallPath(tmpdir, ownership, "data/elastic-agent-1.2.3-SNAPSHOT", "1.2.3-SNAPSHOT") require.NoError(t, err) - require.FileExists(t, filepath.Join(tmpdir, paths.MarkerFileName)) + markerFilePath := filepath.Join(tmpdir, paths.MarkerFileName) + require.FileExists(t, markerFilePath) + + const expectedInstallDescriptor = ` + version: co.elastic.agent/v1 + kind: InstallDescriptor + agentInstalls: + - version: 1.2.3-SNAPSHOT + versioned-home: data/elastic-agent-1.2.3-SNAPSHOT + ` + actualInstallDescriptorBytes, err := os.ReadFile(markerFilePath) + require.NoError(t, err, "error reading actual install descriptor") + assert.YAMLEq(t, expectedInstallDescriptor, string(actualInstallDescriptorBytes), "expected and actual install descriptor do not match") } diff --git a/magefile.go b/magefile.go index ede727682fe..3e9ef701305 100644 --- a/magefile.go +++ b/magefile.go @@ -262,31 +262,7 @@ func (Dev) RegenerateMocks() error { return fmt.Errorf("generating mocks: %w", err) } - // change CWD - workingDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("retrieving CWD: %w", err) - } - // restore the working directory when exiting the function - defer func() { - err := os.Chdir(workingDir) - if err != nil { - panic(fmt.Errorf("failed to restore working dir %q: %w", workingDir, err)) - } - }() - - mPath, err := mocksPath() - if err != nil { - return fmt.Errorf("retrieving mocks path: %w", err) - } - - err = os.Chdir(mPath) - if err != nil { - return fmt.Errorf("changing current directory to %q: %w", mPath, err) - } - - mg.Deps(devtools.AddLicenseHeaders) - mg.Deps(devtools.GoImports) + mg.Deps(devtools.Format) return nil } @@ -438,6 +414,7 @@ func getTestBinariesPath() ([]string, error) { filepath.Join(wd, "internal", "pkg", "agent", "install", "testblocking"), filepath.Join(wd, "pkg", "core", "process", "testsignal"), filepath.Join(wd, "internal", "pkg", "otel", "manager", "testing"), + filepath.Join(wd, "internal", "pkg", "agent", "application", "filelock", "testlocker"), } return testBinaryPkgs, nil } diff --git a/pkg/control/v2/client/client.go b/pkg/control/v2/client/client.go index 0846266293f..1308eb020eb 100644 --- a/pkg/control/v2/client/client.go +++ b/pkg/control/v2/client/client.go @@ -205,7 +205,7 @@ type Client interface { // Restart triggers restarting the current running daemon. Restart(ctx context.Context) error // Upgrade triggers upgrade of the current running daemon. - Upgrade(ctx context.Context, version string, sourceURI string, skipVerify bool, skipDefaultPgp bool, pgpBytes ...string) (string, error) + Upgrade(ctx context.Context, version string, rollback bool, sourceURI string, skipVerify bool, skipDefaultPgp bool, pgpBytes ...string) (string, error) // DiagnosticAgent gathers diagnostics information for the running Elastic Agent. DiagnosticAgent(ctx context.Context, additionalDiags []AdditionalMetrics) ([]DiagnosticFileResult, error) // DiagnosticUnits gathers diagnostics information from specific units (or all if non are provided). @@ -328,13 +328,14 @@ func (c *client) Restart(ctx context.Context) error { } // Upgrade triggers upgrade of the current running daemon. -func (c *client) Upgrade(ctx context.Context, version string, sourceURI string, skipVerify bool, skipDefaultPgp bool, pgpBytes ...string) (string, error) { +func (c *client) Upgrade(ctx context.Context, version string, rollback bool, sourceURI string, skipVerify bool, skipDefaultPgp bool, pgpBytes ...string) (string, error) { res, err := c.client.Upgrade(ctx, &cproto.UpgradeRequest{ Version: version, SourceURI: sourceURI, SkipVerify: skipVerify, PgpBytes: pgpBytes, SkipDefaultPgp: skipDefaultPgp, + Rollback: rollback, }) if err != nil { return "", err diff --git a/pkg/control/v2/cproto/control_v2.pb.go b/pkg/control/v2/cproto/control_v2.pb.go index 674529fdbad..57c0407751a 100644 --- a/pkg/control/v2/cproto/control_v2.pb.go +++ b/pkg/control/v2/cproto/control_v2.pb.go @@ -580,6 +580,8 @@ type UpgradeRequest struct { // // If provided Elastic Agent package embedded PGP key is not checked for signature during upgrade. SkipDefaultPgp bool `protobuf:"varint,5,opt,name=skipDefaultPgp,proto3" json:"skipDefaultPgp,omitempty"` + // If true it indicates that we wish to rollback the current/last upgrade + Rollback bool `protobuf:"varint,6,opt,name=rollback,proto3" json:"rollback,omitempty"` } func (x *UpgradeRequest) Reset() { @@ -649,6 +651,13 @@ func (x *UpgradeRequest) GetSkipDefaultPgp() bool { return false } +func (x *UpgradeRequest) GetRollback() bool { + if x != nil { + return x.Rollback + } + return false +} + // A upgrade response message. type UpgradeResponse struct { state protoimpl.MessageState @@ -2123,7 +2132,7 @@ var file_control_v2_proto_rawDesc = []byte{ 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, - 0xac, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0xc8, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x55, 0x52, 0x49, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, @@ -2133,300 +2142,302 @@ var file_control_v2_proto_rawDesc = []byte{ 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x67, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x6b, 0x69, 0x70, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x67, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, - 0x73, 0x6b, 0x69, 0x70, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x67, 0x70, 0x22, 0x6f, - 0x0a, 0x0f, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x2c, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x14, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, - 0xb5, 0x01, 0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x6e, 0x69, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2d, 0x0a, 0x09, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x63, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x55, 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x75, 0x6e, 0x69, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x6e, 0x69, 0x74, 0x49, 0x64, 0x12, 0x23, - 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, - 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x9f, 0x01, 0x0a, 0x14, 0x43, 0x6f, 0x6d, 0x70, - 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6d, 0x70, - 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, - 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe6, 0x01, 0x0a, 0x0e, 0x43, 0x6f, - 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x23, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, - 0x30, 0x0a, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, + 0x73, 0x6b, 0x69, 0x70, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x67, 0x70, 0x12, 0x1a, + 0x0a, 0x08, 0x72, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x08, 0x72, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x22, 0x6f, 0x0a, 0x0f, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, + 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xb5, 0x01, 0x0a, 0x12, + 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x6e, 0x69, 0x74, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x2d, 0x0a, 0x09, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, + 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x75, 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x75, 0x6e, 0x69, 0x74, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x05, 0x73, 0x74, + 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x22, 0x9f, 0x01, 0x0a, 0x14, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x3a, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x55, 0x6e, 0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x75, 0x6e, 0x69, 0x74, - 0x73, 0x12, 0x3f, 0x0a, 0x0c, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, - 0x6f, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0b, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, - 0x66, 0x6f, 0x22, 0xe0, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, - 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, - 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, - 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, - 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, - 0x74, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, - 0x70, 0x69, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x75, 0x6e, 0x70, 0x72, 0x69, 0x76, 0x69, 0x6c, 0x65, - 0x67, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x75, 0x6e, 0x70, 0x72, 0x69, - 0x76, 0x69, 0x6c, 0x65, 0x67, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x73, 0x4d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x4d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x64, 0x22, 0xc9, 0x02, 0x0a, 0x12, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x63, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1c, 0x0a, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x62, 0x0a, 0x12, 0x43, 0x6f, + 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x65, 0x74, + 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, + 0x4d, 0x65, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe6, 0x01, 0x0a, 0x0e, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x63, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x75, + 0x6e, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x6e, 0x69, + 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x12, 0x3f, 0x0a, + 0x0c, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x0b, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xe0, + 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x63, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x54, 0x69, 0x6d, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x10, 0x0a, + 0x03, 0x70, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, + 0x22, 0x0a, 0x0c, 0x75, 0x6e, 0x70, 0x72, 0x69, 0x76, 0x69, 0x6c, 0x65, 0x67, 0x65, 0x64, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x75, 0x6e, 0x70, 0x72, 0x69, 0x76, 0x69, 0x6c, 0x65, + 0x67, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x73, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x64, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x64, 0x22, 0xc9, 0x02, 0x0a, 0x12, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, + 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x62, 0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4d, 0x61, 0x70, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x2e, 0x43, + 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4d, 0x61, + 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x12, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4d, 0x61, 0x70, 0x1a, 0x61, 0x0a, 0x17, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4d, 0x61, 0x70, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x12, 0x43, 0x6f, 0x6d, 0x70, - 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4d, 0x61, 0x70, 0x1a, 0x61, - 0x0a, 0x17, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, - 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x80, 0x03, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, - 0x23, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, - 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2d, - 0x0a, 0x0a, 0x66, 0x6c, 0x65, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x52, 0x0a, 0x66, 0x6c, 0x65, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x22, 0x0a, - 0x0c, 0x66, 0x6c, 0x65, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x6c, 0x65, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x63, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0f, 0x75, 0x70, 0x67, - 0x72, 0x61, 0x64, 0x65, 0x5f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x67, 0x72, - 0x61, 0x64, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0e, 0x75, 0x70, 0x67, 0x72, - 0x61, 0x64, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x63, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x09, 0x63, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x22, 0xa6, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, - 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, - 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x64, 0x12, 0x3a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x67, - 0x72, 0x61, 0x64, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x87, 0x02, - 0x0a, 0x16, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x63, 0x68, 0x65, - 0x64, 0x75, 0x6c, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x41, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x64, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, - 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, - 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x61, - 0x69, 0x6c, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x26, 0x0a, 0x0f, 0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0d, 0x72, 0x65, 0x74, 0x72, 0x79, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x1f, - 0x0a, 0x0b, 0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x74, 0x72, 0x79, 0x55, 0x6e, 0x74, 0x69, 0x6c, 0x12, - 0x16, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0xdf, 0x01, 0x0a, 0x14, 0x44, 0x69, 0x61, 0x67, - 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, - 0x38, 0x0a, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x22, 0x6c, 0x0a, 0x16, 0x44, 0x69, 0x61, - 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x52, 0x0a, 0x12, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32, - 0x23, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x61, 0x6c, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x52, 0x11, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, 0xb5, 0x01, 0x0a, 0x1b, 0x44, 0x69, 0x61, 0x67, - 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, - 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, - 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x52, 0x0a, 0x12, 0x61, - 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x44, 0x69, 0x61, 0x67, 0x6e, - 0x6f, 0x73, 0x74, 0x69, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x11, 0x61, 0x64, - 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, - 0x3f, 0x0a, 0x1a, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, - 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, - 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x49, 0x64, - 0x22, 0x51, 0x0a, 0x17, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, - 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x72, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, - 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x15, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, - 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, - 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x49, 0x64, - 0x12, 0x2d, 0x0a, 0x09, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x6e, 0x69, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x75, 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x17, 0x0a, 0x07, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x75, 0x6e, 0x69, 0x74, 0x49, 0x64, 0x22, 0x4d, 0x0a, 0x16, 0x44, 0x69, 0x61, 0x67, - 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x33, 0x0a, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, - 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x52, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x22, 0xd1, 0x01, 0x0a, 0x16, 0x44, 0x69, 0x61, 0x67, - 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, - 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x2d, 0x0a, 0x09, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x55, 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x75, 0x6e, 0x69, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x69, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x6e, 0x69, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, - 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x12, 0x36, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x05, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, - 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x8e, 0x01, 0x0a, 0x1b, - 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, - 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x14, - 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x12, 0x36, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, - 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, - 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x4f, 0x0a, 0x17, - 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x22, 0x2a, 0x0a, - 0x10, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2a, 0x85, 0x01, 0x0a, 0x05, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, - 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, 0x4e, 0x47, - 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x48, 0x45, 0x41, 0x4c, 0x54, 0x48, 0x59, 0x10, 0x02, 0x12, - 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0a, 0x0a, - 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x4f, - 0x50, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, - 0x45, 0x44, 0x10, 0x06, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x49, 0x4e, - 0x47, 0x10, 0x07, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x4f, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, - 0x08, 0x2a, 0xbf, 0x01, 0x0a, 0x18, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0e, - 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x12, - 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, - 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4f, 0x4b, 0x10, 0x02, - 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, - 0x72, 0x61, 0x62, 0x6c, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x03, 0x12, 0x18, 0x0a, 0x14, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x61, 0x6e, 0x65, 0x6e, 0x74, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x10, 0x04, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x46, 0x61, 0x74, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x05, 0x12, 0x12, 0x0a, 0x0e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x6f, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x10, 0x06, - 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x6f, 0x70, 0x70, 0x65, - 0x64, 0x10, 0x07, 0x2a, 0x21, 0x0a, 0x08, 0x55, 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x09, 0x0a, 0x05, 0x49, 0x4e, 0x50, 0x55, 0x54, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x55, - 0x54, 0x50, 0x55, 0x54, 0x10, 0x01, 0x2a, 0x28, 0x0a, 0x0c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, - 0x53, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, - 0x2a, 0x7f, 0x0a, 0x0b, 0x50, 0x70, 0x72, 0x6f, 0x66, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x0a, 0x0a, 0x06, 0x41, 0x4c, 0x4c, 0x4f, 0x43, 0x53, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, - 0x4c, 0x4f, 0x43, 0x4b, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4d, 0x44, 0x4c, 0x49, 0x4e, - 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x47, 0x4f, 0x52, 0x4f, 0x55, 0x54, 0x49, 0x4e, 0x45, - 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x45, 0x41, 0x50, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, - 0x4d, 0x55, 0x54, 0x45, 0x58, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x4f, 0x46, 0x49, - 0x4c, 0x45, 0x10, 0x06, 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x43, 0x52, - 0x45, 0x41, 0x54, 0x45, 0x10, 0x07, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, - 0x08, 0x2a, 0x30, 0x0a, 0x1b, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x44, + 0x6e, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x80, 0x03, + 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x2a, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, + 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x23, 0x0a, 0x05, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x63, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2d, 0x0a, 0x0a, 0x66, 0x6c, + 0x65, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, + 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x66, + 0x6c, 0x65, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x66, 0x6c, 0x65, + 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0c, 0x66, 0x6c, 0x65, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x36, 0x0a, + 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0f, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, + 0x5f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, + 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0e, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, + 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x09, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x22, 0xa6, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x44, 0x65, 0x74, 0x61, + 0x69, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, + 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x3a, 0x0a, + 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, + 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, + 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x87, 0x02, 0x0a, 0x16, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, + 0x64, 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x63, 0x68, 0x65, + 0x64, 0x75, 0x6c, 0x65, 0x64, 0x41, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x64, 0x6f, 0x77, 0x6e, 0x6c, + 0x6f, 0x61, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x02, 0x52, 0x0f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, + 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, + 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, + 0x73, 0x67, 0x12, 0x26, 0x0a, 0x0f, 0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x72, 0x65, 0x74, + 0x72, 0x79, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, + 0x74, 0x72, 0x79, 0x5f, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x72, 0x65, 0x74, 0x72, 0x79, 0x55, 0x6e, 0x74, 0x69, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x72, + 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61, + 0x73, 0x6f, 0x6e, 0x22, 0xdf, 0x01, 0x0a, 0x14, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, + 0x69, 0x63, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, + 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, + 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x67, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x65, 0x64, 0x22, 0x6c, 0x0a, 0x16, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, + 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x52, 0x0a, 0x12, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x63, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x07, 0x0a, 0x03, 0x43, 0x50, 0x55, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x4f, 0x4e, - 0x4e, 0x10, 0x01, 0x32, 0xdf, 0x04, 0x0a, 0x13, 0x45, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x31, 0x0a, 0x07, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x17, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, - 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x15, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, - 0x0a, 0x53, 0x74, 0x61, 0x74, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x12, 0x0d, 0x2e, 0x63, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x15, 0x2e, 0x63, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x30, 0x01, 0x12, 0x31, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x0d, - 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x17, 0x2e, - 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, - 0x65, 0x12, 0x16, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, - 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x52, 0x0a, 0x0f, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, - 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, - 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x53, 0x0a, 0x0f, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, - 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x12, 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, - 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x62, 0x0a, 0x14, 0x44, + 0x52, 0x11, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x22, 0xb5, 0x01, 0x0a, 0x1b, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, + 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0a, 0x63, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x52, 0x0a, 0x12, 0x61, 0x64, 0x64, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x64, 0x64, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, + 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x11, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, 0x3f, 0x0a, 0x1a, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x73, 0x12, 0x23, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, - 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, - 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, - 0x34, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e, 0x63, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x29, 0x5a, 0x24, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2f, 0x76, 0x32, 0x2f, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0xf8, 0x01, 0x01, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x51, 0x0a, 0x17, + 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x46, 0x69, 0x6c, 0x65, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, + 0x82, 0x01, 0x0a, 0x15, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, + 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x2d, 0x0a, 0x09, + 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x10, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x08, 0x75, 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x75, + 0x6e, 0x69, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x6e, + 0x69, 0x74, 0x49, 0x64, 0x22, 0x4d, 0x0a, 0x16, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, + 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, + 0x0a, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, + 0x63, 0x55, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x05, 0x75, 0x6e, + 0x69, 0x74, 0x73, 0x22, 0xd1, 0x01, 0x0a, 0x16, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, + 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, + 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x49, + 0x64, 0x12, 0x2d, 0x0a, 0x09, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x6e, + 0x69, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x75, 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x17, 0x0a, 0x07, 0x75, 0x6e, 0x69, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x75, 0x6e, 0x69, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, + 0x36, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1c, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, + 0x73, 0x74, 0x69, 0x63, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, + 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x8e, 0x01, 0x0a, 0x1b, 0x44, 0x69, 0x61, 0x67, + 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, + 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x12, 0x36, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, + 0x6f, 0x73, 0x74, 0x69, 0x63, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, + 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x4f, 0x0a, 0x17, 0x44, 0x69, 0x61, 0x67, + 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, + 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x52, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x22, 0x2a, 0x0a, 0x10, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2a, 0x85, 0x01, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x0f, 0x0a, + 0x0b, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0b, + 0x0a, 0x07, 0x48, 0x45, 0x41, 0x4c, 0x54, 0x48, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, + 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, + 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x4f, 0x50, 0x50, 0x49, 0x4e, + 0x47, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10, 0x06, + 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x12, + 0x0c, 0x0a, 0x08, 0x52, 0x4f, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x08, 0x2a, 0xbf, 0x01, + 0x0a, 0x18, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, 0x0c, + 0x0a, 0x08, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4f, 0x4b, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x62, 0x6c, + 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x03, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x61, 0x6e, 0x65, 0x6e, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x10, 0x04, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x46, 0x61, 0x74, 0x61, + 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x05, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x53, 0x74, 0x6f, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x10, 0x06, 0x12, 0x11, 0x0a, 0x0d, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x10, 0x07, 0x2a, + 0x21, 0x0a, 0x08, 0x55, 0x6e, 0x69, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x49, + 0x4e, 0x50, 0x55, 0x54, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x55, 0x54, 0x50, 0x55, 0x54, + 0x10, 0x01, 0x2a, 0x28, 0x0a, 0x0c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, + 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x2a, 0x7f, 0x0a, 0x0b, + 0x50, 0x70, 0x72, 0x6f, 0x66, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x41, + 0x4c, 0x4c, 0x4f, 0x43, 0x53, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x4c, 0x4f, 0x43, 0x4b, + 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4d, 0x44, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x02, 0x12, + 0x0d, 0x0a, 0x09, 0x47, 0x4f, 0x52, 0x4f, 0x55, 0x54, 0x49, 0x4e, 0x45, 0x10, 0x03, 0x12, 0x08, + 0x0a, 0x04, 0x48, 0x45, 0x41, 0x50, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x55, 0x54, 0x45, + 0x58, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x4f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x06, + 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, + 0x10, 0x07, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x08, 0x2a, 0x30, 0x0a, + 0x1b, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x44, 0x69, 0x61, 0x67, 0x6e, + 0x6f, 0x73, 0x74, 0x69, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x07, 0x0a, 0x03, + 0x43, 0x50, 0x55, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x4f, 0x4e, 0x4e, 0x10, 0x01, 0x32, + 0xdf, 0x04, 0x0a, 0x13, 0x45, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x31, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x1a, 0x17, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x05, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x1a, 0x15, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x0a, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x12, 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x15, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, + 0x31, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x0d, 0x2e, 0x63, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x17, 0x2e, 0x63, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, 0x16, 0x2e, + 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, + 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x52, + 0x0a, 0x0f, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x12, 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, + 0x6f, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, + 0x6f, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x53, 0x0a, 0x0f, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, + 0x55, 0x6e, 0x69, 0x74, 0x73, 0x12, 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, + 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, + 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x55, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x62, 0x0a, 0x14, 0x44, 0x69, 0x61, 0x67, 0x6e, + 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, + 0x23, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, + 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x69, + 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x34, 0x0a, 0x09, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x0d, 0x2e, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x42, 0x29, 0x5a, 0x24, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x6b, + 0x67, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, + 0x76, 0x32, 0x2f, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0xf8, 0x01, 0x01, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/control/v2/server/server.go b/pkg/control/v2/server/server.go index 5ef757b064e..c81d7985c52 100644 --- a/pkg/control/v2/server/server.go +++ b/pkg/control/v2/server/server.go @@ -174,7 +174,7 @@ func (s *Server) Restart(_ context.Context, _ *cproto.Empty) (*cproto.RestartRes // Upgrade performs the upgrade operation. func (s *Server) Upgrade(ctx context.Context, request *cproto.UpgradeRequest) (*cproto.UpgradeResponse, error) { - err := s.coord.Upgrade(ctx, request.Version, request.SourceURI, nil, request.SkipVerify, request.SkipDefaultPgp, request.PgpBytes...) + err := s.coord.Upgrade(ctx, request.Version, request.Rollback, request.SourceURI, nil, request.SkipVerify, request.SkipDefaultPgp, request.PgpBytes...) if err != nil { //nolint:nilerr // ignore the error, return a failure upgrade response return &cproto.UpgradeResponse{ diff --git a/pkg/core/process/process.go b/pkg/core/process/process.go index 97755a63480..99c1be9fba5 100644 --- a/pkg/core/process/process.go +++ b/pkg/core/process/process.go @@ -18,6 +18,7 @@ type Info struct { Process *os.Process Stdin io.WriteCloser Stderr io.ReadCloser + Cmd *exec.Cmd } // CmdOption is an option func to change the underlying command @@ -170,5 +171,11 @@ func startContext(ctx context.Context, path string, uid, gid int, args []string, Process: cmd.Process, Stdin: stdin, Stderr: stderr, + Cmd: cmd, }, err } + +// Terminate is a utility function to gracefully shutdown a process +func Terminate(proc *os.Process) error { + return terminateCmd(proc) +} diff --git a/sonar-project.properties b/sonar-project.properties index d9486a3cd52..cc7181a1d95 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -5,6 +5,7 @@ sonar.sources=. sonar.exclusions=.git/**, dev-tools/**, /magefile.go, changelog/**, \ _meta/**, deploy/**, docs/**, img/**, specs/**, \ */*_test.go, pkg/testing/**, pkg/component/fake/**, testing/**, **/mocks/*.go, \ + internal/pkg/agent/application/filelock/testlocker/**, \ pkg/control/v1/proto/*.pb.go, pkg/control/v2/cproto/*.pb.go sonar.tests=. sonar.test.inclusions=**/*_test.go diff --git a/testing/integration/ess/repackage.go b/testing/integration/ess/repackage.go index 3e21c36f8f7..c042139e0c2 100644 --- a/testing/integration/ess/repackage.go +++ b/testing/integration/ess/repackage.go @@ -11,7 +11,6 @@ import ( "archive/zip" "bytes" "compress/gzip" - "context" "errors" "io" "os" @@ -25,19 +24,13 @@ import ( "github.com/elastic/elastic-agent/dev-tools/mage" v1 "github.com/elastic/elastic-agent/pkg/api/v1" - atesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/version" agtversion "github.com/elastic/elastic-agent/version" ) -func repackageArchive(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, newVersionBuildMetadata string, currentVersion *version.ParsedSemVer, newPackageContainingDir string, parsedNewVersion *version.ParsedSemVer) (*version.ParsedSemVer, error) { - err := startFixture.EnsurePrepared(ctx) - require.NoErrorf(t, err, "fixture should be prepared") - - // retrieve the compressed package file location - srcPackage, err := startFixture.SrcPackage(ctx) - require.NoErrorf(t, err, "error retrieving start fixture source package") - +// repackageArchive will take a srcPackage elastic-agent package and create a modified copy that will present parsedNewVersion +// in package version file, manifest and relevant metadata. +func repackageArchive(t *testing.T, srcPackage string, newVersionBuildMetadata string, currentVersion *version.ParsedSemVer, parsedNewVersion *version.ParsedSemVer) (*version.ParsedSemVer, string, error) { originalPackageFileName := filepath.Base(srcPackage) // integration test fixtures and package names treat the version as a string including the "-SNAPSHOT" suffix @@ -54,8 +47,8 @@ func repackageArchive(ctx context.Context, t *testing.T, startFixture *atesting. // calculate the new package name newPackageFileName := strings.Replace(originalPackageFileName, currentVersion.String(), versionForFixture.String(), 1) t.Logf("originalPackageName: %q newPackageFileName: %q", originalPackageFileName, newPackageFileName) - - newPackageAbsPath := filepath.Join(newPackageContainingDir, newPackageFileName) + outDir := t.TempDir() + newPackageAbsPath := filepath.Join(outDir, newPackageFileName) // hack the package based on type ext := filepath.Ext(originalPackageFileName) @@ -76,9 +69,9 @@ func repackageArchive(ctx context.Context, t *testing.T, startFixture *atesting. } // Create hash file for the new package - err = mage.CreateSHA512File(newPackageAbsPath) + err := mage.CreateSHA512File(newPackageAbsPath) require.NoErrorf(t, err, "error creating .sha512 for file %q", newPackageAbsPath) - return versionForFixture, err + return versionForFixture, newPackageAbsPath, err } func repackageTarArchive(t *testing.T, srcPackagePath string, newPackagePath string, newVersion *version.ParsedSemVer) { diff --git a/testing/integration/ess/upgrade_rollback_test.go b/testing/integration/ess/upgrade_rollback_test.go index 6207639c455..7519740e122 100644 --- a/testing/integration/ess/upgrade_rollback_test.go +++ b/testing/integration/ess/upgrade_rollback_test.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "runtime" "strings" "testing" @@ -24,6 +25,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" "github.com/elastic/elastic-agent/internal/pkg/agent/install" + "github.com/elastic/elastic-agent/pkg/control/v2/client" "github.com/elastic/elastic-agent/pkg/control/v2/cproto" atesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" @@ -39,6 +41,15 @@ agent.upgrade.watcher: error_check.interval: 5s ` +const fastWatcherCfgWithRollbackWindow = ` +agent.upgrade: + watcher: + grace_period: 2m + error_check.interval: 5s + rollback: + window: 10m +` + // TestStandaloneUpgradeRollback tests the scenario where upgrading to a new version // of Agent fails due to the new Agent binary reporting an unhealthy status. It checks // that the Agent is rolled back to the previous version. @@ -218,7 +229,6 @@ func TestStandaloneUpgradeRollbackOnRestarts(t *testing.T) { atesting.WithFetcher(atesting.ArtifactFetcher()), ) require.NoError(t, err) - return fromFixture, toFixture }, }, @@ -232,8 +242,6 @@ func TestStandaloneUpgradeRollbackOnRestarts(t *testing.T) { require.NoError(t, err) // Create a new package with a different version (IAR-style) - newPackageContainingDir := t.TempDir() - // modify the version with the "+buildYYYYMMDDHHMMSS" currentVersion, err := version.ParseVersion(define.Version()) require.NoErrorf(t, err, "define.Version() %q is not parsable.", define.Version()) @@ -241,12 +249,19 @@ func TestStandaloneUpgradeRollbackOnRestarts(t *testing.T) { newVersionBuildMetadata := "build" + time.Now().Format("20060102150405") parsedNewVersion := version.NewParsedSemVer(currentVersion.Major(), currentVersion.Minor(), currentVersion.Patch(), "", newVersionBuildMetadata) - versionForFixture, err := repackageArchive(t.Context(), t, fromFixture, newVersionBuildMetadata, currentVersion, newPackageContainingDir, parsedNewVersion) + err = fromFixture.EnsurePrepared(t.Context()) + require.NoErrorf(t, err, "fixture should be prepared") + + // retrieve the compressed package file location + srcPackage, err := fromFixture.SrcPackage(t.Context()) + require.NoErrorf(t, err, "error retrieving start fixture source package") + + versionForFixture, repackagedArchiveFile, err := repackageArchive(t, srcPackage, newVersionBuildMetadata, currentVersion, parsedNewVersion) require.NoError(t, err, "error repackaging the archive built from the same commit") // I wish I could just pass the location of the package on disk to the whole upgrade tests/fixture/fetcher code // but I would have to break too much code for that, when in Rome... add more code on top of inflexible code - repackagedLocalFetcher := atesting.LocalFetcher(newPackageContainingDir) + repackagedLocalFetcher := atesting.LocalFetcher(filepath.Dir(repackagedArchiveFile)) toFixture, err := atesting.NewFixture( t, versionForFixture.String(), @@ -323,6 +338,79 @@ func TestFleetManagedUpgradeRollbackOnRestarts(t *testing.T) { } } +// TestStandaloneUpgradeManualRollback tests the scenario where, after upgrading to a new version +// of Agent, a manual rollback is triggered. It checks that the Agent is rolled back to the previous version. +func TestStandaloneUpgradeManualRollback(t *testing.T) { + define.Require(t, define.Requirements{ + Group: integration.Upgrade, + Local: false, // requires Agent installation + Sudo: true, // requires Agent installation + }) + + type fixturesSetupFunc func(t *testing.T) (from *atesting.Fixture, to *atesting.Fixture) + testcases := []struct { + name string + fixturesSetup fixturesSetupFunc + }{ + { + name: "upgrade to a repackaged agent built from the same commit", + fixturesSetup: func(t *testing.T) (from *atesting.Fixture, to *atesting.Fixture) { + // Upgrade from the current build to the same build as Independent Agent Release. + + // Start from the build under test. + fromFixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) + require.NoError(t, err) + + // Create a new package with a different version (IAR-style) + // modify the version with the "+buildYYYYMMDDHHMMSS" + currentVersion, err := version.ParseVersion(define.Version()) + require.NoErrorf(t, err, "define.Version() %q is not parsable.", define.Version()) + + newVersionBuildMetadata := "build" + time.Now().Format("20060102150405") + parsedNewVersion := version.NewParsedSemVer(currentVersion.Major(), currentVersion.Minor(), currentVersion.Patch(), "", newVersionBuildMetadata) + + err = fromFixture.EnsurePrepared(t.Context()) + require.NoErrorf(t, err, "fixture should be prepared") + + // retrieve the compressed package file location + srcPackage, err := fromFixture.SrcPackage(t.Context()) + require.NoErrorf(t, err, "error retrieving start fixture source package") + + versionForFixture, repackagedArchiveFile, err := repackageArchive(t, srcPackage, newVersionBuildMetadata, currentVersion, parsedNewVersion) + require.NoError(t, err, "error repackaging the archive built from the same commit") + + repackagedLocalFetcher := atesting.LocalFetcher(filepath.Dir(repackagedArchiveFile)) + toFixture, err := atesting.NewFixture( + t, + versionForFixture.String(), + atesting.WithFetcher(repackagedLocalFetcher), + ) + require.NoError(t, err) + + return fromFixture, toFixture + }, + }, + } + + // set up start ficture with a rollback window of 1h + rollbackWindowConfig := ` +agent.upgrade.rollback.window: 1h +` + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := testcontext.WithDeadline(t, t.Context(), time.Now().Add(10*time.Minute)) + defer cancel() + from, to := tc.fixturesSetup(t) + + err := from.Configure(ctx, []byte(rollbackWindowConfig)) + require.NoError(t, err, "error setting up rollback window") + standaloneManualRollbackTest(ctx, t, from, to) + }) + } + +} + func managedRollbackRestartTest(ctx context.Context, t *testing.T, info *define.Info, from *atesting.Fixture, to *atesting.Fixture) { startVersionInfo, err := from.ExecVersion(ctx) @@ -401,11 +489,29 @@ func managedRollbackRestartTest(ctx context.Context, t *testing.T, info *define. } func standaloneRollbackRestartTest(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, endFixture *atesting.Fixture) { + standaloneRollbackTest(ctx, t, startFixture, endFixture, reallyFastWatcherCfg, details.ReasonWatchFailed, + func(t *testing.T, _ client.Client) { + restartAgentNTimes(t, 3, 10*time.Second) + }) +} + +func standaloneManualRollbackTest(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, endFixture *atesting.Fixture) { + standaloneRollbackTest(ctx, t, startFixture, endFixture, fastWatcherCfgWithRollbackWindow, details.ReasonManualRollback, + func(t *testing.T, client client.Client) { + t.Logf("sending version=%s rollback=%v upgrade to agent", startFixture.Version(), true) + retVal, err := client.Upgrade(ctx, startFixture.Version(), true, "", false, false) + require.NoError(t, err, "error triggering manual rollback to version %s", startFixture.Version()) + t.Logf("received output %s from upgrade command", retVal) + }, + ) +} + +func standaloneRollbackTest(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, endFixture *atesting.Fixture, customConfig string, rollbackReason string, rollbackTrigger func(t *testing.T, client client.Client)) { startVersionInfo, err := startFixture.ExecVersion(ctx) require.NoError(t, err, "failed to get start agent build version info") - endVersionInfo, err := startFixture.ExecVersion(ctx) + endVersionInfo, err := endFixture.ExecVersion(ctx) require.NoError(t, err, "failed to get end agent build version info") t.Logf("Testing Elastic Agent upgrade from %s to %s...", startFixture.Version(), endVersionInfo.Binary.String()) @@ -420,15 +526,20 @@ func standaloneRollbackRestartTest(ctx context.Context, t *testing.T, startFixtu err = upgradetest.PerformUpgrade( ctx, startFixture, endFixture, t, upgradetest.WithPostUpgradeHook(postUpgradeHook), - upgradetest.WithCustomWatcherConfig(reallyFastWatcherCfg), - upgradetest.WithDisableHashCheck(true)) + upgradetest.WithCustomWatcherConfig(customConfig), + upgradetest.WithDisableHashCheck(true), + ) if !errors.Is(err, ErrPostExit) { require.NoError(t, err) } - // A few seconds after the upgrade, deliberately restart upgraded Agent a - // couple of times to simulate Agent crashing. - restartAgentNTimes(t, 3, 10*time.Second) + elasticAgentClient := startFixture.Client() + err = elasticAgentClient.Connect(ctx) + require.NoError(t, err, "error connecting to installed elastic agent") + defer elasticAgentClient.Disconnect() + + // A few seconds after the upgrade, trigger a rollback using the passed trigger + rollbackTrigger(t, elasticAgentClient) // wait for the agent to be healthy and back at the start version err = upgradetest.WaitHealthyAndVersion(ctx, startFixture, startVersionInfo.Binary, 2*time.Minute, 10*time.Second, t) @@ -448,17 +559,15 @@ func standaloneRollbackRestartTest(ctx context.Context, t *testing.T, startFixtu require.NoError(t, err) if !startVersion.Less(*version.NewParsedSemVer(8, 12, 0, "", "")) { - client := startFixture.Client() - err = client.Connect(ctx) require.NoError(t, err) - state, err := client.State(ctx) + state, err := elasticAgentClient.State(ctx) require.NoError(t, err) require.NotNil(t, state.UpgradeDetails) assert.Equal(t, details.StateRollback, details.State(state.UpgradeDetails.State)) if !startVersion.Less(*upgradetest.Version_9_2_0_SNAPSHOT) { - assert.Equal(t, details.ReasonWatchFailed, state.UpgradeDetails.Metadata.Reason) + assert.Equal(t, rollbackReason, state.UpgradeDetails.Metadata.Reason) } } diff --git a/testing/integration/ess/upgrade_standalone_same_commit_test.go b/testing/integration/ess/upgrade_standalone_same_commit_test.go index 48e6e5993f6..cf4f854b7c8 100644 --- a/testing/integration/ess/upgrade_standalone_same_commit_test.go +++ b/testing/integration/ess/upgrade_standalone_same_commit_test.go @@ -9,6 +9,7 @@ package ess import ( "context" "fmt" + "path/filepath" "testing" "time" @@ -82,12 +83,16 @@ func TestStandaloneUpgradeSameCommit(t *testing.T) { newVersionBuildMetadata := "build" + time.Now().Format("20060102150405") parsedNewVersion := version.NewParsedSemVer(currentVersion.Major(), currentVersion.Minor(), currentVersion.Patch(), "", newVersionBuildMetadata) - newPackageContainingDir := t.TempDir() + err = startFixture.EnsurePrepared(t.Context()) + require.NoErrorf(t, err, "fixture should be prepared") - versionForFixture, err := repackageArchive(ctx, t, startFixture, newVersionBuildMetadata, currentVersion, newPackageContainingDir, parsedNewVersion) + // retrieve the compressed package file location + srcPackage, err := startFixture.SrcPackage(t.Context()) + require.NoErrorf(t, err, "error retrieving start fixture source package") - // I wish I could just pass the location of the package on disk to the whole upgrade tests/fixture/fetcher code - // but I would have to break too much code for that, when in Rome... add more code on top of inflexible code + versionForFixture, repackagedArchiveFile, err := repackageArchive(t, srcPackage, newVersionBuildMetadata, currentVersion, parsedNewVersion) + + newPackageContainingDir := filepath.Dir(repackagedArchiveFile) repackagedLocalFetcher := atesting.LocalFetcher(newPackageContainingDir) endFixture, err := atesting.NewFixture(t, versionForFixture.String(), atesting.WithFetcher(repackagedLocalFetcher)) diff --git a/testing/mocks/internal_/pkg/agent/cmd/installation_modifier_mock.go b/testing/mocks/internal_/pkg/agent/cmd/installation_modifier_mock.go deleted file mode 100644 index 14f73d912a8..00000000000 --- a/testing/mocks/internal_/pkg/agent/cmd/installation_modifier_mock.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -// Code generated by mockery v2.53.4. DO NOT EDIT. - -package cmd - -import ( - client "github.com/elastic/elastic-agent/pkg/control/v2/client" - - context "context" - - logp "github.com/elastic/elastic-agent-libs/logp" - - mock "github.com/stretchr/testify/mock" -) - -// InstallationModifier is an autogenerated mock type for the installationModifier type -type InstallationModifier struct { - mock.Mock -} - -type InstallationModifier_Expecter struct { - mock *mock.Mock -} - -func (_m *InstallationModifier) EXPECT() *InstallationModifier_Expecter { - return &InstallationModifier_Expecter{mock: &_m.Mock} -} - -// Cleanup provides a mock function with given fields: log, topDirPath, currentVersionedHome, currentHash, removeMarker, keepLogs -func (_m *InstallationModifier) Cleanup(log *logp.Logger, topDirPath string, currentVersionedHome string, currentHash string, removeMarker bool, keepLogs bool) error { - ret := _m.Called(log, topDirPath, currentVersionedHome, currentHash, removeMarker, keepLogs) - - if len(ret) == 0 { - panic("no return value specified for Cleanup") - } - - var r0 error - if rf, ok := ret.Get(0).(func(*logp.Logger, string, string, string, bool, bool) error); ok { - r0 = rf(log, topDirPath, currentVersionedHome, currentHash, removeMarker, keepLogs) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// InstallationModifier_Cleanup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Cleanup' -type InstallationModifier_Cleanup_Call struct { - *mock.Call -} - -// Cleanup is a helper method to define mock.On call -// - log *logp.Logger -// - topDirPath string -// - currentVersionedHome string -// - currentHash string -// - removeMarker bool -// - keepLogs bool -func (_e *InstallationModifier_Expecter) Cleanup(log interface{}, topDirPath interface{}, currentVersionedHome interface{}, currentHash interface{}, removeMarker interface{}, keepLogs interface{}) *InstallationModifier_Cleanup_Call { - return &InstallationModifier_Cleanup_Call{Call: _e.mock.On("Cleanup", log, topDirPath, currentVersionedHome, currentHash, removeMarker, keepLogs)} -} - -func (_c *InstallationModifier_Cleanup_Call) Run(run func(log *logp.Logger, topDirPath string, currentVersionedHome string, currentHash string, removeMarker bool, keepLogs bool)) *InstallationModifier_Cleanup_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*logp.Logger), args[1].(string), args[2].(string), args[3].(string), args[4].(bool), args[5].(bool)) - }) - return _c -} - -func (_c *InstallationModifier_Cleanup_Call) Return(_a0 error) *InstallationModifier_Cleanup_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *InstallationModifier_Cleanup_Call) RunAndReturn(run func(*logp.Logger, string, string, string, bool, bool) error) *InstallationModifier_Cleanup_Call { - _c.Call.Return(run) - return _c -} - -// Rollback provides a mock function with given fields: ctx, log, c, topDirPath, prevVersionedHome, prevHash -func (_m *InstallationModifier) Rollback(ctx context.Context, log *logp.Logger, c client.Client, topDirPath string, prevVersionedHome string, prevHash string) error { - ret := _m.Called(ctx, log, c, topDirPath, prevVersionedHome, prevHash) - - if len(ret) == 0 { - panic("no return value specified for Rollback") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *logp.Logger, client.Client, string, string, string) error); ok { - r0 = rf(ctx, log, c, topDirPath, prevVersionedHome, prevHash) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// InstallationModifier_Rollback_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Rollback' -type InstallationModifier_Rollback_Call struct { - *mock.Call -} - -// Rollback is a helper method to define mock.On call -// - ctx context.Context -// - log *logp.Logger -// - c client.Client -// - topDirPath string -// - prevVersionedHome string -// - prevHash string -func (_e *InstallationModifier_Expecter) Rollback(ctx interface{}, log interface{}, c interface{}, topDirPath interface{}, prevVersionedHome interface{}, prevHash interface{}) *InstallationModifier_Rollback_Call { - return &InstallationModifier_Rollback_Call{Call: _e.mock.On("Rollback", ctx, log, c, topDirPath, prevVersionedHome, prevHash)} -} - -func (_c *InstallationModifier_Rollback_Call) Run(run func(ctx context.Context, log *logp.Logger, c client.Client, topDirPath string, prevVersionedHome string, prevHash string)) *InstallationModifier_Rollback_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*logp.Logger), args[2].(client.Client), args[3].(string), args[4].(string), args[5].(string)) - }) - return _c -} - -func (_c *InstallationModifier_Rollback_Call) Return(_a0 error) *InstallationModifier_Rollback_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *InstallationModifier_Rollback_Call) RunAndReturn(run func(context.Context, *logp.Logger, client.Client, string, string, string) error) *InstallationModifier_Rollback_Call { - _c.Call.Return(run) - return _c -} - -// NewInstallationModifier creates a new instance of InstallationModifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewInstallationModifier(t interface { - mock.TestingT - Cleanup(func()) -}) *InstallationModifier { - mock := &InstallationModifier{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/testing/mocks/pkg/control/v2/client/client_mock.go b/testing/mocks/pkg/control/v2/client/client_mock.go index c408984634b..967306af978 100644 --- a/testing/mocks/pkg/control/v2/client/client_mock.go +++ b/testing/mocks/pkg/control/v2/client/client_mock.go @@ -539,14 +539,14 @@ func (_c *Client_StateWatch_Call) RunAndReturn(run func(context.Context) (client return _c } -// Upgrade provides a mock function with given fields: ctx, version, sourceURI, skipVerify, skipDefaultPgp, pgpBytes -func (_m *Client) Upgrade(ctx context.Context, version string, sourceURI string, skipVerify bool, skipDefaultPgp bool, pgpBytes ...string) (string, error) { +// Upgrade provides a mock function with given fields: ctx, version, rollback, sourceURI, skipVerify, skipDefaultPgp, pgpBytes +func (_m *Client) Upgrade(ctx context.Context, version string, rollback bool, sourceURI string, skipVerify bool, skipDefaultPgp bool, pgpBytes ...string) (string, error) { _va := make([]interface{}, len(pgpBytes)) for _i := range pgpBytes { _va[_i] = pgpBytes[_i] } var _ca []interface{} - _ca = append(_ca, ctx, version, sourceURI, skipVerify, skipDefaultPgp) + _ca = append(_ca, ctx, version, rollback, sourceURI, skipVerify, skipDefaultPgp) _ca = append(_ca, _va...) ret := _m.Called(_ca...) @@ -556,17 +556,17 @@ func (_m *Client) Upgrade(ctx context.Context, version string, sourceURI string, var r0 string var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, bool, bool, ...string) (string, error)); ok { - return rf(ctx, version, sourceURI, skipVerify, skipDefaultPgp, pgpBytes...) + if rf, ok := ret.Get(0).(func(context.Context, string, bool, string, bool, bool, ...string) (string, error)); ok { + return rf(ctx, version, rollback, sourceURI, skipVerify, skipDefaultPgp, pgpBytes...) } - if rf, ok := ret.Get(0).(func(context.Context, string, string, bool, bool, ...string) string); ok { - r0 = rf(ctx, version, sourceURI, skipVerify, skipDefaultPgp, pgpBytes...) + if rf, ok := ret.Get(0).(func(context.Context, string, bool, string, bool, bool, ...string) string); ok { + r0 = rf(ctx, version, rollback, sourceURI, skipVerify, skipDefaultPgp, pgpBytes...) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(context.Context, string, string, bool, bool, ...string) error); ok { - r1 = rf(ctx, version, sourceURI, skipVerify, skipDefaultPgp, pgpBytes...) + if rf, ok := ret.Get(1).(func(context.Context, string, bool, string, bool, bool, ...string) error); ok { + r1 = rf(ctx, version, rollback, sourceURI, skipVerify, skipDefaultPgp, pgpBytes...) } else { r1 = ret.Error(1) } @@ -582,24 +582,25 @@ type Client_Upgrade_Call struct { // Upgrade is a helper method to define mock.On call // - ctx context.Context // - version string +// - rollback bool // - sourceURI string // - skipVerify bool // - skipDefaultPgp bool // - pgpBytes ...string -func (_e *Client_Expecter) Upgrade(ctx interface{}, version interface{}, sourceURI interface{}, skipVerify interface{}, skipDefaultPgp interface{}, pgpBytes ...interface{}) *Client_Upgrade_Call { +func (_e *Client_Expecter) Upgrade(ctx interface{}, version interface{}, rollback interface{}, sourceURI interface{}, skipVerify interface{}, skipDefaultPgp interface{}, pgpBytes ...interface{}) *Client_Upgrade_Call { return &Client_Upgrade_Call{Call: _e.mock.On("Upgrade", - append([]interface{}{ctx, version, sourceURI, skipVerify, skipDefaultPgp}, pgpBytes...)...)} + append([]interface{}{ctx, version, rollback, sourceURI, skipVerify, skipDefaultPgp}, pgpBytes...)...)} } -func (_c *Client_Upgrade_Call) Run(run func(ctx context.Context, version string, sourceURI string, skipVerify bool, skipDefaultPgp bool, pgpBytes ...string)) *Client_Upgrade_Call { +func (_c *Client_Upgrade_Call) Run(run func(ctx context.Context, version string, rollback bool, sourceURI string, skipVerify bool, skipDefaultPgp bool, pgpBytes ...string)) *Client_Upgrade_Call { _c.Call.Run(func(args mock.Arguments) { - variadicArgs := make([]string, len(args)-5) - for i, a := range args[5:] { + variadicArgs := make([]string, len(args)-6) + for i, a := range args[6:] { if a != nil { variadicArgs[i] = a.(string) } } - run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(bool), args[4].(bool), variadicArgs...) + run(args[0].(context.Context), args[1].(string), args[2].(bool), args[3].(string), args[4].(bool), args[5].(bool), variadicArgs...) }) return _c } @@ -609,7 +610,7 @@ func (_c *Client_Upgrade_Call) Return(_a0 string, _a1 error) *Client_Upgrade_Cal return _c } -func (_c *Client_Upgrade_Call) RunAndReturn(run func(context.Context, string, string, bool, bool, ...string) (string, error)) *Client_Upgrade_Call { +func (_c *Client_Upgrade_Call) RunAndReturn(run func(context.Context, string, bool, string, bool, bool, ...string) (string, error)) *Client_Upgrade_Call { _c.Call.Return(run) return _c }