Skip to content

Commit 64c5b4c

Browse files
pchilapchila
committed
Add manual rollback integration test
1 parent bcd4e79 commit 64c5b4c

File tree

3 files changed

+138
-31
lines changed

3 files changed

+138
-31
lines changed

‎testing/integration/ess/repackage.go

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
"archive/zip"
1212
"bytes"
1313
"compress/gzip"
14-
"context"
1514
"errors"
1615
"io"
1716
"os"
@@ -25,19 +24,13 @@ import (
2524

2625
"github.com/elastic/elastic-agent/dev-tools/mage"
2726
v1 "github.com/elastic/elastic-agent/pkg/api/v1"
28-
atesting "github.com/elastic/elastic-agent/pkg/testing"
2927
"github.com/elastic/elastic-agent/pkg/version"
3028
agtversion "github.com/elastic/elastic-agent/version"
3129
)
3230

33-
func repackageArchive(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, newVersionBuildMetadata string, currentVersion *version.ParsedSemVer, newPackageContainingDir string, parsedNewVersion *version.ParsedSemVer) (*version.ParsedSemVer, error) {
34-
err := startFixture.EnsurePrepared(ctx)
35-
require.NoErrorf(t, err, "fixture should be prepared")
36-
37-
// retrieve the compressed package file location
38-
srcPackage, err := startFixture.SrcPackage(ctx)
39-
require.NoErrorf(t, err, "error retrieving start fixture source package")
40-
31+
// repackageArchive will take a srcPackage elastic-agent package and create a modified copy that will present parsedNewVersion
32+
// in package version file, manifest and relevant metadata.
33+
func repackageArchive(t *testing.T, srcPackage string, newVersionBuildMetadata string, currentVersion *version.ParsedSemVer, parsedNewVersion *version.ParsedSemVer) (*version.ParsedSemVer, string, error) {
4134
originalPackageFileName := filepath.Base(srcPackage)
4235

4336
// integration test fixtures and package names treat the version as a string including the "-SNAPSHOT" suffix
@@ -54,8 +47,8 @@ func repackageArchive(ctx context.Context, t *testing.T, startFixture *atesting.
5447
// calculate the new package name
5548
newPackageFileName := strings.Replace(originalPackageFileName, currentVersion.String(), versionForFixture.String(), 1)
5649
t.Logf("originalPackageName: %q newPackageFileName: %q", originalPackageFileName, newPackageFileName)
57-
58-
newPackageAbsPath := filepath.Join(newPackageContainingDir, newPackageFileName)
50+
outDir := t.TempDir()
51+
newPackageAbsPath := filepath.Join(outDir, newPackageFileName)
5952

6053
// hack the package based on type
6154
ext := filepath.Ext(originalPackageFileName)
@@ -76,9 +69,9 @@ func repackageArchive(ctx context.Context, t *testing.T, startFixture *atesting.
7669
}
7770

7871
// Create hash file for the new package
79-
err = mage.CreateSHA512File(newPackageAbsPath)
72+
err := mage.CreateSHA512File(newPackageAbsPath)
8073
require.NoErrorf(t, err, "error creating .sha512 for file %q", newPackageAbsPath)
81-
return versionForFixture, err
74+
return versionForFixture, newPackageAbsPath, err
8275
}
8376

8477
func repackageTarArchive(t *testing.T, srcPackagePath string, newPackagePath string, newVersion *version.ParsedSemVer) {

‎testing/integration/ess/upgrade_rollback_test.go

Lines changed: 122 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"context"
1111
"errors"
1212
"fmt"
13+
"path/filepath"
1314
"runtime"
1415
"strings"
1516
"testing"
@@ -24,6 +25,7 @@ import (
2425
"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
2526
"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details"
2627
"github.com/elastic/elastic-agent/internal/pkg/agent/install"
28+
"github.com/elastic/elastic-agent/pkg/control/v2/client"
2729
"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
2830
atesting "github.com/elastic/elastic-agent/pkg/testing"
2931
"github.com/elastic/elastic-agent/pkg/testing/define"
@@ -39,6 +41,15 @@ agent.upgrade.watcher:
3941
error_check.interval: 5s
4042
`
4143

44+
const fastWatcherCfgWithRollbackWindow = `
45+
agent.upgrade:
46+
watcher:
47+
grace_period: 2m
48+
error_check.interval: 5s
49+
rollback:
50+
window: 10m
51+
`
52+
4253
// TestStandaloneUpgradeRollback tests the scenario where upgrading to a new version
4354
// of Agent fails due to the new Agent binary reporting an unhealthy status. It checks
4455
// that the Agent is rolled back to the previous version.
@@ -232,21 +243,26 @@ func TestStandaloneUpgradeRollbackOnRestarts(t *testing.T) {
232243
require.NoError(t, err)
233244

234245
// Create a new package with a different version (IAR-style)
235-
newPackageContainingDir := t.TempDir()
236-
237246
// modify the version with the "+buildYYYYMMDDHHMMSS"
238247
currentVersion, err := version.ParseVersion(define.Version())
239248
require.NoErrorf(t, err, "define.Version() %q is not parsable.", define.Version())
240249

241250
newVersionBuildMetadata := "build" + time.Now().Format("20060102150405")
242251
parsedNewVersion := version.NewParsedSemVer(currentVersion.Major(), currentVersion.Minor(), currentVersion.Patch(), "", newVersionBuildMetadata)
243252

244-
versionForFixture, err := repackageArchive(t.Context(), t, fromFixture, newVersionBuildMetadata, currentVersion, newPackageContainingDir, parsedNewVersion)
253+
err = fromFixture.EnsurePrepared(t.Context())
254+
require.NoErrorf(t, err, "fixture should be prepared")
255+
256+
// retrieve the compressed package file location
257+
srcPackage, err := fromFixture.SrcPackage(t.Context())
258+
require.NoErrorf(t, err, "error retrieving start fixture source package")
259+
260+
versionForFixture, repackagedArchiveFile, err := repackageArchive(t, srcPackage, newVersionBuildMetadata, currentVersion, parsedNewVersion)
245261
require.NoError(t, err, "error repackaging the archive built from the same commit")
246262

247263
// I wish I could just pass the location of the package on disk to the whole upgrade tests/fixture/fetcher code
248264
// but I would have to break too much code for that, when in Rome... add more code on top of inflexible code
249-
repackagedLocalFetcher := atesting.LocalFetcher(newPackageContainingDir)
265+
repackagedLocalFetcher := atesting.LocalFetcher(filepath.Dir(repackagedArchiveFile))
250266
toFixture, err := atesting.NewFixture(
251267
t,
252268
versionForFixture.String(),
@@ -323,6 +339,79 @@ func TestFleetManagedUpgradeRollbackOnRestarts(t *testing.T) {
323339
}
324340
}
325341

342+
// TestStandaloneUpgradeManualRollback tests the scenario where, after upgrading to a new version
343+
// of Agent, a manual rollback is triggered. It checks that the Agent is rolled back to the previous version.
344+
func TestStandaloneUpgradeManualRollback(t *testing.T) {
345+
define.Require(t, define.Requirements{
346+
Group: integration.Upgrade,
347+
Local: false, // requires Agent installation
348+
Sudo: true, // requires Agent installation
349+
})
350+
351+
type fixturesSetupFunc func(t *testing.T) (from *atesting.Fixture, to *atesting.Fixture)
352+
testcases := []struct {
353+
name string
354+
fixturesSetup fixturesSetupFunc
355+
}{
356+
{
357+
name: "upgrade to a repackaged agent built from the same commit",
358+
fixturesSetup: func(t *testing.T) (from *atesting.Fixture, to *atesting.Fixture) {
359+
// Upgrade from the current build to the same build as Independent Agent Release.
360+
361+
// Start from the build under test.
362+
fromFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
363+
require.NoError(t, err)
364+
365+
// Create a new package with a different version (IAR-style)
366+
// modify the version with the "+buildYYYYMMDDHHMMSS"
367+
currentVersion, err := version.ParseVersion(define.Version())
368+
require.NoErrorf(t, err, "define.Version() %q is not parsable.", define.Version())
369+
370+
newVersionBuildMetadata := "build" + time.Now().Format("20060102150405")
371+
parsedNewVersion := version.NewParsedSemVer(currentVersion.Major(), currentVersion.Minor(), currentVersion.Patch(), "", newVersionBuildMetadata)
372+
373+
err = fromFixture.EnsurePrepared(t.Context())
374+
require.NoErrorf(t, err, "fixture should be prepared")
375+
376+
// retrieve the compressed package file location
377+
srcPackage, err := fromFixture.SrcPackage(t.Context())
378+
require.NoErrorf(t, err, "error retrieving start fixture source package")
379+
380+
versionForFixture, repackagedArchiveFile, err := repackageArchive(t, srcPackage, newVersionBuildMetadata, currentVersion, parsedNewVersion)
381+
require.NoError(t, err, "error repackaging the archive built from the same commit")
382+
383+
repackagedLocalFetcher := atesting.LocalFetcher(filepath.Dir(repackagedArchiveFile))
384+
toFixture, err := atesting.NewFixture(
385+
t,
386+
versionForFixture.String(),
387+
atesting.WithFetcher(repackagedLocalFetcher),
388+
)
389+
require.NoError(t, err)
390+
391+
return fromFixture, toFixture
392+
},
393+
},
394+
}
395+
396+
// set up start ficture with a rollback window of 1h
397+
rollbackWindowConfig := `
398+
agent.upgrade.rollback.window: 1h
399+
`
400+
401+
for _, tc := range testcases {
402+
t.Run(tc.name, func(t *testing.T) {
403+
ctx, cancel := testcontext.WithDeadline(t, t.Context(), time.Now().Add(10*time.Minute))
404+
defer cancel()
405+
from, to := tc.fixturesSetup(t)
406+
407+
err := from.Configure(ctx, []byte(rollbackWindowConfig))
408+
require.NoError(t, err, "error setting up rollback window")
409+
standaloneManualRollbackTest(ctx, t, from, to)
410+
})
411+
}
412+
413+
}
414+
326415
func managedRollbackRestartTest(ctx context.Context, t *testing.T, info *define.Info, from *atesting.Fixture, to *atesting.Fixture) {
327416

328417
startVersionInfo, err := from.ExecVersion(ctx)
@@ -401,11 +490,29 @@ func managedRollbackRestartTest(ctx context.Context, t *testing.T, info *define.
401490
}
402491

403492
func standaloneRollbackRestartTest(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, endFixture *atesting.Fixture) {
493+
standaloneRollbackTest(ctx, t, startFixture, endFixture, reallyFastWatcherCfg, details.ReasonWatchFailed,
494+
func(t *testing.T, _ client.Client) {
495+
restartAgentNTimes(t, 3, 10*time.Second)
496+
})
497+
}
498+
499+
func standaloneManualRollbackTest(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, endFixture *atesting.Fixture) {
500+
standaloneRollbackTest(ctx, t, startFixture, endFixture, fastWatcherCfgWithRollbackWindow, details.ReasonManualRollback,
501+
func(t *testing.T, client client.Client) {
502+
t.Logf("sending version=%s rollback=%v upgrade to agent", startFixture.Version(), true)
503+
retVal, err := client.Upgrade(ctx, startFixture.Version(), true, "", false, false)
504+
require.NoError(t, err, "error triggering manual rollback to version %s", startFixture.Version())
505+
t.Logf("received output %s from upgrade command", retVal)
506+
},
507+
)
508+
}
509+
510+
func standaloneRollbackTest(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, endFixture *atesting.Fixture, customConfig string, rollbackReason string, rollbackTrigger func(t *testing.T, client client.Client)) {
404511

405512
startVersionInfo, err := startFixture.ExecVersion(ctx)
406513
require.NoError(t, err, "failed to get start agent build version info")
407514

408-
endVersionInfo, err := startFixture.ExecVersion(ctx)
515+
endVersionInfo, err := endFixture.ExecVersion(ctx)
409516
require.NoError(t, err, "failed to get end agent build version info")
410517

411518
t.Logf("Testing Elastic Agent upgrade from %s to %s...", startFixture.Version(), endVersionInfo.Binary.String())
@@ -420,15 +527,19 @@ func standaloneRollbackRestartTest(ctx context.Context, t *testing.T, startFixtu
420527
err = upgradetest.PerformUpgrade(
421528
ctx, startFixture, endFixture, t,
422529
upgradetest.WithPostUpgradeHook(postUpgradeHook),
423-
upgradetest.WithCustomWatcherConfig(reallyFastWatcherCfg),
530+
upgradetest.WithCustomWatcherConfig(customConfig),
424531
upgradetest.WithDisableHashCheck(true))
425532
if !errors.Is(err, ErrPostExit) {
426533
require.NoError(t, err)
427534
}
428535

429-
// A few seconds after the upgrade, deliberately restart upgraded Agent a
430-
// couple of times to simulate Agent crashing.
431-
restartAgentNTimes(t, 3, 10*time.Second)
536+
elasticAgentClient := startFixture.Client()
537+
err = elasticAgentClient.Connect(ctx)
538+
require.NoError(t, err, "error connecting to installed elastic agent")
539+
defer elasticAgentClient.Disconnect()
540+
541+
// A few seconds after the upgrade, trigger a rollback using the passed trigger
542+
rollbackTrigger(t, elasticAgentClient)
432543

433544
// wait for the agent to be healthy and back at the start version
434545
err = upgradetest.WaitHealthyAndVersion(ctx, startFixture, startVersionInfo.Binary, 2*time.Minute, 10*time.Second, t)
@@ -448,17 +559,15 @@ func standaloneRollbackRestartTest(ctx context.Context, t *testing.T, startFixtu
448559
require.NoError(t, err)
449560

450561
if !startVersion.Less(*version.NewParsedSemVer(8, 12, 0, "", "")) {
451-
client := startFixture.Client()
452-
err = client.Connect(ctx)
453562
require.NoError(t, err)
454563

455-
state, err := client.State(ctx)
564+
state, err := elasticAgentClient.State(ctx)
456565
require.NoError(t, err)
457566

458567
require.NotNil(t, state.UpgradeDetails)
459568
assert.Equal(t, details.StateRollback, details.State(state.UpgradeDetails.State))
460569
if !startVersion.Less(*upgradetest.Version_9_2_0_SNAPSHOT) {
461-
assert.Equal(t, details.ReasonWatchFailed, state.UpgradeDetails.Metadata.Reason)
570+
assert.Equal(t, rollbackReason, state.UpgradeDetails.Metadata.Reason)
462571
}
463572
}
464573

‎testing/integration/ess/upgrade_standalone_same_commit_test.go

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ package ess
99
import (
1010
"context"
1111
"fmt"
12+
"path/filepath"
1213
"testing"
1314
"time"
1415

@@ -82,12 +83,16 @@ func TestStandaloneUpgradeSameCommit(t *testing.T) {
8283
newVersionBuildMetadata := "build" + time.Now().Format("20060102150405")
8384
parsedNewVersion := version.NewParsedSemVer(currentVersion.Major(), currentVersion.Minor(), currentVersion.Patch(), "", newVersionBuildMetadata)
8485

85-
newPackageContainingDir := t.TempDir()
86+
err = startFixture.EnsurePrepared(t.Context())
87+
require.NoErrorf(t, err, "fixture should be prepared")
8688

87-
versionForFixture, err := repackageArchive(ctx, t, startFixture, newVersionBuildMetadata, currentVersion, newPackageContainingDir, parsedNewVersion)
89+
// retrieve the compressed package file location
90+
srcPackage, err := startFixture.SrcPackage(t.Context())
91+
require.NoErrorf(t, err, "error retrieving start fixture source package")
8892

89-
// I wish I could just pass the location of the package on disk to the whole upgrade tests/fixture/fetcher code
90-
// but I would have to break too much code for that, when in Rome... add more code on top of inflexible code
93+
versionForFixture, repackagedArchiveFile, err := repackageArchive(t, srcPackage, newVersionBuildMetadata, currentVersion, parsedNewVersion)
94+
95+
newPackageContainingDir := filepath.Dir(repackagedArchiveFile)
9196
repackagedLocalFetcher := atesting.LocalFetcher(newPackageContainingDir)
9297

9398
endFixture, err := atesting.NewFixture(t, versionForFixture.String(), atesting.WithFetcher(repackagedLocalFetcher))

0 commit comments

Comments
 (0)