Add upgrade test on an air-gapped environment (elastic#3724)

AndersonQ · web-flow · commit 092d205f5490 · 2023-11-27T09:52:17.000-03:00
It uses iptable to block outgoing packets to `artifacts.elastic.co`, which prevents the agent from downloading the upgrade artifact and the PGP key. It uses however a ESS cluster for fleet-server and ES.
diff --git a/internal/pkg/composable/providers/host/host_test.go b/internal/pkg/composable/providers/host/host_test.go
@@ -30,7 +30,7 @@ func TestContextProvider(t *testing.T) {
 	require.NoError(t, err)
 
 	const checkInterval = 50 * time.Millisecond
-	const testTimeout = 500 * time.Millisecond
+	const testTimeout = 1 * time.Second
 	c, err := config.NewConfigFrom(map[string]interface{}{
 		"check_interval": checkInterval,
 	})
diff --git a/pkg/testing/fetcher.go b/pkg/testing/fetcher.go
@@ -20,7 +20,8 @@ import (
 	"github.com/hashicorp/go-multierror"
 )
 
-const hashExt = ".sha512"
+const extAsc = ".asc"
+const extHash = ".sha512"
 
 var (
 	// ErrUnsupportedPlatform returned when the operating system and architecture combination is not supported.
@@ -133,7 +134,7 @@ func untar(archivePath string, extractDir string) error {
 
 		fi := f.FileInfo()
 		mode := fi.Mode()
-		abs := filepath.Join(extractDir, f.Name)
+		abs := filepath.Join(extractDir, f.Name) //nolint:gosec // used only in tests
 		switch {
 		case mode.IsRegular():
 			// just to be sure, it should already be created by Dir type
@@ -146,7 +147,7 @@ func untar(archivePath string, extractDir string) error {
 				return fmt.Errorf("failed creating file %s: %w", abs, err)
 			}
 
-			_, err = io.Copy(wf, tr)
+			_, err = io.Copy(wf, tr) //nolint:gosec // used only in tests
 			if closeErr := wf.Close(); closeErr != nil && err == nil {
 				err = closeErr
 			}
@@ -193,7 +194,7 @@ func unzip(archivePath string, extractDir string) error {
 
 		fi := f.FileInfo()
 		mode := fi.Mode()
-		abs := filepath.Join(extractDir, f.Name)
+		abs := filepath.Join(extractDir, f.Name) //nolint:gosec // used only in tests
 		switch {
 		case mode.IsRegular():
 			// just to be sure, it should already be created by Dir type
@@ -211,6 +212,7 @@ func unzip(archivePath string, extractDir string) error {
 				}
 			}()
 
+			//nolint:gosec // used only in tests
 			if _, err = io.Copy(f, rc); err != nil {
 				return fmt.Errorf("error writing file %s: %w", abs, err)
 			}
diff --git a/pkg/testing/fetcher_artifact.go b/pkg/testing/fetcher_artifact.go
@@ -108,7 +108,13 @@ func (r *artifactResult) Fetch(ctx context.Context, l Logger, dir string) error
 	}
 
 	// fetch package hash
-	err = DownloadPackage(ctx, l, r.doer, r.src+hashExt, filepath.Join(dir, r.path+hashExt))
+	err = DownloadPackage(ctx, l, r.doer, r.src+extHash, filepath.Join(dir, r.path+extHash))
+	if err != nil {
+		return fmt.Errorf("failed to download %s: %w", r.src, err)
+	}
+
+	// fetch package asc
+	err = DownloadPackage(ctx, l, r.doer, r.src+extAsc, filepath.Join(dir, r.path+extAsc))
 	if err != nil {
 		return fmt.Errorf("failed to download %s: %w", r.src, err)
 	}
diff --git a/pkg/testing/fetcher_artifact_test.go b/pkg/testing/fetcher_artifact_test.go
diff --git a/pkg/testing/fetcher_local.go b/pkg/testing/fetcher_local.go
@@ -107,7 +107,7 @@ func (r *localFetcherResult) Fetch(_ context.Context, _ Logger, dir string) erro
 	}
 
 	// fetch artifact hash
-	err = copyFile(fullPath+hashExt, path+hashExt)
+	err = copyFile(fullPath+extHash, path+extHash)
 	if err != nil {
 		return fmt.Errorf("error copying file: %w", err)
 	}
diff --git a/pkg/testing/fetcher_local_test.go b/pkg/testing/fetcher_local_test.go
@@ -35,11 +35,11 @@ func TestLocalFetcher(t *testing.T) {
 
 	snapshotPath := fmt.Sprintf("elastic-agent-%s-SNAPSHOT-%s", baseVersion, suffix)
 	require.NoError(t, os.WriteFile(filepath.Join(testdata, snapshotPath), snapshotContent, 0644))
-	snapshotPathHash := fmt.Sprintf("elastic-agent-%s-SNAPSHOT-%s%s", baseVersion, suffix, hashExt)
+	snapshotPathHash := fmt.Sprintf("elastic-agent-%s-SNAPSHOT-%s%s", baseVersion, suffix, extHash)
 	require.NoError(t, os.WriteFile(filepath.Join(testdata, snapshotPathHash), snapshotContentHash, 0644))
 	notSnapshotPath := fmt.Sprintf("elastic-agent-%s-%s", baseVersion, suffix)
 	require.NoError(t, os.WriteFile(filepath.Join(testdata, notSnapshotPath), noSnapshotContent, 0644))
-	notSnapshotPathHash := fmt.Sprintf("elastic-agent-%s-%s%s", baseVersion, suffix, hashExt)
+	notSnapshotPathHash := fmt.Sprintf("elastic-agent-%s-%s%s", baseVersion, suffix, extHash)
 	require.NoError(t, os.WriteFile(filepath.Join(testdata, notSnapshotPathHash), noSnapshotContentHash, 0644))
 
 	tcs := []struct {
@@ -88,7 +88,7 @@ func TestLocalFetcher(t *testing.T) {
 		require.NoError(t, err)
 
 		assert.Equal(t, string(tc.want), string(content))
-		contentHash, err := os.ReadFile(filepath.Join(tmp, got.Name()+hashExt))
+		contentHash, err := os.ReadFile(filepath.Join(tmp, got.Name()+extHash))
 		require.NoError(t, err)
 
 		assert.Equal(t, string(tc.wantHash), string(contentHash))
diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go
@@ -537,7 +537,7 @@ type ExecErr struct {
 }
 
 func (e *ExecErr) Error() string {
-	return e.err.Error()
+	return e.String()
 }
 
 func (e *ExecErr) String() string {
diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go
@@ -144,7 +144,7 @@ func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ..
 			f.collectDiagnostics()
 		}
 
-		// environment variable AGENT_KEEP_INSTALLED=true will skip the uninstall
+		// environment variable AGENT_KEEP_INSTALLED=true will skip the uninstallation
 		// useful to debug the issue with the Elastic Agent
 		if f.t.Failed() && keepInstalledFlag() {
 			f.t.Logf("skipping uninstall; test failed and AGENT_KEEP_INSTALLED=true")
diff --git a/pkg/testing/multipass/provisioner.go b/pkg/testing/multipass/provisioner.go
@@ -158,14 +158,9 @@ func (p *provisioner) launch(ctx context.Context, cfg runner.Config, batch runne
 		return fmt.Errorf("failed to marshal cloud-init configuration: %w", err)
 	}
 
-	p.logger.Logf("Launching multipass instance %s", batch.ID)
 	var output bytes.Buffer
-	proc, err := process.Start("multipass",
-		process.WithContext(ctx),
-		process.WithArgs(args),
-		process.WithCmdOptions(
-			runner.AttachOut(&output),
-			runner.AttachErr(&output)))
+	p.logger.Logf("Launching multipass image %s", batch.ID)
+	proc, err := process.Start("multipass", process.WithContext(ctx), process.WithArgs(args), process.WithCmdOptions(runner.AttachOut(&output), runner.AttachErr(&output)))
 	if err != nil {
 		return fmt.Errorf("failed to run multipass launch: %w", err)
 	}
diff --git a/pkg/testing/ogc/provisioner.go b/pkg/testing/ogc/provisioner.go
@@ -79,7 +79,7 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches
 	defer upCancel()
 	upOutput, err := p.ogcUp(upCtx)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("ogc up failed: %w", err)
 	}
 
 	// fetch the machines and run the batches on the machine
@@ -100,8 +100,8 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches
 	for _, b := range batches {
 		machine, ok := findMachine(machines, b.ID)
 		if !ok {
-			// print the output so its clear what went wrong
-			// without this it's unclear where OGC went wrong it
+			// print the output so its clear what went wrong.
+			// Without this it's unclear where OGC went wrong, it
 			// doesn't do a great job of reporting a clean error
 			fmt.Fprintf(os.Stdout, "%s\n", upOutput)
 			return nil, fmt.Errorf("failed to find machine for batch ID: %s", b.ID)
diff --git a/pkg/testing/runner/provisioner.go b/pkg/testing/runner/provisioner.go
@@ -12,6 +12,9 @@ import (
 
 // Instance represents a provisioned instance.
 type Instance struct {
+	// Provider is the instance provider for the instance.
+	// See INSTANCE_PROVISIONER environment variable for the supported providers.
+	Provider string `yaml:"provider"`
 	// ID is the identifier of the instance.
 	//
 	// This must be the same ID of the OSBatch.
diff --git a/pkg/testing/runner/runner_test.go b/pkg/testing/runner/runner_test.go
@@ -99,19 +99,19 @@ type fakeInstanceProvisioner struct {
 	instances []Instance
 }
 
-func (f *fakeInstanceProvisioner) Name() string {
+func (p *fakeInstanceProvisioner) Name() string {
 	return "fake"
 }
 
-func (f *fakeInstanceProvisioner) SetLogger(_ Logger) {
+func (p *fakeInstanceProvisioner) SetLogger(_ Logger) {
 }
 
-func (f *fakeInstanceProvisioner) Supported(_ define.OS) bool {
+func (p *fakeInstanceProvisioner) Supported(_ define.OS) bool {
 	return true
 }
 
-func (f *fakeInstanceProvisioner) Provision(_ context.Context, _ Config, batches []OSBatch) ([]Instance, error) {
-	f.batches = batches
+func (p *fakeInstanceProvisioner) Provision(_ context.Context, _ Config, batches []OSBatch) ([]Instance, error) {
+	p.batches = batches
 	var instances []Instance
 	for _, batch := range batches {
 		instances = append(instances, Instance{
@@ -126,8 +126,8 @@ func (f *fakeInstanceProvisioner) Provision(_ context.Context, _ Config, batches
 	return instances, nil
 }
 
-func (f *fakeInstanceProvisioner) Clean(_ context.Context, _ Config, instances []Instance) error {
-	f.instances = instances
+func (p *fakeInstanceProvisioner) Clean(_ context.Context, _ Config, instances []Instance) error {
+	p.instances = instances
 	return nil
 }
 
@@ -137,17 +137,17 @@ type fakeStackProvisioner struct {
 	deletedStacks []Stack
 }
 
-func (f *fakeStackProvisioner) Name() string {
+func (p *fakeStackProvisioner) Name() string {
 	return "fake"
 }
 
-func (f *fakeStackProvisioner) SetLogger(_ Logger) {
+func (p *fakeStackProvisioner) SetLogger(_ Logger) {
 }
 
-func (f *fakeStackProvisioner) Create(_ context.Context, request StackRequest) (Stack, error) {
-	f.mx.Lock()
-	defer f.mx.Unlock()
-	f.requests = append(f.requests, request)
+func (p *fakeStackProvisioner) Create(_ context.Context, request StackRequest) (Stack, error) {
+	p.mx.Lock()
+	defer p.mx.Unlock()
+	p.requests = append(p.requests, request)
 	return Stack{
 		ID:            request.ID,
 		Version:       request.Version,
@@ -160,14 +160,14 @@ func (f *fakeStackProvisioner) Create(_ context.Context, request StackRequest) (
 	}, nil
 }
 
-func (f *fakeStackProvisioner) WaitForReady(_ context.Context, stack Stack) (Stack, error) {
+func (p *fakeStackProvisioner) WaitForReady(_ context.Context, stack Stack) (Stack, error) {
 	stack.Ready = true
 	return stack, nil
 }
 
-func (f *fakeStackProvisioner) Delete(_ context.Context, stack Stack) error {
-	f.mx.Lock()
-	defer f.mx.Unlock()
-	f.deletedStacks = append(f.deletedStacks, stack)
+func (p *fakeStackProvisioner) Delete(_ context.Context, stack Stack) error {
+	p.mx.Lock()
+	defer p.mx.Unlock()
+	p.deletedStacks = append(p.deletedStacks, stack)
 	return nil
 }
diff --git a/pkg/testing/tools/artifacts_api.go b/pkg/testing/tools/artifacts_api.go
@@ -244,7 +244,7 @@ type logger interface {
 	Logf(format string, args ...any)
 }
 
-func GetLatestSnapshotVersion(ctx context.Context, log logger, aac *ArtifactAPIClient) (*version.ParsedSemVer, error) {
+func (aac ArtifactAPIClient) GetLatestSnapshotVersion(ctx context.Context, log logger) (*version.ParsedSemVer, error) {
 	vList, err := aac.GetVersions(ctx)
 	if err != nil {
 		return nil, err
diff --git a/pkg/testing/tools/fleettools/fleet.go b/pkg/testing/tools/fleettools/fleet.go
@@ -21,6 +21,7 @@ func GetAgentByPolicyIDAndHostnameFromList(client *kibana.Client, policyID, host
 		return nil, err
 	}
 
+	var agentHostnames []string
 	hostnameAgents := make([]*kibana.AgentExisting, 0)
 	for i, item := range listAgentsResp.Items {
 		agentHostname := item.LocalMetadata.Host.Hostname
@@ -32,7 +33,8 @@ func GetAgentByPolicyIDAndHostnameFromList(client *kibana.Client, policyID, host
 	}
 
 	if len(hostnameAgents) == 0 {
-		return nil, fmt.Errorf("unable to find agent with hostname [%s]", hostname)
+		return nil, fmt.Errorf("unable to find agent with hostname [%s] for policy [%s]. Found: %v",
+			hostname, policyID, agentHostnames)
 	}
 
 	if len(hostnameAgents) > 1 {
diff --git a/testing/integration/upgrade_downgrade_test.go b/testing/integration/upgrade_downgrade_test.go
@@ -40,7 +40,7 @@ func TestStandaloneDowngradeToSpecificSnapshotBuild(t *testing.T) {
 
 	// retrieve all the versions of agent from the artifact API
 	aac := tools.NewArtifactAPIClient()
-	latestSnapshotVersion, err := tools.GetLatestSnapshotVersion(ctx, t, aac)
+	latestSnapshotVersion, err := aac.GetLatestSnapshotVersion(ctx, t)
 	require.NoError(t, err)
 
 	// get all the builds of the snapshot version (need to pass x.y.z-SNAPSHOT format)
diff --git a/testing/integration/upgrade_fleet_test.go b/testing/integration/upgrade_fleet_test.go
diff --git a/testing/upgradetest/upgrader.go b/testing/upgradetest/upgrader.go

Original file line number	Diff line number	Diff line change
`@@ -108,7 +108,13 @@ func (r *artifactResult) Fetch(ctx context.Context, l Logger, dir string) error`
`108`	`108`	`}`
`109`	`109`
`110`	`110`	`// fetch package hash`
`111`		`- err = DownloadPackage(ctx, l, r.doer, r.src+hashExt, filepath.Join(dir, r.path+hashExt))`
	`111`	`+ err = DownloadPackage(ctx, l, r.doer, r.src+extHash, filepath.Join(dir, r.path+extHash))`
	`112`	`+ if err != nil {`
	`113`	`+ return fmt.Errorf("failed to download %s: %w", r.src, err)`
	`114`	`+ }`
	`115`	`+`
	`116`	`+ // fetch package asc`
	`117`	`+ err = DownloadPackage(ctx, l, r.doer, r.src+extAsc, filepath.Join(dir, r.path+extAsc))`
`112`	`118`	`if err != nil {`
`113`	`119`	`return fmt.Errorf("failed to download %s: %w", r.src, err)`
`114`	`120`	`}`
Original file line number	Diff line number	Diff line change
`@@ -107,7 +107,7 @@ func (r *localFetcherResult) Fetch(_ context.Context, _ Logger, dir string) erro`
`107`	`107`	`}`
`108`	`108`
`109`	`109`	`// fetch artifact hash`
`110`		`- err = copyFile(fullPath+hashExt, path+hashExt)`
	`110`	`+ err = copyFile(fullPath+extHash, path+extHash)`
`111`	`111`	`if err != nil {`
`112`	`112`	`return fmt.Errorf("error copying file: %w", err)`
`113`	`113`	`}`
Original file line number	Diff line number	Diff line change
`@@ -537,7 +537,7 @@ type ExecErr struct {`
`537`	`537`	`}`
`538`	`538`
`539`	`539`	`func (e *ExecErr) Error() string {`
`540`		`- return e.err.Error()`
	`540`	`+ return e.String()`
`541`	`541`	`}`
`542`	`542`
`543`	`543`	`func (e *ExecErr) String() string {`
Original file line number	Diff line number	Diff line change
`@@ -144,7 +144,7 @@ func (f Fixture) Install(ctx context.Context, installOpts InstallOpts, opts ..`
`144`	`144`	`f.collectDiagnostics()`
`145`	`145`	`}`
`146`	`146`
`147`		`- // environment variable AGENT_KEEP_INSTALLED=true will skip the uninstall`
	`147`	`+ // environment variable AGENT_KEEP_INSTALLED=true will skip the uninstallation`
`148`	`148`	`// useful to debug the issue with the Elastic Agent`
`149`	`149`	`if f.t.Failed() && keepInstalledFlag() {`
`150`	`150`	`f.t.Logf("skipping uninstall; test failed and AGENT_KEEP_INSTALLED=true")`