[8.19] (backport #7999) [deb/rpm] copy run folder from old agent installation (#8060)

* [deb/rpm] copy run folder from old agent installation (#7999)

* fix: copy run folder from old agent installation

* fix: add debian OS support in deb related integration tests

* doc: add changelog fragment

* fix: deduplicate similar code and improve code readability

* fix: typo

* fix: preserve mode,ownership,timestamps during cp

(cherry picked from commit 815c264)

# Conflicts:
#	dev-tools/packaging/templates/linux/preinstall.sh.tmpl
#	pkg/testing/fixture.go
#	testing/integration/linux_deb_test.go

* fix: resolve conflicts

---------

Co-authored-by: Panos Koutsovasilis <[email protected]>

Author: mergify[bot]

changelog/fragments/1745923884-deb-rpm-preserve-run-state.yaml

@@ -0,0 +1,5 @@
+kind: bug-fix
+summary: Preserve agent run state on DEB and RPM upgrades
+component: "elastic-agent"
+pr: https://github.com/elastic/elastic-agent/pull/7999
+issue: https://github.com/elastic/elastic-agent/issues/3832

dev-tools/packaging/templates/linux/preinstall.sh.tmpl

@@ -40,6 +40,16 @@ if test -L "$symlink"; then
         else
             echo "didn't find $enc_path"
         fi
+
+        old_run_path="$old_agent_dir/run"
+        new_run_path="$new_agent_dir/run"
+        if [ -d "$old_run_path" ]; then
+            echo "found $old_run_path, copy to $new_agent_dir"
+            mkdir -p "$new_run_path"
+            cp -rfp "$old_run_path/." "$new_run_path/"
+        else
+            echo "didn't find $old_run_path"
+        fi
     fi
 else
     echo "no previous installation found"

pkg/testing/fixture.go

@@ -185,6 +185,11 @@ func (f *Fixture) Client() client.Client {
 	return f.c
 }
 
+// Version returns the Elastic Agent version.
+func (f *Fixture) Version() string {
+	return f.version
+}
+
 // Prepare prepares the Elastic Agent for usage.
 //
 // This must be called before `Configure`, `Run`, or `Install` can be called.
@@ -1012,7 +1017,7 @@ func (f *Fixture) prepareComponents(workDir string, components ...UsableComponen
 
 	// now remove all that should not be kept; removal is only
 	// done by removing the spec file, no need to delete the binary
-	componentsDir, err := FindComponentsDir(workDir)
+	componentsDir, err := FindComponentsDir(workDir, "")
 	if err != nil {
 		return err
 	}
@@ -1249,28 +1254,68 @@ func getCacheDir(caller string, name string) (string, error) {
 	return cacheDir, nil
 }
 
-// FindComponentsDir identifies the directory that holds the components.
-func FindComponentsDir(dir string) (string, error) {
+// findAgentDataVersionDir identifies the directory that holds the agent data of the given version.
+func findAgentDataVersionDir(dir, version string) (string, error) {
 	dataDir := filepath.Join(dir, "data")
 	agentVersions, err := os.ReadDir(dataDir)
 	if err != nil {
 		return "", fmt.Errorf("failed to read contents of the data directory %s: %w", dataDir, err)
 	}
 	var versionDir string
 	for _, fi := range agentVersions {
-		if strings.HasPrefix(fi.Name(), "elastic-agent-") && fi.IsDir() {
-			versionDir = fi.Name()
+		filename := fi.Name()
+		if strings.HasPrefix(filename, "elastic-agent-") && fi.IsDir() {
+			// Below we exclude the hash suffix (7 characters) of the directory to check the version
+			if version != "" && filename[:len(filename)-7] != "elastic-agent-"+version {
+				// version specified but version mismatch. in case of upgrade we have multiple
+				// directories, we don't want first found
+				continue
+			}
+			versionDir = filename
 			break
 		}
 	}
-	componentsDir := filepath.Join(dataDir, versionDir, "components")
+	if versionDir == "" {
+		return "", fmt.Errorf("failed to find versioned directory for version %q", version)
+	}
+	return filepath.Join(dataDir, versionDir), nil
+}
+
+// FindComponentsDir identifies the directory that holds the components.
+func FindComponentsDir(dir, version string) (string, error) {
+	versionDir, err := findAgentDataVersionDir(dir, version)
+	if err != nil {
+		return "", err
+	}
+	componentsDir := filepath.Join(versionDir, "components")
 	fi, err := os.Stat(componentsDir)
 	if (err != nil && !os.IsExist(err)) || !fi.IsDir() {
 		return "", fmt.Errorf("failed to find components directory at %s: %w", componentsDir, err)
 	}
 	return componentsDir, nil
 }
 
+// FindRunDir identifies the directory that holds the run folder.
+func FindRunDir(fixture *Fixture) (string, error) {
+	agentWorkDir := fixture.WorkDir()
+	if pf := fixture.PackageFormat(); pf == "deb" || pf == "rpm" {
+		// these are hardcoded paths in packages.yml
+		agentWorkDir = "/var/lib/elastic-agent"
+	}
+
+	version := fixture.Version()
+	versionDir, err := findAgentDataVersionDir(agentWorkDir, version)
+	if err != nil {
+		return "", err
+	}
+	runDir := filepath.Join(versionDir, "run")
+	fi, err := os.Stat(runDir)
+	if (err != nil && !os.IsExist(err)) || !fi.IsDir() {
+		return "", fmt.Errorf("failed to find run directory at %s: %w", runDir, err)
+	}
+	return runDir, nil
+}
+
 // writeSpecFile writes the specification to a specification file at the defined destination.
 func writeSpecFile(dest string, spec *component.Spec) error {
 	data, err := yaml.Marshal(spec)

testing/integration/linux_deb_test.go

@@ -9,7 +9,9 @@ package integration
 import (
 	"context"
 	"fmt"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -38,6 +40,10 @@ func TestDebLogIngestFleetManaged(t *testing.T) {
 				Type:   define.Linux,
 				Distro: "ubuntu",
 			},
+			{
+				Type:   define.Linux,
+				Distro: "debian",
+			},
 		},
 		Local: false,
 		Sudo:  true,
@@ -106,6 +112,10 @@ func TestDebFleetUpgrade(t *testing.T) {
 				Type:   define.Linux,
 				Distro: "ubuntu",
 			},
+			{
+				Type:   define.Linux,
+				Distro: "debian",
+			},
 		},
 		Local: false,
 		Sudo:  true,
@@ -169,6 +179,15 @@ func TestDebFleetUpgrade(t *testing.T) {
 
 	check.ConnectedToFleet(ctx, t, startFixture, 5*time.Minute)
 
+	const migrationMarkerFile = "migration_marker.file"
+	runDir, err := atesting.FindRunDir(startFixture)
+	require.NoError(t, err, "failed at getting run dir")
+
+	runMigrationMarker := filepath.Join(runDir, migrationMarkerFile)
+	f, err := os.Create(runMigrationMarker)
+	require.NoErrorf(t, err, "failed to create %q file", runMigrationMarker)
+	_ = f.Close()
+
 	// 3. Upgrade deb to the build version
 	srcPackage, err := endFixture.SrcPackage(ctx)
 	require.NoError(t, err)
@@ -177,6 +196,12 @@ func TestDebFleetUpgrade(t *testing.T) {
 	out, err := cmd.CombinedOutput() // #nosec G204 -- Need to pass in name of package
 	require.NoError(t, err, string(out))
 
+	newRunDir, err := atesting.FindRunDir(endFixture)
+	require.NoError(t, err, "failed at getting run dir")
+	require.NotEqual(t, runDir, newRunDir, "the run dirs from upgrade should not match")
+	newRunMigrationMarker := filepath.Join(newRunDir, migrationMarkerFile)
+	require.FileExistsf(t, newRunMigrationMarker, "%q is missing", newRunMigrationMarker)
+
 	// 4. Wait for version in Fleet to match
 	// Fleet will not include the `-SNAPSHOT` in the `GetAgentVersion` result
 	noSnapshotVersion := strings.TrimSuffix(define.Version(), "-SNAPSHOT")

testing/integration/linux_rpm_test.go

@@ -9,7 +9,9 @@ package integration
 import (
 	"context"
 	"fmt"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -170,12 +172,27 @@ func TestRpmFleetUpgrade(t *testing.T) {
 
 	check.ConnectedToFleet(ctx, t, startFixture, 5*time.Minute)
 
+	const migrationMarkerFile = "migration_marker.file"
+	runDir, err := atesting.FindRunDir(startFixture)
+	require.NoError(t, err, "failed at getting run dir")
+
+	runMigrationMarker := filepath.Join(runDir, migrationMarkerFile)
+	f, err := os.Create(runMigrationMarker)
+	require.NoErrorf(t, err, "failed to create %q file", runMigrationMarker)
+	_ = f.Close()
+
 	// 3. Upgrade rpm to the build version
 	srcPackage, err := endFixture.SrcPackage(ctx)
 	require.NoError(t, err)
 	out, err := exec.CommandContext(ctx, "sudo", "rpm", "-U", "-v", srcPackage).CombinedOutput() // #nosec G204 -- Need to pass in name of package
 	require.NoError(t, err, string(out))
 
+	newRunDir, err := atesting.FindRunDir(endFixture)
+	require.NoError(t, err, "failed at getting run dir")
+	require.NotEqual(t, runDir, newRunDir, "the run dirs from upgrade should not match")
+	newRunMigrationMarker := filepath.Join(newRunDir, migrationMarkerFile)
+	require.FileExistsf(t, newRunMigrationMarker, "%q is missing", newRunMigrationMarker)
+
 	// 4. Wait for version in Fleet to match
 	// Fleet will not include the `-SNAPSHOT` in the `GetAgentVersion` result
 	noSnapshotVersion := strings.TrimSuffix(define.Version(), "-SNAPSHOT")

testing/integration/otel_test.go

@@ -562,7 +562,7 @@ func TestOtelAPMIngestion(t *testing.T) {
 	err = fixture.EnsurePrepared(ctx)
 	require.NoError(t, err)
 
-	componentsDir, err := aTesting.FindComponentsDir(agentWorkDir)
+	componentsDir, err := aTesting.FindComponentsDir(agentWorkDir, "")
 	require.NoError(t, err)
 
 	// start apm default config just configure ES output

testing/integration/restrict_upgrade_deb_test.go

@@ -28,6 +28,10 @@ func TestRestrictUpgradeDeb(t *testing.T) {
 				Type:   define.Linux,
 				Distro: "ubuntu",
 			},
+			{
+				Type:   define.Linux,
+				Distro: "debian",
+			},
 		},
 	})
 	t.Run("when agent is deployed via deb, a user should not be able to upgrade the agent using the cli", func(t *testing.T) {