[deb/rpm] copy run folder from old agent installation (#7999)

* fix: copy run folder from old agent installation

* fix: add debian OS support in deb related integration tests

* doc: add changelog fragment

* fix: deduplicate similar code and improve code readability

* fix: typo

* fix: preserve mode,ownership,timestamps during cp

Author: pkoutsovasilis

changelog/fragments/1745923884-deb-rpm-preserve-run-state.yaml

@@ -0,0 +1,5 @@
+kind: bug-fix
+summary: Preserve agent run state on DEB and RPM upgrades
+component: "elastic-agent"
+pr: https://github.com/elastic/elastic-agent/pull/7999
+issue: https://github.com/elastic/elastic-agent/issues/3832

dev-tools/packaging/templates/linux/preinstall.sh.tmpl

@@ -41,13 +41,23 @@ if test -L "$symlink"; then
         else
             echo "didn't find $enc_path"
         fi
+
+        old_run_path="$old_agent_dir/run"
+        new_run_path="$new_agent_dir/run"
+        if [ -d "$old_run_path" ]; then
+            echo "found $old_run_path, copy to $new_agent_dir"
+            mkdir -p "$new_run_path"
+            cp -rfp "$old_run_path/." "$new_run_path/"
+        else
+            echo "didn't find $old_run_path"
+        fi
     fi
 else
     echo "no previous installation found"
 
     # create dir in case it does not exist
-    mkdir -p "$new_agent_dir" 
-    
+    mkdir -p "$new_agent_dir"
+
     # 2 is upgrade for Fedora, do not upgrade file when upgrading and file exists
     if [[ "$1" != "2" ]]; then
         if [[ -n "${ELASTIC_AGENT_FLAVOR}" ]]; then

pkg/testing/fixture.go

@@ -1245,8 +1245,8 @@ func getCacheDir(caller string, name string) (string, error) {
 	return cacheDir, nil
 }
 
-// FindComponentsDir identifies the directory that holds the components.
-func FindComponentsDir(dir, version string) (string, error) {
+// findAgentDataVersionDir identifies the directory that holds the agent data of the given version.
+func findAgentDataVersionDir(dir, version string) (string, error) {
 	dataDir := filepath.Join(dir, "data")
 	agentVersions, err := os.ReadDir(dataDir)
 	if err != nil {
@@ -1256,6 +1256,7 @@ func FindComponentsDir(dir, version string) (string, error) {
 	for _, fi := range agentVersions {
 		filename := fi.Name()
 		if strings.HasPrefix(filename, "elastic-agent-") && fi.IsDir() {
+			// Below we exclude the hash suffix (7 characters) of the directory to check the version
 			if version != "" && filename[:len(filename)-7] != "elastic-agent-"+version {
 				// version specified but version mismatch. in case of upgrade we have multiple
 				// directories, we don't want first found
@@ -1268,14 +1269,44 @@ func FindComponentsDir(dir, version string) (string, error) {
 	if versionDir == "" {
 		return "", fmt.Errorf("failed to find versioned directory for version %q", version)
 	}
-	componentsDir := filepath.Join(dataDir, versionDir, "components")
+	return filepath.Join(dataDir, versionDir), nil
+}
+
+// FindComponentsDir identifies the directory that holds the components.
+func FindComponentsDir(dir, version string) (string, error) {
+	versionDir, err := findAgentDataVersionDir(dir, version)
+	if err != nil {
+		return "", err
+	}
+	componentsDir := filepath.Join(versionDir, "components")
 	fi, err := os.Stat(componentsDir)
 	if (err != nil && !os.IsExist(err)) || !fi.IsDir() {
 		return "", fmt.Errorf("failed to find components directory at %s: %w", componentsDir, err)
 	}
 	return componentsDir, nil
 }
 
+// FindRunDir identifies the directory that holds the run folder.
+func FindRunDir(fixture *Fixture) (string, error) {
+	agentWorkDir := fixture.WorkDir()
+	if pf := fixture.PackageFormat(); pf == "deb" || pf == "rpm" {
+		// these are hardcoded paths in packages.yml
+		agentWorkDir = "/var/lib/elastic-agent"
+	}
+
+	version := fixture.Version()
+	versionDir, err := findAgentDataVersionDir(agentWorkDir, version)
+	if err != nil {
+		return "", err
+	}
+	runDir := filepath.Join(versionDir, "run")
+	fi, err := os.Stat(runDir)
+	if (err != nil && !os.IsExist(err)) || !fi.IsDir() {
+		return "", fmt.Errorf("failed to find run directory at %s: %w", runDir, err)
+	}
+	return runDir, nil
+}
+
 // writeSpecFile writes the specification to a specification file at the defined destination.
 func writeSpecFile(dest string, spec *component.Spec) error {
 	data, err := yaml.Marshal(spec)

testing/integration/linux_deb_test.go

@@ -9,7 +9,9 @@ package integration
 import (
 	"context"
 	"fmt"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -39,6 +41,10 @@ func TestDebLogIngestFleetManaged(t *testing.T) {
 				Type:   define.Linux,
 				Distro: "ubuntu",
 			},
+			{
+				Type:   define.Linux,
+				Distro: "debian",
+			},
 		},
 		Local: false,
 		Sudo:  true,
@@ -83,6 +89,10 @@ func TestDebInstallsServers(t *testing.T) {
 				Type:   define.Linux,
 				Distro: "ubuntu",
 			},
+			{
+				Type:   define.Linux,
+				Distro: "debian",
+			},
 		},
 		Local: false,
 		Sudo:  true,
@@ -175,6 +185,10 @@ func TestDebFleetUpgrade(t *testing.T) {
 				Type:   define.Linux,
 				Distro: "ubuntu",
 			},
+			{
+				Type:   define.Linux,
+				Distro: "debian",
+			},
 		},
 		Local: false,
 		Sudo:  true,
@@ -264,6 +278,15 @@ func testDebUpgrade(t *testing.T, upgradeFromVersion *version.ParsedSemVer, info
 
 	check.ConnectedToFleet(ctx, t, startFixture, 5*time.Minute)
 
+	const migrationMarkerFile = "migration_marker.file"
+	runDir, err := atesting.FindRunDir(startFixture)
+	require.NoError(t, err, "failed at getting run dir")
+
+	runMigrationMarker := filepath.Join(runDir, migrationMarkerFile)
+	f, err := os.Create(runMigrationMarker)
+	require.NoErrorf(t, err, "failed to create %q file", runMigrationMarker)
+	_ = f.Close()
+
 	// 3. Upgrade deb to the build version
 	srcPackage, err := endFixture.SrcPackage(ctx)
 	require.NoError(t, err)
@@ -272,6 +295,12 @@ func testDebUpgrade(t *testing.T, upgradeFromVersion *version.ParsedSemVer, info
 	out, err := cmd.CombinedOutput() // #nosec G204 -- Need to pass in name of package
 	require.NoError(t, err, string(out))
 
+	newRunDir, err := atesting.FindRunDir(endFixture)
+	require.NoError(t, err, "failed at getting run dir")
+	require.NotEqual(t, runDir, newRunDir, "the run dirs from upgrade should not match")
+	newRunMigrationMarker := filepath.Join(newRunDir, migrationMarkerFile)
+	require.FileExistsf(t, newRunMigrationMarker, "%q is missing", newRunMigrationMarker)
+
 	// 4. Wait for version in Fleet to match
 	// Fleet will not include the `-SNAPSHOT` in the `GetAgentVersion` result
 	noSnapshotVersion := strings.TrimSuffix(define.Version(), "-SNAPSHOT")

testing/integration/linux_rpm_test.go

@@ -9,7 +9,9 @@ package integration
 import (
 	"context"
 	"fmt"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -264,12 +266,27 @@ func testRpmUpgrade(t *testing.T, upgradeFromVersion *version.ParsedSemVer, info
 
 	check.ConnectedToFleet(ctx, t, startFixture, 5*time.Minute)
 
+	const migrationMarkerFile = "migration_marker.file"
+	runDir, err := atesting.FindRunDir(startFixture)
+	require.NoError(t, err, "failed at getting run dir")
+
+	runMigrationMarker := filepath.Join(runDir, migrationMarkerFile)
+	f, err := os.Create(runMigrationMarker)
+	require.NoErrorf(t, err, "failed to create %q file", runMigrationMarker)
+	_ = f.Close()
+
 	// 3. Upgrade rpm to the build version
 	srcPackage, err := endFixture.SrcPackage(ctx)
 	require.NoError(t, err)
 	out, err := exec.CommandContext(ctx, "sudo", "rpm", "-U", "-v", srcPackage).CombinedOutput() // #nosec G204 -- Need to pass in name of package
 	require.NoError(t, err, string(out))
 
+	newRunDir, err := atesting.FindRunDir(endFixture)
+	require.NoError(t, err, "failed at getting run dir")
+	require.NotEqual(t, runDir, newRunDir, "the run dirs from upgrade should not match")
+	newRunMigrationMarker := filepath.Join(newRunDir, migrationMarkerFile)
+	require.FileExistsf(t, newRunMigrationMarker, "%q is missing", newRunMigrationMarker)
+
 	// 4. Wait for version in Fleet to match
 	// Fleet will not include the `-SNAPSHOT` in the `GetAgentVersion` result
 	noSnapshotVersion := strings.TrimSuffix(define.Version(), "-SNAPSHOT")

testing/integration/restrict_upgrade_deb_test.go

@@ -28,6 +28,10 @@ func TestRestrictUpgradeDeb(t *testing.T) {
 				Type:   define.Linux,
 				Distro: "ubuntu",
 			},
+			{
+				Type:   define.Linux,
+				Distro: "debian",
+			},
 		},
 	})
 	t.Run("when agent is deployed via deb, a user should not be able to upgrade the agent using the cli", func(t *testing.T) {