[RPM/DEB] Changes to binary distribution (#6638) (#7473)

(cherry picked from commit 5d4b00e)

Co-authored-by: Michal Pristas <[email protected]>

Author: mergify[bot]

dev-tools/packaging/templates/linux/postinstall.sh.tmpl

@@ -17,6 +17,8 @@ fi
 echo "create symlink "$symlink" to "$new_agent_dir/elastic-agent""
 ln -s "$new_agent_dir/elastic-agent" "$symlink"
 
+$new_agent_dir/elastic-agent apply-flavor
+
 # reload systemctl and then restart service
 echo "systemd enable/restart elastic-agent"
 systemctl daemon-reload 2> /dev/null

dev-tools/packaging/templates/linux/preinstall.sh.tmpl

@@ -5,6 +5,7 @@ set -e
 commit_hash="{{ commit_short }}"
 version_dir="{{agent_package_version}}{{snapshot_suffix}}"
 symlink="/usr/share/elastic-agent/bin/elastic-agent"
+flavor_file="/var/lib/elastic-agent/.flavor"
 new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$version_dir-$commit_hash"
 old_agent_dir=""
 
@@ -43,4 +44,19 @@ if test -L "$symlink"; then
     fi
 else
     echo "no previous installation found"
+
+    # create dir in case it does not exist
+    mkdir -p "$new_agent_dir" 
+    
+    # 2 is upgrade for Fedora, do not upgrade file when upgrading and file exists
+    if [[ "$1" != "2" ]]; then
+        if [[ -n "${ELASTIC_AGENT_FLAVOR}" ]]; then
+            # Do not modify the file if it already exists
+            echo "${ELASTIC_AGENT_FLAVOR}" > "$flavor_file"
+        else
+            # Defaults to basic installation
+            echo "basic" > "$flavor_file"
+        fi
+    fi
 fi
+

internal/pkg/agent/cmd/apply_flavor.go

@@ -0,0 +1,59 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
+	"github.com/elastic/elastic-agent/internal/pkg/agent/install"
+	"github.com/elastic/elastic-agent/internal/pkg/cli"
+	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
+)
+
+func newApplyFlavorCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "apply-flavor",
+		Short: "Apply Flavor cleans up unnecessary components from agent installation directory",
+		Run: func(c *cobra.Command, _ []string) {
+			if err := applyCmd(); err != nil {
+				fmt.Fprintf(streams.Err, "Error: %v\n%s\n", err, troubleshootMessage())
+				logExternal(fmt.Sprintf("%s apply flavor failed: %s", paths.BinaryName, err))
+				os.Exit(1)
+			}
+		},
+		Hidden: true,
+	}
+
+	return cmd
+}
+
+func applyCmd() error {
+	topPath := paths.Top()
+	detectedFlavor, err := install.UsedFlavor(topPath, "")
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil
+		}
+		return err
+	}
+	if detectedFlavor == "" {
+		return nil
+	}
+
+	versionedHome := paths.VersionedHome(topPath)
+	manifestFilePath := filepath.Join(versionedHome, v1.ManifestFileName)
+	flavor, err := install.Flavor(detectedFlavor, manifestFilePath, nil)
+	if err != nil {
+		return err
+	}
+
+	return install.ApplyFlavor(versionedHome, flavor)
+}

internal/pkg/agent/cmd/common.go

@@ -94,6 +94,7 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command {
 	addCommandIfNotNil(cmd, newComponentCommandWithArgs(args, streams))
 	addCommandIfNotNil(cmd, newLogsCommandWithArgs(args, streams))
 	addCommandIfNotNil(cmd, newOtelCommandWithArgs(args, streams))
+	addCommandIfNotNil(cmd, newApplyFlavorCommandWithArgs(args, streams))
 
 	// windows special hidden sub-command (only added on Windows)
 	reexec := newReExecWindowsCommand(args, streams)

internal/pkg/agent/install/flavors.go

@@ -48,7 +48,7 @@ func UsedFlavor(topPath, defaultFlavor string) (string, error) {
 		return "", fmt.Errorf("failed reading flavor marker file: %w", err)
 	}
 
-	return string(content), nil
+	return strings.TrimSpace(string(content)), nil
 }
 
 func Flavor(detectedFlavor string, registryPath string, flavorsRegistry map[string][]string) (FlavorDefinition, error) {
@@ -118,7 +118,7 @@ func ApplyFlavor(versionedHome string, flavor FlavorDefinition) error {
 	}
 
 	for _, ftr := range filesToRemove {
-		if removeErr := os.RemoveAll(ftr); !os.IsNotExist(removeErr) {
+		if removeErr := os.RemoveAll(ftr); removeErr != nil && !os.IsNotExist(removeErr) {
 			err = fmt.Errorf("failed cleaning components: %w", removeErr)
 		}
 	}

internal/pkg/agent/install/flavors_test.go

@@ -13,13 +13,22 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+
+	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
 )
 
 var flavorsRegistry = map[string][]string{
 	"basic":   {"agentbeat", "endpoint-security", "pf-host-agent"},
 	"servers": {"agentbeat", "endpoint-security", "pf-host-agent", "cloudbeat", "apm-server", "fleet-server", "pf-elastic-symbolizer", "pf-elastic-collector"},
 }
 
+var manifest = v1.PackageManifest{
+	Package: v1.PackageDesc{
+		Flavors: flavorsRegistry,
+	},
+}
+
 func TestSubpathsForComponent(t *testing.T) {
 	binarySuffix := ""
 	if runtime.GOOS == "windows" {
@@ -190,6 +199,42 @@ func TestAllowedSubpathsForFlavor(t *testing.T) {
 			sort.Strings(subpaths)
 			assert.Equal(t, tt.wantSubpaths, subpaths)
 		})
+
+		t.Run(tt.name+"-file", func(t *testing.T) {
+			tmpDir := t.TempDir()
+			manifestFile := filepath.Join(tmpDir, "manifest.yaml")
+			mb, err := yaml.Marshal(manifest)
+			require.NoError(t, err)
+			require.NoError(t, os.WriteFile(manifestFile, mb, 0o644))
+			defer os.Remove(manifestFile)
+
+			// Create temp dir with spec files
+			componentsDir := filepath.Join(versionedHome, "components")
+			require.NoError(t, os.MkdirAll(componentsDir, 0755))
+
+			// Write spec files
+			for component, content := range tt.specFiles {
+				specPath := filepath.Join(componentsDir, component+".spec.yml")
+				require.NoError(t, os.WriteFile(specPath, []byte(content), 0644))
+				defer os.Remove(specPath)
+			}
+
+			// Test function
+			flavor, err := Flavor(tt.flavor, manifestFile, nil)
+			if tt.wantError {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.errorContains)
+				return
+			}
+
+			subpaths, err := allowedSubpathsForFlavor(versionedHome, flavor)
+			assert.NoError(t, err)
+
+			require.NoError(t, err)
+			sort.Strings(tt.wantSubpaths)
+			sort.Strings(subpaths)
+			assert.Equal(t, tt.wantSubpaths, subpaths)
+		})
 	}
 }
 

pkg/testing/fixture_install.go

@@ -461,7 +461,11 @@ func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, shou
 	}
 
 	// sudo apt-get install the deb
-	out, err := exec.CommandContext(ctx, "sudo", "apt-get", "install", "-y", f.srcPackage).CombinedOutput() // #nosec G204 -- Need to pass in name of package
+	cmd := exec.CommandContext(ctx, "sudo", "-E", "apt-get", "install", "-y", f.srcPackage) // #nosec G204 -- Need to pass in name of package
+	if installOpts.InstallServers {
+		cmd.Env = append(cmd.Env, "ELASTIC_AGENT_FLAVOR=servers")
+	}
+	out, err := cmd.CombinedOutput() // #nosec G204 -- Need to pass in name of package
 	if err != nil {
 		return out, fmt.Errorf("apt install failed: %w output:%s", err, string(out))
 	}
@@ -546,7 +550,11 @@ func (f *Fixture) installRpm(ctx context.Context, installOpts *InstallOpts, shou
 	}
 
 	// sudo rpm -iv elastic-agent rpm
-	out, err := exec.CommandContext(ctx, "sudo", "rpm", "-i", "-v", f.srcPackage).CombinedOutput() // #nosec G204 -- Need to pass in name of package
+	cmd := exec.CommandContext(ctx, "sudo", "-E", "rpm", "-i", "-v", f.srcPackage) // #nosec G204 -- Need to pass in name of package
+	if installOpts.InstallServers {
+		cmd.Env = append(cmd.Env, "ELASTIC_AGENT_FLAVOR=servers")
+	}
+	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return out, fmt.Errorf("rpm install failed: %w output:%s", err, string(out))
 	}

testing/integration/install_test.go

@@ -438,6 +438,10 @@ func testInstallWithoutBasePathWithCustomUser(ctx context.Context, t *testing.T,
 func testComponentsPresence(ctx context.Context, fixture *atesting.Fixture, requiredComponents []componentPresenceDefinition, unwantedComponents []componentPresenceDefinition) func(*testing.T) {
 	return func(t *testing.T) {
 		agentWorkDir := fixture.WorkDir()
+		if pf := fixture.PackageFormat(); pf == "deb" || pf == "rpm" {
+			// these are hardcoded paths in packages.yml
+			agentWorkDir = "/var/lib/elastic-agent"
+		}
 		componentsDir, err := aTesting.FindComponentsDir(agentWorkDir)
 		require.NoError(t, err)
 

testing/integration/linux_deb_test.go

@@ -9,6 +9,7 @@ package integration
 import (
 	"context"
 	"fmt"
+	"os"
 	"os/exec"
 	"strings"
 	"testing"
@@ -49,6 +50,75 @@ func TestDebLogIngestFleetManaged(t *testing.T) {
 	agentFixture, err := define.NewFixtureFromLocalBuild(t, define.Version(), atesting.WithPackageFormat("deb"))
 	require.NoError(t, err)
 
+	installOpts := atesting.InstallOpts{
+		NonInteractive: true,
+		Force:          true,
+		InstallServers: false,
+	}
+
+	testDebLogIngestFleetManagedWithCheck(ctx, t, agentFixture, info, installOpts,
+		testComponentsPresence(ctx, agentFixture,
+			[]componentPresenceDefinition{
+				{"agentbeat", []string{"windows", "linux", "darwin"}},
+				{"endpoint-security", []string{"windows", "linux", "darwin"}},
+				{"pf-host-agent", []string{"linux"}},
+			},
+			[]componentPresenceDefinition{
+				{"cloudbeat", []string{"linux"}},
+				{"apm-server", []string{"windows", "linux", "darwin"}},
+				{"fleet-server", []string{"windows", "linux", "darwin"}},
+				{"pf-elastic-symbolizer", []string{"linux"}},
+				{"pf-elastic-collector", []string{"linux"}},
+			},
+		),
+	)
+}
+
+func TestDebInstallsServers(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Group: Deb,
+		Stack: &define.Stack{},
+		OS: []define.OS{
+			{
+				Type:   define.Linux,
+				Distro: "ubuntu",
+			},
+		},
+		Local: false,
+		Sudo:  true,
+	})
+
+	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
+	defer cancel()
+
+	agentFixture, err := define.NewFixtureFromLocalBuild(t, define.Version(), atesting.WithPackageFormat("deb"))
+	require.NoError(t, err)
+
+	installOpts := atesting.InstallOpts{
+		NonInteractive: true,
+		Force:          true,
+		InstallServers: true,
+	}
+
+	testDebLogIngestFleetManagedWithCheck(ctx, t, agentFixture, info, installOpts,
+		testComponentsPresence(ctx, agentFixture,
+			[]componentPresenceDefinition{
+				{"agentbeat", []string{"windows", "linux", "darwin"}},
+				{"endpoint-security", []string{"windows", "linux", "darwin"}},
+				{"pf-host-agent", []string{"linux"}},
+				{"cloudbeat", []string{"linux"}},
+				{"apm-server", []string{"windows", "linux", "darwin"}},
+				{"fleet-server", []string{"windows", "linux", "darwin"}},
+				{"pf-elastic-symbolizer", []string{"linux"}},
+				{"pf-elastic-collector", []string{"linux"}},
+			},
+			[]componentPresenceDefinition{},
+		),
+	)
+}
+
+func testDebLogIngestFleetManagedWithCheck(ctx context.Context, t *testing.T, agentFixture *atesting.Fixture, info *define.Info, installOpts atesting.InstallOpts, componentCheck func(t *testing.T)) {
+
 	// 1. Create a policy in Fleet with monitoring enabled.
 	// To ensure there are no conflicts with previous test runs against
 	// the same ESS stack, we add the current time at the end of the policy
@@ -70,10 +140,12 @@ func TestDebLogIngestFleetManaged(t *testing.T) {
 		},
 	}
 
-	installOpts := atesting.InstallOpts{
-		NonInteractive: true,
-		Force:          true,
-	}
+	// remove flavor to start fresh, in case there is some leftover.
+	_ = os.Remove("/var/lib/elastic-agent/.flavor")
+	t.Cleanup(func() {
+		// cleanup after ourselves
+		_ = os.Remove("/var/lib/elastic-agent/.flavor")
+	})
 
 	// 2. Install the Elastic-Agent with the policy that
 	// was just created.
@@ -88,6 +160,10 @@ func TestDebLogIngestFleetManaged(t *testing.T) {
 	t.Logf("created policy: %s", policy.ID)
 	check.ConnectedToFleet(ctx, t, agentFixture, 5*time.Minute)
 
+	if componentCheck != nil {
+		t.Run("check components set", componentCheck)
+	}
+
 	t.Run("Monitoring logs are shipped", func(t *testing.T) {
 		testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy)
 	})