(fleet) fix agent install without a pre-populated config (#25833)

* (fleet) fix agent install without a pre-populated config

* nice to haves

* align logic with the postinst

* remove check that is no longer needed

* oops

* fix filemode for the datadog.yaml

Author: arbll

pkg/fleet/installer/service/datadog_agent.go

@@ -63,51 +63,57 @@ func SetupAgent(ctx context.Context) (err error) {
 
 	for _, unit := range stableUnits {
 		if err = loadUnit(ctx, unit); err != nil {
-			return
+			return err
 		}
 	}
 	for _, unit := range experimentalUnits {
 		if err = loadUnit(ctx, unit); err != nil {
-			return
+			return err
 		}
 	}
 	if err = os.MkdirAll("/etc/datadog-agent", 0755); err != nil {
-		return
+		return err
 	}
 	ddAgentUID, ddAgentGID, err := getAgentIDs()
 	if err != nil {
 		return fmt.Errorf("error getting dd-agent user and group IDs: %w", err)
 	}
 
 	if err = os.Chown("/etc/datadog-agent", ddAgentUID, ddAgentGID); err != nil {
-		return
+		return err
 	}
 
 	if err = systemdReload(ctx); err != nil {
-		return
+		return err
 	}
 
 	for _, unit := range stableUnits {
 		if err = enableUnit(ctx, unit); err != nil {
-			return
-		}
-	}
-	for _, unit := range stableUnits {
-		if err = startUnit(ctx, unit); err != nil {
-			return
+			return err
 		}
 	}
 	if err = exec.CommandContext(ctx, "ln", "-sf", "/opt/datadog-packages/datadog-agent/stable/bin/agent/agent", agentSymlink).Run(); err != nil {
-		return
+		return err
 	}
 
 	// write installinfo before start, or the agent could write it
 	// TODO: add installer version properly
 	if err = installinfo.WriteInstallInfo("installer_package", "manual_update"); err != nil {
-		return
+		return err
 	}
 
-	return
+	_, err = os.Stat("/etc/datadog-agent/datadog.yaml")
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	// this is expected during a fresh install with the install script / asible / chef / etc...
+	// the config is populated afterwards by the install method and the agent is restarted
+	if !os.IsNotExist(err) {
+		if err = startUnit(ctx, agentUnit); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // RemoveAgent stops and removes the agent
@@ -152,7 +158,7 @@ func RemoveAgent(ctx context.Context) {
 
 // StartAgentExperiment starts the agent experiment
 func StartAgentExperiment(ctx context.Context) error {
-	return startUnit(ctx, agentExp)
+	return startUnit(ctx, agentExp, "--no-block")
 }
 
 // StopAgentExperiment stops the agent experiment

pkg/fleet/installer/service/datadog_installer.go

@@ -212,7 +212,7 @@ func RemoveInstaller(ctx context.Context) {
 
 // StartInstallerExperiment installs the experimental systemd units for the installer
 func StartInstallerExperiment(ctx context.Context) error {
-	return startUnit(ctx, installerUnitExp)
+	return startUnit(ctx, installerUnitExp, "--no-block")
 }
 
 // StopInstallerExperiment starts the stable systemd units for the installer

pkg/fleet/installer/service/systemd.go

@@ -38,18 +38,20 @@ func findSystemdPath() (systemdPath string) {
 	return debSystemdPath
 }
 
-func stopUnit(ctx context.Context, unit string) error {
+func stopUnit(ctx context.Context, unit string, args ...string) error {
 	span, _ := tracer.StartSpanFromContext(ctx, "stop_unit")
 	defer span.Finish()
 	span.SetTag("unit", unit)
-	return exec.CommandContext(ctx, "systemctl", "stop", unit, "--no-block").Run()
+	args = append([]string{"stop", unit}, args...)
+	return exec.CommandContext(ctx, "systemctl", args...).Run()
 }
 
-func startUnit(ctx context.Context, unit string) error {
+func startUnit(ctx context.Context, unit string, args ...string) error {
 	span, _ := tracer.StartSpanFromContext(ctx, "start_unit")
 	defer span.Finish()
 	span.SetTag("unit", unit)
-	return exec.CommandContext(ctx, "systemctl", "start", unit, "--no-block").Run()
+	args = append([]string{"start", unit}, args...)
+	return exec.CommandContext(ctx, "systemctl", args...).Run()
 }
 
 func enableUnit(ctx context.Context, unit string) error {

test/new-e2e/tests/installer/all_packages_test.go

@@ -64,7 +64,7 @@ var (
 var packagesConfig = []testPackageConfig{
 	{name: "datadog-installer", defaultVersion: fmt.Sprintf("pipeline-%v", os.Getenv("CI_PIPELINE_ID")), registry: "669783387624.dkr.ecr.us-east-1.amazonaws.com", auth: "ecr"},
 	{name: "datadog-agent", defaultVersion: fmt.Sprintf("pipeline-%v", os.Getenv("CI_PIPELINE_ID")), registry: "669783387624.dkr.ecr.us-east-1.amazonaws.com", auth: "ecr"},
-	{name: "datadog-apm-inject", defaultVersion: "latest", registry: "gcr.io/datadoghq", auth: "docker"},
+	{name: "datadog-apm-inject", defaultVersion: "latest"},
 	{name: "datadog-apm-library-java", defaultVersion: "latest"},
 	{name: "datadog-apm-library-ruby", defaultVersion: "latest"},
 	{name: "datadog-apm-library-js", defaultVersion: "latest"},
@@ -145,7 +145,7 @@ func (s *packageBaseSuite) ProvisionerOptions() []awshost.ProvisionerOption {
 
 func (s *packageBaseSuite) SetupSuite() {
 	s.BaseSuite.SetupSuite()
-	s.setupGlobalEnv()
+	s.setupFakeIntake()
 	s.host = host.New(s.T(), s.Env().RemoteHost, s.os, s.arch)
 }
 
@@ -168,14 +168,10 @@ func (s *packageBaseSuite) Purge() {
 	s.Env().RemoteHost.MustExecute("sudo apt-get remove -y --purge datadog-installer || sudo yum remove -y datadog-installer")
 }
 
-// setupGlobalEnv sets up the global environment variables for the agent processes
+// setupFakeIntake sets up the fake intake for the agent and trace agent.
 // This is done with SystemD environment files overrides to avoid having to touch the agent configuration files
 // and potentially interfere with the tests.
-func (s *packageBaseSuite) setupGlobalEnv() {
-	// FIXME: this should be done in the installer
-	s.Env().RemoteHost.MustExecute("sudo mkdir -p /etc/datadog-agent")
-	s.Env().RemoteHost.MustExecute(`echo "api_key: deadbeefdeadbeefdeadbeefdeadbeef" | sudo tee /etc/datadog-agent/datadog.yaml`)
-
+func (s *packageBaseSuite) setupFakeIntake() {
 	var env []string
 	if s.Env().FakeIntake != nil {
 		env = append(env, []string{
@@ -189,8 +185,8 @@ func (s *packageBaseSuite) setupGlobalEnv() {
 	}
 	s.Env().RemoteHost.MustExecute("sudo mkdir -p /etc/systemd/system/datadog-agent.service.d")
 	s.Env().RemoteHost.MustExecute("sudo mkdir -p /etc/systemd/system/datadog-agent-trace.service.d")
-	s.Env().RemoteHost.MustExecute(`printf "[Service]\nEnvironmentFile=-/etc/environment\n" | sudo tee /etc/systemd/system/datadog-agent-trace.service.d/inject.conf`)
-	s.Env().RemoteHost.MustExecute(`printf "[Service]\nEnvironmentFile=-/etc/environment\n" | sudo tee /etc/systemd/system/datadog-agent-trace.service.d/inject.conf`)
+	s.Env().RemoteHost.MustExecute(`printf "[Service]\nEnvironmentFile=-/etc/environment\n" | sudo tee /etc/systemd/system/datadog-agent-trace.service.d/fake-intake.conf`)
+	s.Env().RemoteHost.MustExecute(`printf "[Service]\nEnvironmentFile=-/etc/environment\n" | sudo tee /etc/systemd/system/datadog-agent-trace.service.d/fake-intake.conf`)
 	s.Env().RemoteHost.MustExecute("sudo systemctl daemon-reload")
 }
 

test/new-e2e/tests/installer/host/host.go

@@ -21,6 +21,7 @@ import (
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client"
 	e2eos "github.com/DataDog/test-infra-definitions/components/os"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // Host is a remote host environment.
@@ -88,7 +89,8 @@ func (h *Host) DeletePath(path string) {
 // WaitForUnitActive waits for a systemd unit to be active
 func (h *Host) WaitForUnitActive(units ...string) {
 	for _, unit := range units {
-		h.remote.MustExecute(fmt.Sprintf("timeout=30; unit=%s; while ! systemctl is-active --quiet $unit && [ $timeout -gt 0 ]; do sleep 1; ((timeout--)); done; [ $timeout -ne 0 ]", unit))
+		_, err := h.remote.Execute(fmt.Sprintf("timeout=30; unit=%s; while ! systemctl is-active --quiet $unit && [ $timeout -gt 0 ]; do sleep 1; ((timeout--)); done; [ $timeout -ne 0 ]", unit))
+		require.NoError(h.t, err, "unit %s did not become active", unit)
 	}
 }
 

test/new-e2e/tests/installer/package_agent_test.go

@@ -34,6 +34,7 @@ func (s *packageAgentSuite) TestInstall() {
 	state.AssertUnitsDead("datadog-agent-sysprobe.service", "datadog-agent-security.service")
 
 	state.AssertFileExists("/etc/datadog-agent/install_info", 0644, "root", "root")
+	state.AssertFileExists("/etc/datadog-agent/datadog.yaml", 0640, "dd-agent", "dd-agent")
 	// FIXME: this file is either dd-agent or root depending on the OS for some reason
 	// state.AssertFileExists("/etc/datadog-agent/install.json", 0644, "dd-agent", "dd-agent")
 }