Skip to content

Commit cfaadec

Browse files
authored
Add config option to deploy custom elastic-agents as test services (#786)
* Add config option to deploy custom elastic-agents as test services * Tidy go.mod * Simplify runner changes * Clean go.mod * Move logic to custom_agent deployer * Use DockerComposeDeployedService code * Connect to network before healthcheck * Add test, docs, and initialize the same as a composer deployer * Change Makefile * Format test package * Add test to CI * Bump stack versions * Ignore errors in test pipeline * Use apache as custom-agent test and revert stack version changes * Fail if docker-compose.yml is not found * Change custom agent deployer to use a base agent config * Replace apache custom agent example with auditd_manager * Wrap compose service deployer * Format * Overwrite setup and teardown to avoid changing compose deployer * Use service variant to avoid overriding teardown entirely * Update docs * Revert Makefile unrelated change * Use installed resources instead of a temp file * add version to custom-agent.yml * quote version field
1 parent 6831dbd commit cfaadec

File tree

23 files changed

+2441
-4
lines changed

23 files changed

+2441
-4
lines changed

.ci/Jenkinsfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ pipeline {
103103
'stack-command-8x': generateTestCommandStage(command: 'test-stack-command-8x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*']),
104104
'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
105105
'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
106+
'check-packages-with-custom-agent': generateTestCommandStage(command: 'test-check-packages-with-custom-agent', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
106107
'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/integrations/*.sig']),
107108
'profiles-command': generateTestCommandStage(command: 'test-profiles-command')
108109
]

Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ test-stack-command-8x:
6565

6666
test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
6767

68-
test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel
68+
test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent
6969

7070
test-check-packages-with-kind:
7171
PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
@@ -76,6 +76,9 @@ test-check-packages-other:
7676
test-check-packages-parallel:
7777
PACKAGE_TEST_TYPE=parallel ./scripts/test-check-packages.sh
7878

79+
test-check-packages-with-custom-agent:
80+
PACKAGE_TEST_TYPE=with-custom-agent ./scripts/test-check-packages.sh
81+
7982
test-build-zip:
8083
./scripts/test-build-zip.sh
8184

docs/howto/system_testing.md

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ or the data stream's level:
6464

6565
`<service deployer>` - a name of the supported service deployer:
6666
* `docker` - Docker Compose
67+
* `agent` - Custom `elastic-agent` with Docker Compose
6768
* `k8s` - Kubernetes
6869
* `tf` - Terraform
6970

@@ -106,6 +107,58 @@ volumes:
106107
mysqldata:
107108
```
108109

110+
### Agent service deployer
111+
112+
When using the Agent service deployer, the `elastic-agent` provided by the stack
113+
will not be used. An agent will be deployed as a Docker compose service named `docker-custom-agent`
114+
which base configuration is provided [here](../../internal/install/_static/docker-custom-agent-base.yml).
115+
This configuration will be merged with the one provided in the `custom-agent.yml` file.
116+
This is useful if you need different capabilities than the provided by the
117+
`elastic-agent` used by the `elastic-package stack` command.
118+
119+
`custom-agent.yml`
120+
```
121+
version: '2.3'
122+
services:
123+
docker-custom-agent:
124+
pid: host
125+
cap_add:
126+
- AUDIT_CONTROL
127+
- AUDIT_READ
128+
user: root
129+
```
130+
131+
This will result in an agent configuration such as:
132+
133+
```
134+
version: '2.3'
135+
services:
136+
docker-custom-agent:
137+
hostname: docker-custom-agent
138+
image: "docker.elastic.co/beats/elastic-agent-complete:8.2.0"
139+
pid: host
140+
cap_add:
141+
- AUDIT_CONTROL
142+
- AUDIT_READ
143+
user: root
144+
healthcheck:
145+
test: "elastic-agent status"
146+
retries: 180
147+
interval: 1s
148+
environment:
149+
FLEET_ENROLL: "1"
150+
FLEET_INSECURE: "1"
151+
FLEET_URL: "http://fleet-server:8220"
152+
```
153+
154+
And in the test config:
155+
156+
```
157+
data_stream:
158+
vars:
159+
# ...
160+
```
161+
109162

110163
### Terraform service deployer
111164

internal/configuration/locations/locations.go

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,15 @@ const (
2727
fieldsCachedDir = "cache/fields"
2828

2929
terraformDeployerYmlFile = "terraform-deployer.yml"
30+
31+
dockerCustomAgentDeployerYmlFile = "docker-custom-agent-base.yml"
3032
)
3133

3234
var (
33-
serviceLogsDir = filepath.Join(temporaryDir, "service_logs")
34-
kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes")
35-
terraformDeployerDir = filepath.Join(deployerDir, "terraform")
35+
serviceLogsDir = filepath.Join(temporaryDir, "service_logs")
36+
kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes")
37+
terraformDeployerDir = filepath.Join(deployerDir, "terraform")
38+
dockerCustomAgentDeployerDir = filepath.Join(deployerDir, "docker_custom_agent")
3639
)
3740

3841
//LocationManager maintains an instance of a config path location
@@ -96,6 +99,16 @@ func (loc LocationManager) TerraformDeployerYml() string {
9699
return filepath.Join(loc.stackPath, terraformDeployerDir, terraformDeployerYmlFile)
97100
}
98101

102+
// DockerCustomAgentDeployerDir returns the DockerCustomAgent Directory
103+
func (loc LocationManager) DockerCustomAgentDeployerDir() string {
104+
return filepath.Join(loc.stackPath, dockerCustomAgentDeployerDir)
105+
}
106+
107+
// DockerCustomAgentDeployerYml returns the DockerCustomAgent deployer yml file
108+
func (loc LocationManager) DockerCustomAgentDeployerYml() string {
109+
return filepath.Join(loc.stackPath, dockerCustomAgentDeployerDir, dockerCustomAgentDeployerYmlFile)
110+
}
111+
99112
// ServiceLogDir returns the log directory
100113
func (loc LocationManager) ServiceLogDir() string {
101114
return filepath.Join(loc.stackPath, serviceLogsDir)
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
version: "2.3"
2+
services:
3+
docker-custom-agent:
4+
image: "${ELASTIC_AGENT_IMAGE_REF}"
5+
healthcheck:
6+
test: "elastic-agent status"
7+
retries: 180
8+
interval: 1s
9+
hostname: docker-custom-agent
10+
environment:
11+
- FLEET_ENROLL=1
12+
- FLEET_INSECURE=1
13+
- FLEET_URL=http://fleet-server:8220
14+
volumes:
15+
- ${SERVICE_LOGS_DIR}:/tmp/service_logs/

internal/install/install.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ func EnsureInstalled() error {
6767
return errors.Wrap(err, "writing Terraform deployer resources failed")
6868
}
6969

70+
err = writeDockerCustomAgentResources(elasticPackagePath)
71+
if err != nil {
72+
return errors.Wrap(err, "writing Terraform deployer resources failed")
73+
}
74+
7075
if err := createServiceLogsDir(elasticPackagePath); err != nil {
7176
return errors.Wrap(err, "creating service logs directory failed")
7277
}
@@ -218,6 +223,17 @@ func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManag
218223
return nil
219224
}
220225

226+
func writeDockerCustomAgentResources(elasticPackagePath *locations.LocationManager) error {
227+
dir := elasticPackagePath.DockerCustomAgentDeployerDir()
228+
if err := os.MkdirAll(dir, 0755); err != nil {
229+
return errors.Wrapf(err, "creating directory failed (path: %s)", dir)
230+
}
231+
if err := writeStaticResource(nil, elasticPackagePath.DockerCustomAgentDeployerYml(), dockerCustomAgentBaseYml); err != nil {
232+
return errors.Wrap(err, "writing static resource failed")
233+
}
234+
return nil
235+
}
236+
221237
func writeConfigFile(elasticPackagePath *locations.LocationManager) error {
222238
var err error
223239
err = writeStaticResource(err, filepath.Join(elasticPackagePath.RootDir(), applicationConfigurationYmlFile), applicationConfigurationYml)

internal/install/static.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,6 @@ var geoIpCountryMmdb string
2929

3030
//go:embed _static/service_tokens
3131
var serviceTokens string
32+
33+
//go:embed _static/docker-custom-agent-base.yml
34+
var dockerCustomAgentBaseYml string
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License;
3+
// you may not use this file except in compliance with the Elastic License.
4+
5+
package servicedeployer
6+
7+
import (
8+
_ "embed"
9+
"fmt"
10+
11+
"github.com/pkg/errors"
12+
13+
"github.com/elastic/elastic-package/internal/compose"
14+
"github.com/elastic/elastic-package/internal/configuration/locations"
15+
"github.com/elastic/elastic-package/internal/docker"
16+
"github.com/elastic/elastic-package/internal/files"
17+
"github.com/elastic/elastic-package/internal/install"
18+
"github.com/elastic/elastic-package/internal/kibana"
19+
"github.com/elastic/elastic-package/internal/logger"
20+
"github.com/elastic/elastic-package/internal/stack"
21+
)
22+
23+
const dockerCustomAgentName = "docker-custom-agent"
24+
25+
// CustomAgentDeployer knows how to deploy a custom elastic-agent defined via
26+
// a Docker Compose file.
27+
type CustomAgentDeployer struct {
28+
cfg string
29+
}
30+
31+
// NewCustomAgentDeployer returns a new instance of a deployedCustomAgent.
32+
func NewCustomAgentDeployer(cfgPath string) (*CustomAgentDeployer, error) {
33+
return &CustomAgentDeployer{
34+
cfg: cfgPath,
35+
}, nil
36+
}
37+
38+
// SetUp sets up the service and returns any relevant information.
39+
func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) {
40+
logger.Debug("setting up service using Docker Compose service deployer")
41+
42+
appConfig, err := install.Configuration()
43+
if err != nil {
44+
return nil, errors.Wrap(err, "can't read application configuration")
45+
}
46+
47+
kibanaClient, err := kibana.NewClient()
48+
if err != nil {
49+
return nil, errors.Wrap(err, "can't create Kibana client")
50+
}
51+
52+
stackVersion, err := kibanaClient.Version()
53+
if err != nil {
54+
return nil, errors.Wrap(err, "can't read Kibana injected metadata")
55+
}
56+
57+
env := append(
58+
appConfig.StackImageRefs(stackVersion).AsEnv(),
59+
fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local),
60+
)
61+
62+
ymlPaths, err := d.loadComposeDefinitions()
63+
if err != nil {
64+
return nil, err
65+
}
66+
67+
service := dockerComposeDeployedService{
68+
ymlPaths: ymlPaths,
69+
project: "elastic-package-service",
70+
sv: ServiceVariant{
71+
Name: dockerCustomAgentName,
72+
Env: env,
73+
},
74+
}
75+
76+
outCtxt := inCtxt
77+
78+
p, err := compose.NewProject(service.project, service.ymlPaths...)
79+
if err != nil {
80+
return nil, errors.Wrap(err, "could not create Docker Compose project for service")
81+
}
82+
83+
// Verify the Elastic stack network
84+
err = stack.EnsureStackNetworkUp()
85+
if err != nil {
86+
return nil, errors.Wrap(err, "Elastic stack network is not ready")
87+
}
88+
89+
// Clean service logs
90+
err = files.RemoveContent(outCtxt.Logs.Folder.Local)
91+
if err != nil {
92+
return nil, errors.Wrap(err, "removing service logs failed")
93+
}
94+
95+
inCtxt.Name = dockerCustomAgentName
96+
serviceName := inCtxt.Name
97+
opts := compose.CommandOptions{
98+
Env: env,
99+
ExtraArgs: []string{"--build", "-d"},
100+
}
101+
err = p.Up(opts)
102+
if err != nil {
103+
return nil, errors.Wrap(err, "could not boot up service using Docker Compose")
104+
}
105+
106+
// Connect service network with stack network (for the purpose of metrics collection)
107+
err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network())
108+
if err != nil {
109+
return nil, errors.Wrapf(err, "can't attach service container to the stack network")
110+
}
111+
112+
err = p.WaitForHealthy(opts)
113+
if err != nil {
114+
processServiceContainerLogs(p, compose.CommandOptions{
115+
Env: opts.Env,
116+
}, outCtxt.Name)
117+
return nil, errors.Wrap(err, "service is unhealthy")
118+
}
119+
120+
// Build service container name
121+
outCtxt.Hostname = p.ContainerName(serviceName)
122+
123+
logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName))
124+
serviceComposeConfig, err := p.Config(compose.CommandOptions{Env: env})
125+
if err != nil {
126+
return nil, errors.Wrap(err, "could not get Docker Compose configuration for service")
127+
}
128+
129+
s := serviceComposeConfig.Services[serviceName]
130+
outCtxt.Ports = make([]int, len(s.Ports))
131+
for idx, port := range s.Ports {
132+
outCtxt.Ports[idx] = port.InternalPort
133+
}
134+
135+
// Shortcut to first port for convenience
136+
if len(outCtxt.Ports) > 0 {
137+
outCtxt.Port = outCtxt.Ports[0]
138+
}
139+
140+
outCtxt.Agent.Host.NamePrefix = inCtxt.Name
141+
service.ctxt = outCtxt
142+
return &service, nil
143+
}
144+
145+
func (d *CustomAgentDeployer) loadComposeDefinitions() ([]string, error) {
146+
locationManager, err := locations.NewLocationManager()
147+
if err != nil {
148+
return nil, errors.Wrap(err, "can't locate Docker Compose file for Custom Agent deployer")
149+
}
150+
return []string{locationManager.DockerCustomAgentDeployerYml(), d.cfg}, nil
151+
}

internal/testrunner/runners/system/servicedeployer/factory.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,13 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) {
5151
}
5252
return NewDockerComposeServiceDeployer([]string{dockerComposeYMLPath}, sv)
5353
}
54+
case "agent":
55+
customAgentCfgYMLPath := filepath.Join(serviceDeployerPath, "custom-agent.yml")
56+
if _, err := os.Stat(customAgentCfgYMLPath); err != nil {
57+
return nil, errors.Wrap(err, "can't find expected file custom-agent.yml")
58+
}
59+
return NewCustomAgentDeployer(customAgentCfgYMLPath)
60+
5461
case "tf":
5562
if _, err := os.Stat(serviceDeployerPath); err == nil {
5663
return NewTerraformServiceDeployer(serviceDeployerPath)
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
dependencies:
2+
ecs:
3+
reference: [email protected]

0 commit comments

Comments
 (0)