Add logstash to stack as an optional service (#1458)

Logstash is added as an optional service as part of 'elastic-package stack' command. It is controlled by the profile config options.

Author: bhapas

.buildkite/pipeline.trigger.integration.tests.sh

@@ -34,6 +34,7 @@ CHECK_PACKAGES_TESTS=(
     test-check-packages-with-kind
     test-check-packages-with-custom-agent
     test-check-packages-benchmarks
+    test-check-packages-with-logstash
 )
 for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "      - label: \":go: Running integration test: ${test}\""

Makefile

@@ -70,7 +70,7 @@ test-stack-command-8x:
 
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
 
-test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives
+test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash
 
 test-check-packages-with-kind:
 	PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
@@ -81,6 +81,9 @@ test-check-packages-other:
 test-check-packages-false-positives:
 	PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh
 
+test-check-packages-with-logstash:
+	PACKAGE_TEST_TYPE=with-logstash ./scripts/test-check-packages.sh
+
 test-check-packages-benchmarks:
 	PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh
 

cmd/stack.go

@@ -24,6 +24,7 @@ var availableServices = map[string]struct{}{
 	"fleet-server":     {},
 	"kibana":           {},
 	"package-registry": {},
+	"logstash":         {},
 }
 
 const stackLongDescription = `Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version <version>.

docs/howto/custom_images.md

@@ -20,6 +20,7 @@ The current images that could be overwritten are:
 | Elasticsearch | ELASTICSEARCH_IMAGE_REF_OVERRIDE | elasticsearch |
 | Kibana | KIBANA_IMAGE_REF_OVERRIDE | kibana |
 | Elastic Agent | ELASTIC_AGENT_IMAGE_REF_OVERRIDE | elastic-agent |
+| Logstash | LOGSTASH_IMAGE_REF_OVERRIDE | logstash |
 
 
 For the following two examples, it will be used as example overwriting elastic-agent image.

docs/howto/system_testing.md

@@ -579,6 +579,25 @@ Example `expected_errors` file content:
 <testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>
 ```
 
+### System testing with logstash
+
+It is possible to test packages that output to Logstash which in turn publishes events to Elasticsearch.
+A profile config option `stack.logstash_enabled` has been added to profile configuration.
+
+When this profile config is enabled
+- Logstash output is added in Fleet with id `fleet-logstash-output`
+- Logstash service is created in the stack which reads from `elastic-agent` input and outputs to `elasticsearch`.
+- Logstash is also configured with `elastic-integration` plugin. Once configured to point to an Elasticsearch cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, auto-detecting the event’s data-stream and its default pipeline.
+
+A sample workflow would look like:
+
+- You can [create](https://github.com/elastic/elastic-package#elastic-package-profiles-create) a new profile / [use existing profile](https://github.com/elastic/elastic-package#elastic-package-profiles-use) to test this.
+- Navigate to `~/.elastic-package/profiles/<profilename>/`.
+- Rename `config.yml.example` to `config.yml` [ If config is not used before ]
+- Add the following line (or uncomment if present) `stack.logstash_enabled: true`
+- Run `elastic-package stack up -d -v`
+- Navigate to the package folder in integrations and run `elastic-package test system -v`
+
 ## Continuous Integration
 
 `elastic-package` runs a set of system tests on some [dummy packages](https://github.com/elastic/elastic-package/tree/main/test/packages) to ensure it's functionalities work as expected. This allows to test changes affecting package testing within `elastic-package` before merging and releasing the changes.

internal/install/application_configuration.go

@@ -29,6 +29,7 @@ const (
 	elasticAgentCompleteImageName       = "docker.elastic.co/elastic-agent/elastic-agent-complete"
 	elasticsearchImageName              = "docker.elastic.co/elasticsearch/elasticsearch"
 	kibanaImageName                     = "docker.elastic.co/kibana/kibana"
+	logstashImageName                   = "docker.elastic.co/logstash/logstash"
 
 	applicationConfigurationYmlFile = "config.yml"
 )
@@ -87,6 +88,7 @@ func (s stack) ImageRefOverridesForVersion(version string) ImageRefs {
 		ElasticAgent:  checkImageRefOverride("ELASTIC_AGENT_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.ElasticAgent, "")),
 		Elasticsearch: checkImageRefOverride("ELASTICSEARCH_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.Elasticsearch, "")),
 		Kibana:        checkImageRefOverride("KIBANA_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.Kibana, "")),
+		Logstash:      checkImageRefOverride("LOGSTASH_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.Logstash, "")),
 	}
 }
 
@@ -95,6 +97,7 @@ type ImageRefs struct {
 	ElasticAgent  string `yaml:"elastic-agent"`
 	Elasticsearch string `yaml:"elasticsearch"`
 	Kibana        string `yaml:"kibana"`
+	Logstash      string `yaml:"logstash"`
 }
 
 // AsEnv method returns key=value representation of image refs.
@@ -103,6 +106,7 @@ func (ir ImageRefs) AsEnv() []string {
 	vars = append(vars, "ELASTIC_AGENT_IMAGE_REF="+ir.ElasticAgent)
 	vars = append(vars, "ELASTICSEARCH_IMAGE_REF="+ir.Elasticsearch)
 	vars = append(vars, "KIBANA_IMAGE_REF="+ir.Kibana)
+	vars = append(vars, "LOGSTASH_IMAGE_REF="+ir.Logstash)
 	return vars
 }
 
@@ -112,6 +116,7 @@ func (ac *ApplicationConfiguration) StackImageRefs(version string) ImageRefs {
 	refs.ElasticAgent = stringOrDefault(refs.ElasticAgent, fmt.Sprintf("%s:%s", selectElasticAgentImageName(version), version))
 	refs.Elasticsearch = stringOrDefault(refs.Elasticsearch, fmt.Sprintf("%s:%s", elasticsearchImageName, version))
 	refs.Kibana = stringOrDefault(refs.Kibana, fmt.Sprintf("%s:%s", kibanaImageName, version))
+	refs.Logstash = stringOrDefault(refs.Logstash, fmt.Sprintf("%s:%s", logstashImageName, version))
 	return refs
 }
 

internal/kibana/policies.go

@@ -21,6 +21,7 @@ type Policy struct {
 	Revision           int      `json:"revision,omitempty"`
 	MonitoringEnabled  []string `json:"monitoring_enabled,omitempty"`
 	MonitoringOutputID string   `json:"monitoring_output_id,omitempty"`
+	DataOutputID       string   `json:"data_output_id,omitempty"`
 }
 
 // CreatePolicy persists the given Policy in Fleet.

internal/profile/_static/config.yml.example

@@ -10,3 +10,7 @@
 # Region where the Serverless project is going to be created
 # stack.serverless.region: aws-us-east-1
 
+## Enable logstash for testing
+# Flag to enable logstash in elastic-package stack profile config
+# stack.logstash_enabled: true
+

internal/profile/_testdata/config.yml

@@ -1,5 +1,6 @@
 # An expected setting.
 stack.geoip_dir: "/home/foo/Documents/ingest-geoip"
+stack.logstash_enabled: true
 
 # An empty string, should exist, but return empty.
 other.empty: ""

internal/profile/config_test.go

@@ -47,6 +47,11 @@ func TestLoadProfileConfig(t *testing.T) {
 			expected: "false",
 			found:    true,
 		},
+		{
+			name:     "stack.logstash_enabled",
+			expected: "true",
+			found:    true,
+		},
 		{
 			name:  "not.present",
 			found: false,