Add flag to enable logsdb mode in logs data streams (#1939)

Add a new profile parameter, `stack.logsdb_enabled`, to enable the feature flag that
enables the logs index mode in all logs data streams (elastic/elasticsearch#108762),
in stacks that support it.

Author: jsoriano

README.md

@@ -650,6 +650,8 @@ The following settings are available per profile:
   an absolute path, out of the `.elastic-package` directory.
 * `stack.kibana_http2_enabled` can be used to control if HTTP/2 should be used in versions of
   kibana that support it. Defaults to true.
+* `stack.logsdb_enabled` can be set to true to activate the feature flag in Elasticsearch that
+  enables logs index mode in all data streams that support it. Defaults to false.
 * `stack.logstash_enabled` can be set to true to start Logstash and configure it as the
   default output for tests using elastic-package. Supported only by the compose provider.
   Defaults to false.

internal/profile/_static/config.yml.example

@@ -14,6 +14,10 @@
 # Flag to enable apm-server in elastic-package stack profile config
 # stack.apm_enabled: true
 
+## Logs DB
+# Flag to enable the logs index mode in logs data stream.
+# stack.logsdb_enabled: true
+
 ## Enable logstash for testing
 # Flag to enable logstash in elastic-package stack profile config
 # stack.logstash_enabled: true

internal/stack/_static/elasticsearch.yml.tmpl

@@ -13,7 +13,12 @@ xpack.security.http.ssl.certificate: "certs/cert.pem"
 
 ingest.geoip.downloader.enabled: false
 
-{{ $version := fact "elasticsearch_version" }}
+{{- $version := fact "elasticsearch_version" -}}
+{{- $logsdb_enabled := fact "logsdb_enabled" -}}
+{{ if (and (eq $logsdb_enabled "true") (not (semverLessThan $version "8.15.0-SNAPSHOT"))) }}
+cluster.logsdb.enabled: true
+{{- end -}}
+
 {{ if semverLessThan $version "8.0.0" }}
 script.max_compilations_rate: "use-context"
 script.context.template.max_compilations_rate: "unlimited"

internal/stack/resources.go

@@ -60,6 +60,7 @@ const (
 	configAPMEnabled         = "stack.apm_enabled"
 	configGeoIPDir           = "stack.geoip_dir"
 	configKibanaHTTP2Enabled = "stack.kibana_http2_enabled"
+	configLogsDBEnabled      = "stack.logsdb_enabled"
 	configLogstashEnabled    = "stack.logstash_enabled"
 	configSelfMonitorEnabled = "stack.self_monitor_enabled"
 )
@@ -158,12 +159,13 @@ func applyResources(profile *profile.Profile, stackVersion string) error {
 		"username": elasticsearchUsername,
 		"password": elasticsearchPassword,
 
+		"agent_publish_ports":  strings.Join(agentPorts, ","),
 		"apm_enabled":          profile.Config(configAPMEnabled, "false"),
 		"geoip_dir":            profile.Config(configGeoIPDir, "./ingest-geoip"),
+		"kibana_http2_enabled": profile.Config(configKibanaHTTP2Enabled, "true"),
+		"logsdb_enabled":       profile.Config(configLogsDBEnabled, "false"),
 		"logstash_enabled":     profile.Config(configLogstashEnabled, "false"),
 		"self_monitor_enabled": profile.Config(configSelfMonitorEnabled, "false"),
-		"agent_publish_ports":  strings.Join(agentPorts, ","),
-		"kibana_http2_enabled": profile.Config(configKibanaHTTP2Enabled, "true"),
 	})
 
 	if err := os.MkdirAll(stackDir, 0755); err != nil {

internal/testrunner/runners/system/tester.go

@@ -607,60 +607,38 @@ func (r *tester) runTestPerVariant(ctx context.Context, result *testrunner.Resul
 	return partial, nil
 }
 
-func (r *tester) isSyntheticsEnabled(ctx context.Context, dataStream, componentTemplatePackage string) (bool, error) {
-	resp, err := r.esAPI.Cluster.GetComponentTemplate(
-		r.esAPI.Cluster.GetComponentTemplate.WithContext(ctx),
-		r.esAPI.Cluster.GetComponentTemplate.WithName(componentTemplatePackage),
+func (r *tester) isSyntheticsEnabled(ctx context.Context, dataStreamName string) (bool, error) {
+	resp, err := r.esAPI.Indices.SimulateIndexTemplate(dataStreamName,
+		r.esAPI.Indices.SimulateIndexTemplate.WithContext(ctx),
 	)
 	if err != nil {
-		return false, fmt.Errorf("could not get component template %s from data stream %s: %w", componentTemplatePackage, dataStream, err)
+		return false, fmt.Errorf("could not simulate index template for %s: %w", dataStreamName, err)
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode == http.StatusNotFound {
-		// @package component template doesn't exist before 8.2. On these versions synthetics was not supported
-		// in any case, so just return false.
-		logger.Debugf("no component template %s found for data stream %s", componentTemplatePackage, dataStream)
-		return false, nil
-	}
 	if resp.IsError() {
-		return false, fmt.Errorf("could not get component template %s for data stream %s: %s", componentTemplatePackage, dataStream, resp.String())
+		return false, fmt.Errorf("could not simulate index template for %s: %s", dataStreamName, resp.String())
 	}
 
 	var results struct {
-		ComponentTemplates []struct {
-			Name              string `json:"name"`
-			ComponentTemplate struct {
-				Template struct {
-					Mappings struct {
-						Source *struct {
-							Mode string `json:"mode"`
-						} `json:"_source,omitempty"`
-					} `json:"mappings"`
-				} `json:"template"`
-			} `json:"component_template"`
-		} `json:"component_templates"`
+		Template struct {
+			Mappings struct {
+				Source *struct {
+					Mode string `json:"mode"`
+				} `json:"_source,omitempty"`
+			} `json:"mappings"`
+		} `json:"template"`
 	}
 
 	if err := json.NewDecoder(resp.Body).Decode(&results); err != nil {
-		return false, fmt.Errorf("could not decode search results response: %w", err)
-	}
-
-	if len(results.ComponentTemplates) == 0 {
-		logger.Debugf("no component template %s found for data stream %s", componentTemplatePackage, dataStream)
-		return false, nil
+		return false, fmt.Errorf("could not decode index template simulation response: %w", err)
 	}
-	if len(results.ComponentTemplates) != 1 {
-		return false, fmt.Errorf("ambiguous response, expected one component template for %s, found %d", componentTemplatePackage, len(results.ComponentTemplates))
-	}
-
-	template := results.ComponentTemplates[0]
 
-	if template.ComponentTemplate.Template.Mappings.Source == nil {
+	if results.Template.Mappings.Source == nil {
 		return false, nil
 	}
 
-	return template.ComponentTemplate.Template.Mappings.Source.Mode == "synthetic", nil
+	return results.Template.Mappings.Source.Mode == "synthetic", nil
 }
 
 type hits struct {
@@ -962,11 +940,6 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		dataStreamDataset,
 		ds.Namespace,
 	)
-	componentTemplatePackage := fmt.Sprintf(
-		"%s-%s@package",
-		ds.Inputs[0].Streams[0].DataStream.Type,
-		dataStreamDataset,
-	)
 
 	r.cleanTestScenarioHandler = func(ctx context.Context) error {
 		logger.Debugf("Deleting data stream for testing %s", scenario.dataStream)
@@ -1130,10 +1103,10 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		return nil, testrunner.ErrTestCaseFailed{Reason: fmt.Sprintf("could not find hits in %s data stream", scenario.dataStream)}
 	}
 
-	logger.Debugf("check whether or not synthetics is enabled (component template %s)...", componentTemplatePackage)
-	scenario.syntheticEnabled, err = r.isSyntheticsEnabled(ctx, scenario.dataStream, componentTemplatePackage)
+	logger.Debugf("check whether or not synthetics is enabled (data stream %s)...", scenario.dataStream)
+	scenario.syntheticEnabled, err = r.isSyntheticsEnabled(ctx, scenario.dataStream)
 	if err != nil {
-		return nil, fmt.Errorf("failed to check if synthetic source is enabled: %w", err)
+		return nil, fmt.Errorf("failed to check if synthetic source is enabled for data stream %s: %w", scenario.dataStream, err)
 	}
 	logger.Debugf("data stream %s has synthetics enabled: %t", scenario.dataStream, scenario.syntheticEnabled)
 

tools/readme/readme.md.tmpl

@@ -192,6 +192,8 @@ The following settings are available per profile:
   an absolute path, out of the `.elastic-package` directory.
 * `stack.kibana_http2_enabled` can be used to control if HTTP/2 should be used in versions of
   kibana that support it. Defaults to true.
+* `stack.logsdb_enabled` can be set to true to activate the feature flag in Elasticsearch that
+  enables logs index mode in all data streams that support it. Defaults to false.
 * `stack.logstash_enabled` can be set to true to start Logstash and configure it as the
   default output for tests using elastic-package. Supported only by the compose provider.
   Defaults to false.