Add synthetic mode as default when creating a datastream (#1261)

Added questions to set synthetic or time series modes in the data stream
manifest when using the CLI command elastic-package create data-stream

Author: mrodm

cmd/create_data_stream.go

@@ -19,9 +19,11 @@ const createDataStreamLongDescription = `Use this command to create a new data s
 The command can extend the package with a new data stream using embedded data stream template and wizard.`
 
 type newDataStreamAnswers struct {
-	Name  string
-	Title string
-	Type  string
+	Name                   string
+	Title                  string
+	Type                   string
+	SyntheticAndTimeSeries bool
+	Synthetic              bool
 }
 
 func createDataStreamCommandAction(cmd *cobra.Command, args []string) error {
@@ -68,6 +70,40 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error {
 		return errors.Wrap(err, "prompt failed")
 	}
 
+	if answers.Type == "metrics" {
+		qs := []*survey.Question{
+			{
+				Name: "syntheticAndTimeSeries",
+				Prompt: &survey.Confirm{
+					Message: "Enable time series and synthetic source?",
+					Default: true,
+				},
+				Validate: survey.Required,
+			},
+		}
+		err = survey.Ask(qs, &answers)
+		if err != nil {
+			return errors.Wrap(err, "prompt failed")
+		}
+
+		if !answers.SyntheticAndTimeSeries {
+			qs := []*survey.Question{
+				{
+					Name: "synthetic",
+					Prompt: &survey.Confirm{
+						Message: "Enable synthetic source?",
+						Default: true,
+					},
+					Validate: survey.Required,
+				},
+			}
+			err = survey.Ask(qs, &answers)
+			if err != nil {
+				return errors.Wrap(err, "prompt failed")
+			}
+		}
+	}
+
 	descriptor := createDataStreamDescriptorFromAnswers(answers, packageRoot)
 	err = archetype.CreateDataStream(descriptor)
 	if err != nil {
@@ -79,12 +115,27 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error {
 }
 
 func createDataStreamDescriptorFromAnswers(answers newDataStreamAnswers, packageRoot string) archetype.DataStreamDescriptor {
+	manifest := packages.DataStreamManifest{
+		Name:  answers.Name,
+		Title: answers.Title,
+		Type:  answers.Type,
+	}
+
+	if !answers.SyntheticAndTimeSeries && !answers.Synthetic {
+		return archetype.DataStreamDescriptor{
+			Manifest:    manifest,
+			PackageRoot: packageRoot,
+		}
+	}
+	elasticsearch := packages.Elasticsearch{
+		SourceMode: "synthetic",
+	}
+	if answers.SyntheticAndTimeSeries {
+		elasticsearch.IndexMode = "time_series"
+	}
+	manifest.Elasticsearch = &elasticsearch
 	return archetype.DataStreamDescriptor{
-		Manifest: packages.DataStreamManifest{
-			Name:  answers.Name,
-			Title: answers.Title,
-			Type:  answers.Type,
-		},
+		Manifest:    manifest,
 		PackageRoot: packageRoot,
 	}
 }

internal/packages/archetype/data_stream_manifest.go

@@ -26,5 +26,14 @@ streams:{{if eq .Manifest.Type "logs" }}
         type: text
         title: Period
         default: 10s
+{{ if .Manifest.Elasticsearch }}
+elasticsearch:
+{{ if .Manifest.Elasticsearch.SourceMode }}
+  source_mode: {{ .Manifest.Elasticsearch.SourceMode }}
+{{- end}}
+{{ if .Manifest.Elasticsearch.IndexMode }}
+  index_mode: {{ .Manifest.Elasticsearch.IndexMode }}
+{{- end}}
+{{- end}}
 {{- end}}
 `

internal/packages/packages.go

@@ -132,22 +132,26 @@ type PackageManifest struct {
 	Categories      []string         `config:"categories" json:"categories" yaml:"categories"`
 }
 
+type Elasticsearch struct {
+	IndexTemplate *struct {
+		IngestPipeline *struct {
+			Name string `config:"name" json:"name" yaml:"name"`
+		} `config:"ingest_pipeline" json:"ingest_pipeline" yaml:"ingest_pipeline"`
+	} `config:"index_template" json:"index_template" yaml:"index_template"`
+	SourceMode string `config:"source_mode" json:"source_mode" yaml:"source_mode"`
+	IndexMode  string `config:"index_mode" json:"index_mode" yaml:"index_mode"`
+}
+
 // DataStreamManifest represents the structure of a data stream's manifest
 type DataStreamManifest struct {
-	Name          string `config:"name" json:"name" yaml:"name"`
-	Title         string `config:"title" json:"title" yaml:"title"`
-	Type          string `config:"type" json:"type" yaml:"type"`
-	Dataset       string `config:"dataset" json:"dataset" yaml:"dataset"`
-	Hidden        bool   `config:"hidden" json:"hidden" yaml:"hidden"`
-	Release       string `config:"release" json:"release" yaml:"release"`
-	Elasticsearch *struct {
-		IndexTemplate *struct {
-			IngestPipeline *struct {
-				Name string `config:"name" json:"name" yaml:"name"`
-			} `config:"ingest_pipeline" json:"ingest_pipeline" yaml:"ingest_pipeline"`
-		} `config:"index_template" json:"index_template" yaml:"index_template"`
-	} `config:"elasticsearch" json:"elasticsearch" yaml:"elasticsearch"`
-	Streams []struct {
+	Name          string         `config:"name" json:"name" yaml:"name"`
+	Title         string         `config:"title" json:"title" yaml:"title"`
+	Type          string         `config:"type" json:"type" yaml:"type"`
+	Dataset       string         `config:"dataset" json:"dataset" yaml:"dataset"`
+	Hidden        bool           `config:"hidden" json:"hidden" yaml:"hidden"`
+	Release       string         `config:"release" json:"release" yaml:"release"`
+	Elasticsearch *Elasticsearch `config:"elasticsearch" json:"elasticsearch" yaml:"elasticsearch"`
+	Streams       []struct {
 		Input string     `config:"input" json:"input" yaml:"input"`
 		Vars  []Variable `config:"vars" json:"vars" yaml:"vars"`
 	} `config:"streams" json:"streams" yaml:"streams"`