Skip to content

Commit 9be83fe

Browse files
mrodmmrodm
authored
Create input packages (#1424)
Add support to create input packages using the command elastic-package create package. It takes into account if the data stream is going to be logs or metrics too.
1 parent 1e57eff commit 9be83fe

21 files changed

+247
-321
lines changed

‎cmd/create_data_stream.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,15 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error {
4040
return errors.New("package root not found, you can only create new data stream in the package context")
4141
}
4242

43+
manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
44+
if err != nil {
45+
return fmt.Errorf("reading package manifest failed (path: %s): %w", packageRoot, err)
46+
}
47+
48+
if manifest.Type == "input" {
49+
return fmt.Errorf("data-streams are not supported in input packages")
50+
}
51+
4352
qs := []*survey.Question{
4453
{
4554
Name: "name",

‎cmd/create_package.go‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const (
2828

2929
type newPackageAnswers struct {
3030
Name string
31+
Type string
3132
Version string
3233
SourceLicense string `survey:"source_license"`
3334
Title string
@@ -36,12 +37,31 @@ type newPackageAnswers struct {
3637
KibanaVersion string `survey:"kibana_version"`
3738
ElasticSubscription string `survey:"elastic_subscription"`
3839
GithubOwner string `survey:"github_owner"`
40+
DataStreamType string `survey:"datastream_type"`
3941
}
4042

4143
func createPackageCommandAction(cmd *cobra.Command, args []string) error {
4244
cmd.Println("Create a new package")
4345

4446
qs := []*survey.Question{
47+
{
48+
Name: "type",
49+
Prompt: &survey.Select{
50+
Message: "Package type:",
51+
Options: []string{"input", "integration"},
52+
Default: "integration",
53+
},
54+
Validate: survey.Required,
55+
},
56+
}
57+
58+
var answers newPackageAnswers
59+
err := survey.Ask(qs, &answers)
60+
if err != nil {
61+
return fmt.Errorf("prompt failed: %w", err)
62+
}
63+
64+
qs = []*survey.Question{
4565
{
4666
Name: "name",
4767
Prompt: &survey.Input{
@@ -132,8 +152,23 @@ func createPackageCommandAction(cmd *cobra.Command, args []string) error {
132152
},
133153
}
134154

135-
var answers newPackageAnswers
136-
err := survey.Ask(qs, &answers)
155+
if answers.Type == "input" {
156+
inputQs := []*survey.Question{
157+
{
158+
Name: "datastream_type",
159+
Prompt: &survey.Select{
160+
Message: "Input Data Stream type:",
161+
Options: []string{"logs", "metrics"},
162+
Default: "logs",
163+
},
164+
Validate: survey.Required,
165+
},
166+
}
167+
168+
qs = append(qs, inputQs...)
169+
}
170+
171+
err = survey.Ask(qs, &answers)
137172
if err != nil {
138173
return fmt.Errorf("prompt failed: %w", err)
139174
}
@@ -153,11 +188,16 @@ func createPackageDescriptorFromAnswers(answers newPackageAnswers) archetype.Pac
153188
if answers.SourceLicense != noLicenseValue {
154189
sourceLicense = answers.SourceLicense
155190
}
191+
192+
inputDataStreamType := ""
193+
if answers.Type == "input" {
194+
inputDataStreamType = answers.DataStreamType
195+
}
156196
return archetype.PackageDescriptor{
157197
Manifest: packages.PackageManifest{
158198
Name: answers.Name,
159199
Title: answers.Title,
160-
Type: "integration",
200+
Type: answers.Type,
161201
Version: answers.Version,
162202
Source: packages.Source{
163203
License: sourceLicense,
@@ -177,5 +217,6 @@ func createPackageDescriptorFromAnswers(answers newPackageAnswers) archetype.Pac
177217
Description: answers.Description,
178218
Categories: answers.Categories,
179219
},
220+
InputDataStreamType: inputDataStreamType,
180221
}
181222
}

‎internal/packages/archetype/_static/dataStream-agent-stream.yml.tmpl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{{if eq .Manifest.Type "logs"}}paths:
2+
{{ "{{#each paths as |path i|}}" }}
3+
- {{ "{{path}}" }}
4+
{{ "{{/each}}" }}
5+
exclude_files: [".gz$"]
6+
processors:
7+
- add_locale: ~
8+
{{else}}metricsets: ["sample_metricset"]
9+
hosts:
10+
{{ "{{#each hosts}}" }}
11+
- {{ "{{this}}" }}
12+
{{ "{{/each}}" }}
13+
period: {{ "{{period}}" }}
14+
{{end}}

‎internal/packages/archetype/_static/dataStream-elasticsearch-ingest-pipeline.yml.tmpl‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
description: Pipeline for processing sample logs
3+
processors:
4+
- set:
5+
field: sample_field
6+
value: "1"
7+
on_failure:
8+
- set:
9+
field: error.message
10+
value: {{ "'{{ _ingest.on_failure_message }}'" }}

internal/packages/archetype/data_stream_manifest.go renamed to internal/packages/archetype/_static/dataStream-manifest.yml.tmpl

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,4 @@
1-
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2-
// or more contributor license agreements. Licensed under the Elastic License;
3-
// you may not use this file except in compliance with the Elastic License.
4-
5-
package archetype
6-
7-
const dataStreamManifestTemplate = `title: "{{.Manifest.Title}}"
1+
title: "{{.Manifest.Title}}"
82
type: {{.Manifest.Type}}
93
streams:{{if eq .Manifest.Type "logs" }}
104
- input: logfile
@@ -35,5 +29,4 @@ elasticsearch:
3529
index_mode: {{ .Manifest.Elasticsearch.IndexMode }}
3630
{{- end}}
3731
{{- end}}
38-
{{- end}}
39-
`
32+
{{- end}}

‎internal/packages/archetype/_static/fields-base.yml.tmpl‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
- name: data_stream.type
2+
type: constant_keyword
3+
description: Data stream type.
4+
- name: data_stream.dataset
5+
type: constant_keyword
6+
description: Data stream dataset.
7+
- name: data_stream.namespace
8+
type: constant_keyword
9+
description: Data stream namespace.
10+
- name: '@timestamp'
11+
type: date
12+
description: Event timestamp.

‎internal/packages/archetype/_static/input-package-agent-config.yml.tmpl‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
data_stream:
2+
dataset: {{ "{{data_stream.dataset}}" }}
3+
{{if eq .InputDataStreamType "logs"}}paths:
4+
{{ "{{#each paths as |path i|}}" }}
5+
- {{"{{path}}"}}
6+
{{ "{{/each}}" }}
7+
exclude_files: [".gz$"]
8+
processors:
9+
- add_locale: ~
10+
{{else}}metricsets: ["sample_metricset"]
11+
hosts:
12+
{{ "{{#each hosts}}" }}
13+
- {{ "{{this}}" }}
14+
{{ "{{/each}}" }}
15+
period: {{ "{{period}}" }}
16+
{{end}}

‎internal/packages/archetype/_static/package-changelog.yml.tmpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# newer versions go on top
2+
- version: "{{.Manifest.Version}}"
3+
changes:
4+
- description: Initial draft of the package
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link

internal/packages/archetype/package_docs_readme.go renamed to internal/packages/archetype/_static/package-docs-readme.md.tmpl

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,4 @@
1-
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2-
// or more contributor license agreements. Licensed under the Elastic License;
3-
// you may not use this file except in compliance with the Elastic License.
4-
5-
package archetype
6-
7-
const packageDocsReadme = `<!-- Use this template language as a starting point, replacing {placeholder text} with details about the integration. -->
1+
<!-- Use this template language as a starting point, replacing {placeholder text} with details about the integration. -->
82
<!-- Find more detailed documentation guidelines in https://github.com/elastic/integrations/blob/main/docs/documentation_guidelines.md -->
93

104
# {{.Manifest.Title}}
@@ -57,12 +51,12 @@ For step-by-step instructions on how to set up an integration, see the
5751
<!-- Repeat for each data stream of the current type -->
5852
<!-- ### {Data stream name}
5953

60-
The ` + "`{data stream name}`" + ` data stream provides events from {source} of the following types: {list types}. -->
54+
The `{data stream name}` data stream provides events from {source} of the following types: {list types}. -->
6155

6256
<!-- Optional -->
6357
<!-- #### Example
6458

65-
An example event for ` + "`{data stream name}`" + ` looks as following:
59+
An example event for `{data stream name}` looks as following:
6660

6761
{code block with example} -->
6862

@@ -76,15 +70,15 @@ An example event for ` + "`{data stream name}`" + ` looks as following:
7670
<!-- Repeat for each data stream of the current type -->
7771
<!-- ### {Data stream name}
7872

79-
The ` + "`{data stream name}`" + ` data stream provides events from {source} of the following types: {list types}. -->
73+
The `{data stream name}` data stream provides events from {source} of the following types: {list types}. -->
8074

8175
<!-- Optional -->
8276
<!-- #### Example
8377

84-
An example event for ` + "`{data stream name}`" + ` looks as following:
78+
An example event for `{data stream name}` looks as following:
8579

8680
{code block with example} -->
8781

8882
<!-- #### Exported fields
8983

90-
{insert table} -->`
84+
{insert table} -->

internal/packages/archetype/package_manifest.go renamed to internal/packages/archetype/_static/package-manifest.yml.tmpl

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,4 @@
1-
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2-
// or more contributor license agreements. Licensed under the Elastic License;
3-
// you may not use this file except in compliance with the Elastic License.
4-
5-
package archetype
6-
7-
const packageManifestTemplate = `format_version: 2.8.0
1+
format_version: 2.10.0
82
name: {{.Manifest.Name}}
93
title: "{{.Manifest.Title}}"
104
version: {{.Manifest.Version}}
@@ -31,13 +25,50 @@ icons:
3125
size: 32x32
3226
type: image/svg+xml
3327
policy_templates:
28+
{{- if eq .Manifest.Type "integration" }}
3429
- name: sample
3530
title: Sample logs
3631
description: Collect sample logs
3732
inputs:
3833
- type: logfile
3934
title: Collect sample logs from instances
4035
description: Collecting sample logs
36+
{{ else -}}
37+
{{ if eq .InputDataStreamType "logs"}}
38+
- name: sample
39+
type: logs
40+
title: Sample logs
41+
description: Collect sample logs
42+
input: logfile
43+
template_path: input.yml.hbs
44+
vars:
45+
- name: paths
46+
type: text
47+
title: Paths
48+
multi: true
49+
default:
50+
- /var/log/*.log
51+
{{ else }}
52+
- name: sample
53+
type: metrics
54+
title: Sample metrics
55+
description: Collect sample metrics
56+
input: sample/metrics
57+
template_path: input.yml.hbs
58+
vars:
59+
- name: period
60+
type: text
61+
title: Period
62+
default: 10s
63+
- name: hosts
64+
type: text
65+
title: Hosts
66+
multi: true
67+
required: true
68+
show_user: true
69+
default:
70+
- localhost
71+
{{ end }}
72+
{{ end -}}
4173
owner:
42-
github: {{.Manifest.Owner.Github}}
43-
`
74+
github: {{.Manifest.Owner.Github}}

0 commit comments

Comments
 (0)