Validate expected fields normalization (#765)

ECS fields can include normalization rules, indicated by the `normalize` parameter.

By now only the `array` rule exist, that indicates that values should be included in the
field as an array, even if there is a single element.

Validate this on test results for packages with `format_version` >= 2.0.0.

Author: jsoriano

internal/common/helpers.go

@@ -22,3 +22,17 @@ func StringSliceContains(slice []string, s string) bool {
 	}
 	return false
 }
+
+// StringSlicesUnion joins multiple slices and returns an slice with the distinct
+// elements of all of them.
+func StringSlicesUnion(slices ...[]string) (result []string) {
+	for _, slice := range slices {
+		for _, elem := range slice {
+			if StringSliceContains(result, elem) {
+				continue
+			}
+			result = append(result, elem)
+		}
+	}
+	return
+}

internal/common/helpers_test.go

@@ -39,3 +39,22 @@ func TestStringSliceContains(t *testing.T) {
 		assert.Equalf(t, c.expected, found, "checking if slice %v contains '%s'", c.slice, c.s)
 	}
 }
+
+func TestStringSlicesUnion(t *testing.T) {
+	cases := []struct {
+		slices   [][]string
+		expected []string
+	}{
+		{nil, nil},
+		{[][]string{{"foo", "bar"}, nil}, []string{"foo", "bar"}},
+		{[][]string{nil, {"foo", "bar"}}, []string{"foo", "bar"}},
+		{[][]string{{"foo", "bar"}, {"foo", "bar"}}, []string{"foo", "bar"}},
+		{[][]string{{"foo", "baz"}, {"foo", "bar"}}, []string{"foo", "bar", "baz"}},
+		{[][]string{{"foo", "bar"}, {"foo", "baz"}}, []string{"foo", "bar", "baz"}},
+	}
+
+	for _, c := range cases {
+		result := StringSlicesUnion(c.slices...)
+		assert.ElementsMatch(t, c.expected, result)
+	}
+}

internal/configuration/locations/locations.go

@@ -40,7 +40,7 @@ var (
 	dockerCustomAgentDeployerDir = filepath.Join(deployerDir, "docker_custom_agent")
 )
 
-//LocationManager maintains an instance of a config path location
+// LocationManager maintains an instance of a config path location
 type LocationManager struct {
 	stackPath string
 }

internal/fields/dependency_manager.go

@@ -262,6 +262,10 @@ func transformImportedField(fd FieldDefinition) common.MapStr {
 		m["doc_values"] = *fd.DocValues
 	}
 
+	if len(fd.Normalize) > 0 {
+		m["normalize"] = fd.Normalize
+	}
+
 	if len(fd.MultiFields) > 0 {
 		var t []common.MapStr
 		for _, f := range fd.MultiFields {

internal/fields/dependency_manager_test.go

@@ -200,6 +200,51 @@ func TestDependencyManagerInjectExternalFields(t *testing.T) {
 			changed: true,
 			valid:   true,
 		},
+		{
+			title: "array field",
+			defs: []common.MapStr{
+				{
+					"name":     "host.ip",
+					"external": "test",
+				},
+			},
+			result: []common.MapStr{
+				{
+					"name":        "host.ip",
+					"type":        "ip",
+					"description": "Host ip addresses.",
+					"normalize": []string{
+						"array",
+					},
+				},
+			},
+			changed: true,
+			valid:   true,
+		},
+		{
+			title: "array field override",
+			defs: []common.MapStr{
+				{
+					"name":     "container.id",
+					"external": "test",
+					"normalize": []string{
+						"array",
+					},
+				},
+			},
+			result: []common.MapStr{
+				{
+					"name":        "container.id",
+					"type":        "keyword",
+					"description": "Container identifier.",
+					"normalize": []string{
+						"array",
+					},
+				},
+			},
+			changed: true,
+			valid:   true,
+		},
 		{
 			title: "unknown field",
 			defs: []common.MapStr{
@@ -335,6 +380,14 @@ func TestDependencyManagerInjectExternalFields(t *testing.T) {
 			Index:       &indexFalse,
 			DocValues:   &indexFalse,
 		},
+		{
+			Name:        "host.ip",
+			Description: "Host ip addresses.",
+			Type:        "ip",
+			Normalize: []string{
+				"array",
+			},
+		},
 		{
 			Name:        "source.mac",
 			Description: "MAC address of the source.",

internal/fields/model.go

@@ -28,6 +28,7 @@ type FieldDefinition struct {
 	External       string            `yaml:"external"`
 	Index          *bool             `yaml:"index"`
 	DocValues      *bool             `yaml:"doc_values"`
+	Normalize      []string          `yaml:"normalize,omitempty"`
 	Fields         FieldDefinitions  `yaml:"fields,omitempty"`
 	MultiFields    []FieldDefinition `yaml:"multi_fields,omitempty"`
 }
@@ -73,6 +74,10 @@ func (orig *FieldDefinition) Update(fd FieldDefinition) {
 		orig.DocValues = fd.DocValues
 	}
 
+	if len(fd.Normalize) > 0 {
+		orig.Normalize = common.StringSlicesUnion(orig.Normalize, fd.Normalize)
+	}
+
 	if len(fd.Fields) > 0 {
 		orig.Fields = updateFields(orig.Fields, fd.Fields)
 	}

internal/fields/testdata/fields/fields.yml

@@ -13,3 +13,7 @@
       value: correct
     - name: ip_address
       type: ip
+- name: container.image.tag
+  type: keyword
+  normalize:
+  - array

internal/fields/testdata/invalid-array-normalization.json

@@ -0,0 +1,3 @@
+{
+  "container.image.tag": "sometag"
+}

internal/fields/testdata/valid-array-normalization.json

@@ -0,0 +1,3 @@
+{
+  "container.image.tag": ["sometag"]
+}

internal/fields/validate.go

@@ -15,6 +15,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/Masterminds/semver"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
 
@@ -32,6 +33,9 @@ type Validator struct {
 	// FieldDependencyManager resolves references to external fields
 	FieldDependencyManager *DependencyManager
 
+	// SpecVersion contains the version of the spec used by the package.
+	specVersion semver.Version
+
 	defaultNumericConversion bool
 	numericKeywordFields     map[string]struct{}
 
@@ -44,6 +48,18 @@ type Validator struct {
 // ValidatorOption represents an optional flag that can be passed to  CreateValidatorForDirectory.
 type ValidatorOption func(*Validator) error
 
+// WithSpecVersion enables validation dependant of the spec version used by the package.
+func WithSpecVersion(version string) ValidatorOption {
+	return func(v *Validator) error {
+		sv, err := semver.NewVersion(version)
+		if err != nil {
+			return fmt.Errorf("invalid version %q: %v", version, err)
+		}
+		v.specVersion = *sv
+		return nil
+	}
+}
+
 // WithDefaultNumericConversion configures the validator to accept defined keyword (or constant_keyword) fields as numeric-type.
 func WithDefaultNumericConversion() ValidatorOption {
 	return func(v *Validator) error {
@@ -255,7 +271,12 @@ func (v *Validator) validateScalarElement(key string, val interface{}) error {
 		val = fmt.Sprintf("%q", val)
 	}
 
-	err := v.parseElementValue(key, *definition, val)
+	err := v.validateExpectedNormalization(*definition, val)
+	if err != nil {
+		return errors.Wrapf(err, "field %q is not normalized as expected", key)
+	}
+
+	err = v.parseElementValue(key, *definition, val)
 	if err != nil {
 		return errors.Wrap(err, "parsing field value failed")
 	}
@@ -361,6 +382,22 @@ func compareKeys(key string, def FieldDefinition, searchedKey string) bool {
 	return false
 }
 
+func (v *Validator) validateExpectedNormalization(definition FieldDefinition, val interface{}) error {
+	// Validate expected normalization starting with packages following spec v2 format.
+	if v.specVersion.LessThan(semver.MustParse("2.0.0")) {
+		return nil
+	}
+	for _, normalize := range definition.Normalize {
+		switch normalize {
+		case "array":
+			if _, isArray := val.([]interface{}); val != nil && !isArray {
+				return fmt.Errorf("expected array, found %q (%T)", val, val)
+			}
+		}
+	}
+	return nil
+}
+
 // validSubField checks if the extra part that didn't match with any field definition,
 // matches with the possible sub field of complex fields like geo_point or histogram.
 func validSubField(def FieldDefinition, extraPart string) bool {