Fix validation of histograms and fields with object_type (#939)

Author: jsoriano

internal/fields/model.go

@@ -18,6 +18,7 @@ type FieldDefinition struct {
 	Name           string            `yaml:"name"`
 	Description    string            `yaml:"description"`
 	Type           string            `yaml:"type"`
+	ObjectType     string            `yaml:"object_type"`
 	Value          string            `yaml:"value"` // The value to associate with a constant_keyword field.
 	AllowedValues  AllowedValues     `yaml:"allowed_values"`
 	ExpectedValues []string          `yaml:"expected_values"`
@@ -41,6 +42,9 @@ func (orig *FieldDefinition) Update(fd FieldDefinition) {
 	if fd.Type != "" {
 		orig.Type = fd.Type
 	}
+	if fd.ObjectType != "" {
+		orig.ObjectType = fd.ObjectType
+	}
 	if fd.Value != "" {
 		orig.Value = fd.Value
 	}

internal/fields/validate.go

@@ -350,14 +350,44 @@ func compareKeys(key string, def FieldDefinition, searchedKey string) bool {
 		return true
 	}
 
-	// Workaround for potential geo_point, as "lon" and "lat" fields are not present in field definitions.
-	// Unfortunately we have to assume that imported field could be a geo_point (nasty workaround).
+	// Workaround for potential subfields of certain types as geo_point or histogram.
 	if len(searchedKey) > j {
-		if def.Type == "geo_point" || def.External != "" {
-			extraPart := searchedKey[j:]
-			if extraPart == ".lon" || extraPart == ".lat" {
-				return true
-			}
+		extraPart := searchedKey[j:]
+		if validSubField(def, extraPart) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// validSubField checks if the extra part that didn't match with any field definition,
+// matches with the possible sub field of complex fields like geo_point or histogram.
+func validSubField(def FieldDefinition, extraPart string) bool {
+	fieldType := def.Type
+	if def.Type == "object" && def.ObjectType != "" {
+		fieldType = def.ObjectType
+	}
+
+	subFields := []string{".lat", ".lon", ".values", ".counts"}
+	perType := map[string][]string{
+		"geo_point": subFields[0:2],
+		"histogram": subFields[2:4],
+	}
+
+	allowed, found := perType[fieldType]
+	if !found {
+		if def.External != "" {
+			// An unresolved external field could be anything.
+			allowed = subFields
+		} else {
+			return false
+		}
+	}
+
+	for _, a := range allowed {
+		if a == extraPart {
+			return true
 		}
 	}
 

internal/fields/validate_test.go

@@ -525,12 +525,42 @@ func TestCompareKeys(t *testing.T) {
 			searchedKey: "example.geo.lat",
 			expected:    true,
 		},
+		{
+			key:         "example.*",
+			def:         FieldDefinition{Type: "object", ObjectType: "geo_point"},
+			searchedKey: "example.geo.lon",
+			expected:    true,
+		},
 		{
 			key:         "example.*",
 			def:         FieldDefinition{Type: "geo_point"},
 			searchedKey: "example.geo.foo",
 			expected:    false,
 		},
+		{
+			key:         "example.histogram",
+			def:         FieldDefinition{Type: "histogram"},
+			searchedKey: "example.histogram.counts",
+			expected:    true,
+		},
+		{
+			key:         "example.*",
+			def:         FieldDefinition{Type: "histogram"},
+			searchedKey: "example.histogram.counts",
+			expected:    true,
+		},
+		{
+			key:         "example.*",
+			def:         FieldDefinition{Type: "histogram"},
+			searchedKey: "example.histogram.values",
+			expected:    true,
+		},
+		{
+			key:         "example.*",
+			def:         FieldDefinition{Type: "histogram"},
+			searchedKey: "example.histogram.foo",
+			expected:    false,
+		},
 	}
 
 	for _, c := range cases {

test/packages/other/fields_tests/data_stream/first/fields/histogram-fields.yml

@@ -0,0 +1,3 @@
+- name: service.status.*.histogram
+  type: object
+  object_type: histogram

test/packages/other/fields_tests/data_stream/first/sample_event.json

@@ -4,5 +4,23 @@
         "lon": "2.0"
     },
     "destination.geo.location.lat": 3.0,
-    "destination.geo.location.lon": 4.0
+    "destination.geo.location.lon": 4.0,
+    "service.status.duration.histogram": {
+        "counts": [
+            8,
+            17,
+            8,
+            7,
+            6,
+            2
+        ],
+        "values": [
+            0.1,
+            0.25,
+            0.35,
+            0.4,
+            0.45,
+            0.5
+        ]
+    }
 }

test/packages/other/fields_tests/docs/README.md

@@ -9,7 +9,25 @@ An example event for `first` looks as following:
         "lon": "2.0"
     },
     "destination.geo.location.lat": 3.0,
-    "destination.geo.location.lon": 4.0
+    "destination.geo.location.lon": 4.0,
+    "service.status.duration.histogram": {
+        "counts": [
+            8,
+            17,
+            8,
+            7,
+            6,
+            2
+        ],
+        "values": [
+            0.1,
+            0.25,
+            0.35,
+            0.4,
+            0.45,
+            0.5
+        ]
+    }
 }
 ```
 
@@ -22,4 +40,5 @@ An example event for `first` looks as following:
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
 | destination.geo.location | Longitude and latitude. | geo_point |
+| service.status.\*.histogram |  | object |
 | source.geo.location | Longitude and latitude. | geo_point |