Fix: geo_points can be imported from ECS (#476)

Author: mtojek

internal/fields/validate.go

@@ -265,8 +265,9 @@ func compareKeys(key string, def FieldDefinition, searchedKey string) bool {
 	k = strings.ReplaceAll(k, "*", "[^.]+")
 
 	// Workaround for potential geo_point, as "lon" and "lat" fields are not present in field definitions.
-	if def.Type == "geo_point" {
-		k += "\\.(lon|lat)"
+	// Unfortunately we have to assume that imported field could be a geo_point (nasty workaround).
+	if def.Type == "geo_point" || def.External != "" {
+		k += "(\\.lon|\\.lat|)"
 	}
 
 	k = fmt.Sprintf("^%s$", k)

internal/fields/validate_test.go

@@ -223,3 +223,14 @@ func readSampleEvent(t *testing.T, path string) json.RawMessage {
 	require.NoError(t, err)
 	return c
 }
+
+func TestValidate_geo_point(t *testing.T) {
+	validator, err := CreateValidatorForDataStream("../../test/packages/fields_tests/data_stream/first")
+
+	require.NoError(t, err)
+	require.NotNil(t, validator)
+
+	e := readSampleEvent(t, "../../test/packages/fields_tests/data_stream/first/sample_event.json")
+	errs := validator.ValidateDocumentBody(e)
+	require.Empty(t, errs)
+}

test/packages/fields_tests/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

test/packages/fields_tests/_dev/build/docs/README.md

@@ -0,0 +1,5 @@
+# Fields Tests
+
+{{event "first"}}
+
+{{fields "first"}}

test/packages/fields_tests/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/0 # FIXME Replace with the real PR link

test/packages/fields_tests/data_stream/first/agent/stream/stream.yml.hbs

@@ -0,0 +1,7 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~

test/packages/fields_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,10 @@
+---
+description: Pipeline for processing sample logs
+processors:
+- set:
+    field: sample_field
+    value: "1"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

test/packages/fields_tests/data_stream/first/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/fields_tests/data_stream/first/fields/geo-fields.yml

@@ -0,0 +1,6 @@
+- name: destination.geo.location
+  external: ecs
+- name: geo.location
+  external: ecs
+- name: source.geo.location
+  external: ecs

test/packages/fields_tests/data_stream/first/manifest.yml

@@ -0,0 +1,13 @@
+title: "First"
+type: logs
+streams:
+  - input: logfile
+    title: Sample logs
+    description: Collect sample logs
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        default:
+          - /var/log/*.log