Use fields if synthetic mode is enabled (#1283)

Use field definitions from `fields` key of the documents when synthetic
source mode is enabled. In case synthetic mode is enabled, sample_event
files are going to be generated using the same `field` definitions and they
will be formatted expanding objects, as they were when field were retrieve
from `_source` key.

Author: mrodm

internal/common/helpers.go

@@ -4,7 +4,10 @@
 
 package common
 
-import "strings"
+import (
+	"fmt"
+	"strings"
+)
 
 // TrimStringSlice removes whitespace from the beginning and end of the contents of a []string.
 func TrimStringSlice(slice []string) {
@@ -36,3 +39,21 @@ func StringSlicesUnion(slices ...[]string) (result []string) {
 	}
 	return
 }
+
+// ToStringSlice returns the list of strings from an interface variable
+func ToStringSlice(val interface{}) ([]string, error) {
+	vals, ok := val.([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("conversion error")
+	}
+
+	var s []string
+	for _, v := range vals {
+		str, ok := v.(string)
+		if !ok {
+			return nil, fmt.Errorf("conversion error")
+		}
+		s = append(s, str)
+	}
+	return s, nil
+}

internal/common/mapstr_test.go

@@ -0,0 +1,64 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package common
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMapStrGetValue(t *testing.T) {
+
+	cases := []struct {
+		title         string
+		testFile      string
+		fieldKey      string
+		expectedValue interface{}
+	}{
+		{
+			title:         "string value",
+			testFile:      "./testdata/source.json",
+			fieldKey:      "host.architecture",
+			expectedValue: "x86_64",
+		},
+		{
+			title:         "float64 value",
+			testFile:      "./testdata/source.json",
+			fieldKey:      "metricset.period",
+			expectedValue: float64(10000),
+		},
+		{
+			title:         "slice value",
+			testFile:      "./testdata/source.json",
+			fieldKey:      "tags",
+			expectedValue: []interface{}{"apache_tomcat-cache", "forwarded"},
+		},
+		{
+			title:         "map value",
+			testFile:      "./testdata/source.json",
+			fieldKey:      "data_stream",
+			expectedValue: map[string]interface{}{"dataset": "apache_tomcat.cache", "namespace": "ep", "type": "metrics"},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			b, err := os.ReadFile("./testdata/source.json")
+			require.NoError(t, err)
+
+			var given MapStr
+			err = json.Unmarshal(b, &given)
+			require.NoError(t, err)
+
+			val, err := given.GetValue(c.fieldKey)
+			assert.NoError(t, err)
+			assert.Equal(t, c.expectedValue, val)
+		})
+	}
+}

internal/common/testdata/source.json

@@ -0,0 +1,92 @@
+{
+    "@timestamp": "2023-05-30T11:42:07.923Z",
+    "agent": {
+        "ephemeral_id": "7dd994f8-d866-4068-9f29-d6c90a28738e",
+        "id": "5cad1bda-ddc8-4333-8d88-333b6ef311b5",
+        "name": "docker-fleet-agent",
+        "type": "metricbeat",
+        "version": "8.8.0"
+    },
+    "apache_tomcat": {
+        "cache": {
+            "application_name": "/",
+            "hit": {
+                "count": 15.0
+            },
+            "lookup": {
+                "count": 30.0
+            },
+            "object": {
+                "size": {
+                    "max": {
+                        "kb": 512.0
+                    }
+                }
+            },
+            "size": {
+                "current": {
+                    "kb": 19.0
+                },
+                "max": {
+                    "kb": 10240.0
+                }
+            },
+            "ttl": {
+                "ms": 5000.0
+            }
+        }
+    },
+    "data_stream": {
+        "dataset": "apache_tomcat.cache",
+        "namespace": "ep",
+        "type": "metrics"
+    },
+    "ecs": {
+        "version": "8.7.0"
+    },
+    "elastic_agent": {
+        "id": "5cad1bda-ddc8-4333-8d88-333b6ef311b5",
+        "snapshot": true,
+        "version": "8.8.0"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "category": "web",
+        "dataset": "apache_tomcat.cache",
+        "duration": 136411023,
+        "ingested": "2023-05-30T11:42:09Z",
+        "kind": "metric",
+        "module": "apache_tomcat",
+        "type": "info"
+    },
+    "host": {
+        "architecture": "x86_64",
+        "containerized": false,
+        "hostname": "docker-fleet-agent",
+        "id": "4e0d02de87bd4101b7215d3aaf5f623b",
+        "ip": "172.23.0.7",
+        "mac": "02-42-AC-17-00-07",
+        "name": "docker-fleet-agent",
+        "os": {
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "5.19.0-42-generic",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.6 LTS (Focal Fossa)"
+        }
+    },
+    "metricset": {
+        "name": "collector",
+        "period": 10000
+    },
+    "service": {
+        "address": "http://elastic-package-service-apache_tomcat-1:9090/metrics",
+        "type": "prometheus"
+    },
+    "tags": [
+        "apache_tomcat-cache",
+        "forwarded"
+    ]
+}