Merge pull request #85 from eranra/fix_decode_json

JSON decode: keep value types, do not convert to string

Author: eranra

pkg/pipeline/decode/decode_json.go

@@ -19,7 +19,6 @@ package decode
 
 import (
 	"encoding/json"
-	"fmt"
 	"github.com/netobserv/flowlogs2metrics/pkg/config"
 	log "github.com/sirupsen/logrus"
 )
@@ -45,9 +44,7 @@ func (c *decodeJson) Decode(in []interface{}) []config.GenericMap {
 			if v == nil {
 				continue
 			}
-			// ensure we have a string variable
-			s := fmt.Sprintf("%v", v)
-			decodedLine2[k] = s
+			decodedLine2[k] = v
 		}
 		out = append(out, decodedLine2)
 	}

pkg/pipeline/decode/decode_json_test.go

@@ -46,10 +46,23 @@ func TestDecodeJson(t *testing.T) {
 	in = append(in, inputStringErr)
 	out = decodeJson.Decode(in)
 	require.Equal(t, len(out), 3)
-	// verify that all items come back as strings
-	require.Equal(t, "12", out[0]["varInt"])
+	require.Equal(t, float64(12), out[0]["varInt"])
 	require.Equal(t, "testString", out[0]["varString"])
-	require.Equal(t, "false", out[0]["varBool"])
+	require.Equal(t, bool(false), out[0]["varBool"])
 
 	// TODO: Check for more complicated json structures
 }
+
+func TestDecodeJsonTimestamps(t *testing.T) {
+	newDecode := initNewDecodeJson(t)
+	decodeJson := newDecode.(*decodeJson)
+	inputString1 := "{\"unixTime\": 1645104030 }"
+	var in []interface{}
+	var out []config.GenericMap
+	out = decodeJson.Decode(in)
+	require.Equal(t, 0, len(out))
+	in = append(in, inputString1)
+	out = decodeJson.Decode(in)
+	require.Equal(t, len(out), 1)
+	require.Equal(t, float64(1645104030), out[0]["unixTime"])
+}

pkg/pipeline/transform/transform_generic_test.go

@@ -50,10 +50,10 @@ func getGenericExpectedOutput() config.GenericMap {
 	return config.GenericMap{
 		"SrcAddr":  "10.0.0.1",
 		"srcIP":    "10.0.0.1",
-		"SrcPort":  "11777",
+		"SrcPort":  11777,
 		"Protocol": "tcp",
 		"DstAddr":  "20.0.0.2",
-		"DstPort":  "22",
+		"DstPort":  22,
 	}
 }
 

pkg/pipeline/transform/transform_multiple_test.go

@@ -61,10 +61,10 @@ pipeline:
 func getMultipleExpectedOutput() config.GenericMap {
 	return config.GenericMap{
 		"SrcAddr2":  "10.0.0.1",
-		"SrcPort2":  "11777",
+		"SrcPort2":  11777,
 		"Protocol2": "tcp",
 		"DstAddr2":  "20.0.0.2",
-		"DstPort2":  "22",
+		"DstPort2":  22,
 	}
 }
 

pkg/pipeline/transform/transform_network_test.go

@@ -96,10 +96,10 @@ func getExpectedOutput() config.GenericMap {
 	return config.GenericMap{
 		"level":                           "error",
 		"protocol":                        "tcp",
-		"protocol_num":                    "6",
+		"protocol_num":                    6,
 		"message":                         "test message",
 		"dstIP":                           "20.0.0.2",
-		"dstPort":                         "22",
+		"dstPort":                         22,
 		"srcIP":                           "10.0.0.1",
 		"subnet16SrcIP":                   "10.0.0.0/16",
 		"subnet24SrcIP":                   "10.0.0.0/24",
@@ -109,7 +109,7 @@ func getExpectedOutput() config.GenericMap {
 		"smaller_than_10_Evaluate":        true,
 		"service":                         "ssh",
 		"service_protocol_num":            "ssh",
-		"srcPort":                         "11777",
+		"srcPort":                         11777,
 		"8888IP":                          "8.8.8.8",
 		"8888IP_location_CountryName":     "US",
 		"8888IP_location_CountryLongName": "United States of America",

pkg/test/utils.go

@@ -31,16 +31,16 @@ func GetIngestMockEntry(missingKey bool) config.GenericMap {
 		"8888IP":       "8.8.8.8",
 		"emptyIP":      "",
 		"level":        "error",
-		"srcPort":      "11777",
+		"srcPort":      11777,
 		"protocol":     "tcp",
-		"protocol_num": "6",
+		"protocol_num": 6,
 		"value":        "7",
 		"message":      "test message",
 	}
 
 	if !missingKey {
 		entry["dstIP"] = "20.0.0.2"
-		entry["dstPort"] = "22"
+		entry["dstPort"] = 22
 	}
 
 	return entry