Skip to content

Commit 4a62eb8

Browse files
KalmanMethKalmanMeth
committed
added tests for decode module
1 parent ba66fdc commit 4a62eb8

File tree

7 files changed

+280
-10
lines changed

7 files changed

+280
-10
lines changed

‎pkg/pipeline/decode/decode_aws_test.go‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
/*
2+
* Copyright (C) 2022 IBM, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
*/
17+
18+
package decode
19+
20+
import (
21+
"bufio"
22+
jsoniter "github.com/json-iterator/go"
23+
"github.com/netobserv/flowlogs2metrics/pkg/config"
24+
"github.com/netobserv/flowlogs2metrics/pkg/test"
25+
"github.com/stretchr/testify/require"
26+
"strings"
27+
"testing"
28+
)
29+
30+
const testConfig1 = `
31+
pipeline:
32+
decode:
33+
type: aws
34+
`
35+
const testConfig2 = `
36+
pipeline:
37+
decode:
38+
type: aws
39+
aws:
40+
fields:
41+
- version
42+
- vpc-id
43+
- subnet-id
44+
- instance-id
45+
- interface-id
46+
- account-id
47+
- type
48+
- srcaddr
49+
- dstaddr
50+
- srcport
51+
- dstport
52+
- pkt-srcaddr
53+
- pkt-dstaddr
54+
- protocol
55+
- bytes
56+
- packets
57+
- start
58+
- end
59+
- action
60+
- tcp-flags
61+
- log-status
62+
`
63+
64+
const testConfigErr = `
65+
pipeline:
66+
decode:
67+
type: aws
68+
aws:
69+
fields:
70+
version
71+
vpc-id
72+
`
73+
74+
// aws version 2 standard format
75+
const input1 = `2 123456789010 eni-1235b8ca123456789 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK
76+
2 123456789010 eni-1235b8ca123456789 172.31.9.69 172.31.9.12 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK
77+
2 123456789010 eni-1235b8ca123456789 203.0.113.12 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK
78+
`
79+
80+
const inputErr = `2 123456789010 eni-1235b8ca123456789 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK
81+
2 12345 6789010 eni-1235b8ca123456789 172.31.9.69 172.31.9.12 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK
82+
2 123456789010 eni-1235b8ca123456789 203.0.113.12 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK
83+
`
84+
85+
// aws version 3 custom format
86+
const input2 = `3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK
87+
3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43416 10.0.0.62 52.213.180.42 6 376 7 1566848875 1566848933 ACCEPT 18 OK
88+
`
89+
90+
func initNewDecodeAws(t *testing.T, testConfig string) Decoder {
91+
v := test.InitConfig(t, testConfig)
92+
val := v.Get("pipeline.decode.type")
93+
require.Equal(t, "aws", val)
94+
95+
tmp := v.Get("pipeline.decode.aws")
96+
if tmp != nil {
97+
var json = jsoniter.ConfigCompatibleWithStandardLibrary
98+
b, err := json.Marshal(&tmp)
99+
require.Equal(t, err, nil)
100+
// perform initializations usually done in main.go
101+
config.Opt.PipeLine.Decode.Aws = string(b)
102+
} else {
103+
config.Opt.PipeLine.Decode.Aws = ""
104+
}
105+
106+
newDecode, err := NewDecodeAws()
107+
require.Equal(t, nil, err)
108+
return newDecode
109+
}
110+
111+
func breakIntoLines(input string) []interface{} {
112+
lines := make([]interface{}, 0)
113+
r := strings.NewReader(input)
114+
scanner := bufio.NewScanner(r)
115+
for scanner.Scan() {
116+
text := scanner.Text()
117+
lines = append(lines, text)
118+
}
119+
return lines
120+
}
121+
122+
func TestDecodeAwsDefault(t *testing.T) {
123+
newDecode := initNewDecodeAws(t, testConfig1)
124+
decodeAws := newDecode.(*decodeAws)
125+
require.Equal(t, defaultKeys, decodeAws.keyTags)
126+
127+
lines := breakIntoLines(input1)
128+
nLines := len(lines)
129+
output := decodeAws.Decode(lines)
130+
require.Equal(t, nLines, len(output))
131+
132+
expectedResult := config.GenericMap{
133+
"version": "2",
134+
"account-id": "123456789010",
135+
"interface-id": "eni-1235b8ca123456789",
136+
"srcaddr": "172.31.16.139",
137+
"dstaddr": "172.31.16.21",
138+
"srcport": "20641",
139+
"dstport": "22",
140+
"protocol": "6",
141+
"packets": "20",
142+
"bytes": "4249",
143+
"start": "1418530010",
144+
"end": "1418530070",
145+
"action": "ACCEPT",
146+
"log-status": "OK",
147+
}
148+
require.Equal(t, expectedResult, output[0])
149+
}
150+
151+
func TestDecodeAwsDefaultErr(t *testing.T) {
152+
newDecode := initNewDecodeAws(t, testConfig1)
153+
decodeAws := newDecode.(*decodeAws)
154+
require.Equal(t, defaultKeys, decodeAws.keyTags)
155+
156+
lines := breakIntoLines(inputErr)
157+
nLines := len(lines) - 1
158+
output := decodeAws.Decode(lines)
159+
require.Equal(t, nLines, len(output))
160+
}
161+
162+
func TestDecodeAwsCustom(t *testing.T) {
163+
newDecode := initNewDecodeAws(t, testConfig2)
164+
decodeAws := newDecode.(*decodeAws)
165+
166+
lines := breakIntoLines(input2)
167+
nLines := len(lines)
168+
output := decodeAws.Decode(lines)
169+
require.Equal(t, nLines, len(output))
170+
171+
expectedResult := config.GenericMap{
172+
"version": "3",
173+
"vpc-id": "vpc-abcdefab012345678",
174+
"subnet-id": "subnet-aaaaaaaa012345678",
175+
"instance-id": "i-01234567890123456",
176+
"interface-id": "eni-1235b8ca123456789",
177+
"account-id": "123456789010",
178+
"type": "IPv4",
179+
"srcaddr": "52.213.180.42",
180+
"dstaddr": "10.0.0.62",
181+
"srcport": "43416",
182+
"dstport": "5001",
183+
"pkt-srcaddr": "52.213.180.42",
184+
"pkt-dstaddr": "10.0.0.62",
185+
"protocol": "6",
186+
"bytes": "568",
187+
"packets": "8",
188+
"start": "1566848875",
189+
"end": "1566848933",
190+
"action": "ACCEPT",
191+
"tcp-flags": "2",
192+
"log-status": "OK",
193+
}
194+
require.Equal(t, expectedResult, output[0])
195+
}
196+
197+
func TestConfigErr(t *testing.T) {
198+
v := test.InitConfig(t, testConfigErr)
199+
val := v.Get("pipeline.decode.type")
200+
require.Equal(t, "aws", val)
201+
202+
tmp := v.Get("pipeline.decode.aws")
203+
var json = jsoniter.ConfigCompatibleWithStandardLibrary
204+
b, err := json.Marshal(&tmp)
205+
require.Equal(t, err, nil)
206+
// perform initializations usually done in main.go
207+
config.Opt.PipeLine.Decode.Aws = string(b)
208+
209+
newDecode, _ := NewDecodeAws()
210+
require.Equal(t, nil, newDecode)
211+
}

‎pkg/pipeline/decode/decode_json.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func (c *decodeJson) Decode(in []interface{}) []config.GenericMap {
3737
var decodedLine map[string]interface{}
3838
err := json.Unmarshal(line2, &decodedLine)
3939
if err != nil {
40-
log.Errorf("error unmarshalling a line: %v", err)
40+
log.Errorf("decodeJson Decode: error unmarshalling a line: %v", err)
4141
continue
4242
}
4343
decodedLine2 := make(config.GenericMap, len(decodedLine))

‎pkg/pipeline/decode/decode_json_test.go‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,31 +18,38 @@
1818
package decode
1919

2020
import (
21+
"github.com/netobserv/flowlogs2metrics/pkg/config"
2122
"github.com/stretchr/testify/require"
2223
"testing"
2324
)
2425

2526
func initNewDecodeJson(t *testing.T) Decoder {
2627
newDecode, err := NewDecodeJson()
27-
require.Equal(t, err, nil)
28+
require.Equal(t, nil, err)
2829
return newDecode
2930
}
3031

3132
func TestDecodeJson(t *testing.T) {
3233
newDecode := initNewDecodeJson(t)
3334
decodeJson := newDecode.(*decodeJson)
3435
inputString1 := "{\"varInt\": 12, \"varString\":\"testString\", \"varBool\":false}"
35-
inputString2 := "{\"varInt\": 12, \"varString\":\"testString\", \"varBool\":false}"
36+
inputString2 := "{\"varInt\": 14, \"varString\":\"testString2\", \"varBool\":true}"
37+
inputString3 := "{}"
38+
inputStringErr := "{\"varInt\": 14, \"varString\",\"testString2\", \"varBool\":true}"
3639
var in []interface{}
40+
var out []config.GenericMap
41+
out = decodeJson.Decode(in)
42+
require.Equal(t, 0, len(out))
3743
in = append(in, inputString1)
3844
in = append(in, inputString2)
39-
out := decodeJson.Decode(in)
40-
require.Equal(t, len(out), 2)
45+
in = append(in, inputString3)
46+
in = append(in, inputStringErr)
47+
out = decodeJson.Decode(in)
48+
require.Equal(t, len(out), 3)
4149
// verify that all items come back as strings
42-
require.Equal(t, out[0]["varInt"], "12")
43-
require.Equal(t, out[0]["varString"], "testString")
44-
require.Equal(t, out[0]["varBool"], "false")
50+
require.Equal(t, "12", out[0]["varInt"])
51+
require.Equal(t, "testString", out[0]["varString"])
52+
require.Equal(t, "false", out[0]["varBool"])
4553

4654
// TODO: Check for more complicated json structures
47-
// TODO: check for error cases
4855
}

‎pkg/pipeline/decode/decode_test.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright (C) 2022 IBM, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
*/
17+
18+
package decode
19+
20+
import (
21+
"github.com/netobserv/flowlogs2metrics/pkg/config"
22+
"github.com/stretchr/testify/require"
23+
"testing"
24+
)
25+
26+
func initNewDecodeNone(t *testing.T) Decoder {
27+
newDecode, err := NewDecodeNone()
28+
require.Equal(t, err, nil)
29+
return newDecode
30+
}
31+
32+
func TestDecodeNone(t *testing.T) {
33+
newDecode := initNewDecodeNone(t)
34+
decodeNone := newDecode.(*decodeNone)
35+
inputString1 := "{\"varInt\": 12, \"varString\":\"testString\", \"varBool\":false}"
36+
inputString2 := "{\"varInt\": 14, \"varString\":\"testString2\", \"varBool\":true}"
37+
inputString3 := "{}"
38+
inputStringErr := "{\"varInt\": 14, \"varString\",\"testString2\", \"varBool\":true}"
39+
var in []interface{}
40+
var out []config.GenericMap
41+
out = decodeNone.Decode(in)
42+
require.Equal(t, 0, len(out))
43+
in = append(in, inputString1)
44+
in = append(in, inputString2)
45+
in = append(in, inputString3)
46+
in = append(in, inputStringErr)
47+
out = decodeNone.Decode(in)
48+
require.Equal(t, len(out), 0)
49+
}

‎playground/aws1.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
log-level: debug
12
pipeline:
23
ingest:
34
type: file
@@ -28,5 +29,5 @@ pipeline:
2829
encode:
2930
type: none
3031
write:
31-
type: none
32+
type: stdout
3233

‎playground/aws2.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
log-level: debug
12
pipeline:
23
ingest:
34
type: file

‎playground/aws3.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
log-level: debug
12
pipeline:
23
ingest:
34
type: file

0 commit comments

Comments
 (0)