Skip to content

Commit d59f209

Browse files
lspgnlspgn
authored
mapping: support endianness (cloudflare#130)
1 parent 40e5ef0 commit d59f209

File tree

4 files changed

+126
-35
lines changed

4 files changed

+126
-35
lines changed

‎cmd/goflow2/mapping.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ netflowv9:
1616
destination: CustomInteger1
1717
- field: 11
1818
destination: CustomInteger2
19+
- field: 34 # samplingInterval
20+
destination: SamplingRate
21+
endian: little
1922
sflow:
2023
mapping:
2124
- layer: 4 # Layer 4: TCP or UDP

‎producer/producer_nf.go‎

Lines changed: 82 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,38 @@ func NetFlowPopulate(dataFields []netflow.DataField, typeId uint16, addr interfa
9797
return exists
9898
}
9999

100+
func WriteUDecoded(o uint64, out interface{}) error {
101+
switch t := out.(type) {
102+
case *byte:
103+
*t = byte(o)
104+
case *uint16:
105+
*t = uint16(o)
106+
case *uint32:
107+
*t = uint32(o)
108+
case *uint64:
109+
*t = o
110+
default:
111+
return errors.New("The parameter is not a pointer to a byte/uint16/uint32/uint64 structure")
112+
}
113+
return nil
114+
}
115+
116+
func WriteDecoded(o int64, out interface{}) error {
117+
switch t := out.(type) {
118+
case *int8:
119+
*t = int8(o)
120+
case *int16:
121+
*t = int16(o)
122+
case *int32:
123+
*t = int32(o)
124+
case *int64:
125+
*t = o
126+
default:
127+
return errors.New("The parameter is not a pointer to a int8/int16/int32/int64 structure")
128+
}
129+
return nil
130+
}
131+
100132
func DecodeUNumber(b []byte, out interface{}) error {
101133
var o uint64
102134
l := len(b)
@@ -120,19 +152,33 @@ func DecodeUNumber(b []byte, out interface{}) error {
120152
return errors.New(fmt.Sprintf("Non-regular number of bytes for a number: %v", l))
121153
}
122154
}
123-
switch t := out.(type) {
124-
case *byte:
125-
*t = byte(o)
126-
case *uint16:
127-
*t = uint16(o)
128-
case *uint32:
129-
*t = uint32(o)
130-
case *uint64:
131-
*t = o
155+
return WriteUDecoded(o, out)
156+
}
157+
158+
func DecodeUNumberLE(b []byte, out interface{}) error {
159+
var o uint64
160+
l := len(b)
161+
switch l {
162+
case 1:
163+
o = uint64(b[0])
164+
case 2:
165+
o = uint64(binary.LittleEndian.Uint16(b))
166+
case 4:
167+
o = uint64(binary.LittleEndian.Uint32(b))
168+
case 8:
169+
o = binary.LittleEndian.Uint64(b)
132170
default:
133-
return errors.New("The parameter is not a pointer to a byte/uint16/uint32/uint64 structure")
171+
if l < 8 {
172+
var iter uint
173+
for i := range b {
174+
o |= uint64(b[i]) << uint(8*(iter))
175+
iter++
176+
}
177+
} else {
178+
return errors.New(fmt.Sprintf("Non-regular number of bytes for a number: %v", l))
179+
}
134180
}
135-
return nil
181+
return WriteUDecoded(o, out)
136182
}
137183

138184
func DecodeNumber(b []byte, out interface{}) error {
@@ -158,19 +204,33 @@ func DecodeNumber(b []byte, out interface{}) error {
158204
return errors.New(fmt.Sprintf("Non-regular number of bytes for a number: %v", l))
159205
}
160206
}
161-
switch t := out.(type) {
162-
case *int8:
163-
*t = int8(o)
164-
case *int16:
165-
*t = int16(o)
166-
case *int32:
167-
*t = int32(o)
168-
case *int64:
169-
*t = o
207+
return WriteDecoded(o, out)
208+
}
209+
210+
func DecodeNumberLE(b []byte, out interface{}) error {
211+
var o int64
212+
l := len(b)
213+
switch l {
214+
case 1:
215+
o = int64(int8(b[0]))
216+
case 2:
217+
o = int64(int16(binary.LittleEndian.Uint16(b)))
218+
case 4:
219+
o = int64(int32(binary.LittleEndian.Uint32(b)))
220+
case 8:
221+
o = int64(binary.LittleEndian.Uint64(b))
170222
default:
171-
return errors.New("The parameter is not a pointer to a int8/int16/int32/int64 structure")
223+
if l < 8 {
224+
var iter int
225+
for i := range b {
226+
o |= int64(b[i]) << int(8*(iter))
227+
iter++
228+
}
229+
} else {
230+
return errors.New(fmt.Sprintf("Non-regular number of bytes for a number: %v", l))
231+
}
172232
}
173-
return nil
233+
return WriteDecoded(o, out)
174234
}
175235

176236
func allZeroes(v []byte) bool {

‎producer/producer_sf.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func ParseEthernetHeader(flowMessage *flowmessage.FlowMessage, data []byte, conf
5858

5959
for _, configLayer := range GetSFlowConfigLayer(config, 0) {
6060
extracted := GetBytes(data, configLayer.Offset, configLayer.Length)
61-
MapCustom(flowMessage, extracted, configLayer.Destination)
61+
MapCustom(flowMessage, extracted, configLayer.Destination, configLayer.Endian)
6262
}
6363

6464
etherType := data[12:14]
@@ -121,7 +121,7 @@ func ParseEthernetHeader(flowMessage *flowmessage.FlowMessage, data []byte, conf
121121

122122
for _, configLayer := range GetSFlowConfigLayer(config, 3) {
123123
extracted := GetBytes(data, offset*8+configLayer.Offset, configLayer.Length)
124-
MapCustom(flowMessage, extracted, configLayer.Destination)
124+
MapCustom(flowMessage, extracted, configLayer.Destination, configLayer.Endian)
125125
}
126126

127127
if etherType[0] == 0x8 && etherType[1] == 0x0 { // IPv4
@@ -159,7 +159,7 @@ func ParseEthernetHeader(flowMessage *flowmessage.FlowMessage, data []byte, conf
159159

160160
for _, configLayer := range GetSFlowConfigLayer(config, 4) {
161161
extracted := GetBytes(data, offset*8+configLayer.Offset, configLayer.Length)
162-
MapCustom(flowMessage, extracted, configLayer.Destination)
162+
MapCustom(flowMessage, extracted, configLayer.Destination, configLayer.Endian)
163163
}
164164

165165
appOffset := 0
@@ -187,7 +187,7 @@ func ParseEthernetHeader(flowMessage *flowmessage.FlowMessage, data []byte, conf
187187
if appOffset > 0 {
188188
for _, configLayer := range GetSFlowConfigLayer(config, 7) {
189189
extracted := GetBytes(data, (offset+appOffset)*8+configLayer.Offset, configLayer.Length)
190-
MapCustom(flowMessage, extracted, configLayer.Destination)
190+
MapCustom(flowMessage, extracted, configLayer.Destination, configLayer.Endian)
191191
}
192192
}
193193

‎producer/reflect.go‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,13 @@ import (
88
flowmessage "github.com/netsampler/goflow2/pb"
99
)
1010

11+
type EndianType string
12+
13+
var (
14+
BigEndian EndianType = "big"
15+
LittleEndian EndianType = "little"
16+
)
17+
1118
func GetBytes(d []byte, offset int, length int) []byte {
1219
if length == 0 {
1320
return nil
@@ -56,11 +63,11 @@ func MapCustomNetFlow(flowMessage *flowmessage.FlowMessage, df netflow.DataField
5663
mapped, ok := mapper.Map(df)
5764
if ok {
5865
v := df.Value.([]byte)
59-
MapCustom(flowMessage, v, mapped.Destination)
66+
MapCustom(flowMessage, v, mapped.Destination, mapped.Endian)
6067
}
6168
}
6269

63-
func MapCustom(flowMessage *flowmessage.FlowMessage, v []byte, destination string) {
70+
func MapCustom(flowMessage *flowmessage.FlowMessage, v []byte, destination string, endianness EndianType) {
6471
vfm := reflect.ValueOf(flowMessage)
6572
vfm = reflect.Indirect(vfm)
6673

@@ -78,9 +85,17 @@ func MapCustom(flowMessage *flowmessage.FlowMessage, v []byte, destination strin
7885
item := reflect.New(typeDest.Elem())
7986

8087
if IsUInt(typeDest.Elem().Kind()) {
81-
DecodeUNumber(v, item.Interface())
88+
if endianness == LittleEndian {
89+
DecodeUNumberLE(v, item.Interface())
90+
} else {
91+
DecodeUNumber(v, item.Interface())
92+
}
8293
} else if IsUInt(typeDest.Elem().Kind()) {
83-
DecodeUNumber(v, item.Interface())
94+
if endianness == LittleEndian {
95+
DecodeUNumberLE(v, item.Interface())
96+
} else {
97+
DecodeUNumber(v, item.Interface())
98+
}
8499
}
85100

86101
itemi := reflect.Indirect(item)
@@ -89,9 +104,17 @@ func MapCustom(flowMessage *flowmessage.FlowMessage, v []byte, destination strin
89104
}
90105

91106
} else if fieldValueAddr.IsValid() && IsUInt(typeDest.Kind()) {
92-
DecodeUNumber(v, fieldValueAddr.Interface())
107+
if endianness == LittleEndian {
108+
DecodeUNumberLE(v, fieldValueAddr.Interface())
109+
} else {
110+
DecodeUNumber(v, fieldValueAddr.Interface())
111+
}
93112
} else if fieldValueAddr.IsValid() && IsInt(typeDest.Kind()) {
94-
DecodeNumber(v, fieldValueAddr.Interface())
113+
if endianness == LittleEndian {
114+
DecodeUNumberLE(v, fieldValueAddr.Interface())
115+
} else {
116+
DecodeUNumber(v, fieldValueAddr.Interface())
117+
}
95118
}
96119
}
97120
}
@@ -101,7 +124,8 @@ type NetFlowMapField struct {
101124
Type uint16 `json:"field" yaml:"field"`
102125
Pen uint32 `json:"pen" yaml:"pen"`
103126

104-
Destination string `json:"destination" yaml:"destination"`
127+
Destination string `json:"destination" yaml:"destination"`
128+
Endian EndianType `json:"endianness" yaml:"endianness"`
105129
//DestinationLength uint8 `json:"dlen"` // could be used if populating a slice of uint16 that aren't in protobuf
106130
}
107131

@@ -119,7 +143,8 @@ type SFlowMapField struct {
119143
Offset int `json:"offset"` // offset in bits
120144
Length int `json:"length"` // length in bits
121145

122-
Destination string `json:"destination"`
146+
Destination string `json:"destination" yaml:"destination"`
147+
Endian EndianType `json:"endianness" yaml:"endianness"`
123148
//DestinationLength uint8 `json:"dlen"`
124149
}
125150

@@ -137,6 +162,7 @@ type ProducerConfig struct {
137162

138163
type DataMap struct {
139164
Destination string
165+
Endian EndianType
140166
}
141167

142168
type NetFlowMapper struct {
@@ -151,7 +177,7 @@ func (m *NetFlowMapper) Map(field netflow.DataField) (DataMap, bool) {
151177
func MapFieldsNetFlow(fields []NetFlowMapField) *NetFlowMapper {
152178
ret := make(map[string]DataMap)
153179
for _, field := range fields {
154-
ret[fmt.Sprintf("%v-%d-%d", field.PenProvided, field.Pen, field.Type)] = DataMap{Destination: field.Destination}
180+
ret[fmt.Sprintf("%v-%d-%d", field.PenProvided, field.Pen, field.Type)] = DataMap{Destination: field.Destination, Endian: field.Endian}
155181
}
156182
return &NetFlowMapper{ret}
157183
}
@@ -160,6 +186,7 @@ type DataMapLayer struct {
160186
Offset int
161187
Length int
162188
Destination string
189+
Endian EndianType
163190
}
164191

165192
type SFlowMapper struct {
@@ -180,6 +207,7 @@ func MapFieldsSFlow(fields []SFlowMapField) *SFlowMapper {
180207
Offset: field.Offset,
181208
Length: field.Length,
182209
Destination: field.Destination,
210+
Endian: field.Endian,
183211
}
184212
retLayer := ret[field.Layer]
185213
retLayer = append(retLayer, retLayerEntry)

0 commit comments

Comments
 (0)