Skip to content

Commit d3d5ce9

Browse files
jpinsonneaujpinsonneau
authored
NETOBSERV-2143 improve pcapng implementation (#293)
* allow custom registry * vendor * ng writer * remove scanner return code * switch to gopacket/gopacket
1 parent 604ab5d commit d3d5ce9

File tree

153 files changed

+33513
-15626
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

153 files changed

+33513
-15626
lines changed

‎cmd/flow_capture.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,7 @@ var flowCmd = &cobra.Command{
2525
}
2626

2727
func runFlowCapture(_ *cobra.Command, _ []string) {
28-
go func() {
29-
if !scanner() {
30-
return
31-
}
32-
// scanner returns on exit request
33-
os.Exit(0)
34-
}()
28+
go scanner()
3529

3630
captureType = "Flow"
3731
wg := sync.WaitGroup{}

‎cmd/flow_display.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -235,10 +235,10 @@ func printBuf() {
235235
}
236236

237237
// scanner returns true in case of normal exit (end of program execution) or false in case of error
238-
func scanner() bool {
238+
func scanner() {
239239
if err := keyboard.Open(); err != nil {
240240
keyboardError = fmt.Sprintf("Keyboard not supported %v", err)
241-
return false
241+
return
242242
}
243243
defer func() {
244244
_ = keyboard.Close()
@@ -253,7 +253,8 @@ func scanner() bool {
253253
case key == keyboard.KeyCtrlC, stopReceived:
254254
log.Info("Ctrl-C pressed, exiting program.")
255255
// exit program
256-
return true
256+
stopReceived = true
257+
return
257258
case key == keyboard.KeyArrowUp:
258259
showCount++
259260
case key == keyboard.KeyArrowDown:

‎cmd/map_format.go‎

Lines changed: 43 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -474,6 +474,14 @@ func ToTableColWidth(id string) int {
474474
return 6
475475
}
476476

477+
func toColID(field string) string {
478+
colIndex := slices.IndexFunc(cfg.Columns, func(c *ColumnConfig) bool { return c.Field == field })
479+
if colIndex != -1 {
480+
return cfg.Columns[colIndex].ID
481+
}
482+
return ""
483+
}
484+
477485
func toFieldName(id string) string {
478486
colIndex := slices.IndexFunc(cfg.Columns, func(c *ColumnConfig) bool { return c.ID == id })
479487
if colIndex != -1 {
@@ -482,45 +490,47 @@ func toFieldName(id string) string {
482490
return ""
483491
}
484492

493+
func toDisplayValue(genericMap config.GenericMap, colID string, fieldName string) interface{} {
494+
switch colID {
495+
case "EndTime":
496+
if captureType == "Flow" {
497+
return toTimeString(genericMap, "TimeFlowEndMs")
498+
}
499+
return toTimeString(genericMap, "Time")
500+
// special cases where autocompletes are involved
501+
case "FlowDirection", "IfDirections":
502+
return toDirection(genericMap, fieldName)
503+
case "Proto":
504+
return toProto(genericMap, fieldName)
505+
case "Dscp":
506+
return toDSCP(genericMap, fieldName)
507+
// bytes count
508+
case "Bytes":
509+
return toCount(genericMap, "Bytes")
510+
case "PktDropBytes":
511+
return toCount(genericMap, "PktDropBytes")
512+
// duration parsing
513+
case "DNSLatency":
514+
return toDuration(genericMap, fieldName, time.Millisecond)
515+
case "TimeFlowRttMs":
516+
return toDuration(genericMap, fieldName, time.Nanosecond)
517+
case "NetworkEvents":
518+
events := ovnutils.NetworkEventsToStrings(genericMap)
519+
return strings.Join(events, ", ")
520+
default:
521+
// else simply pick field value as text from column name
522+
return toValue(genericMap, fieldName)
523+
}
524+
}
525+
485526
func ToTableRow(genericMap config.GenericMap, colIDs []string) []interface{} {
486527
row := []interface{}{}
487528

488529
for _, colID := range colIDs {
489530
// convert column id to its field accordingly
490531
fieldName := toFieldName(colID)
491-
492-
switch colID {
493-
case "EndTime":
494-
if captureType == "Flow" {
495-
row = append(row, toTimeString(genericMap, "TimeFlowEndMs"))
496-
} else {
497-
row = append(row, toTimeString(genericMap, "Time"))
498-
}
499-
// special cases where autocompletes are involved
500-
case "FlowDirection", "IfDirections":
501-
row = append(row, toDirection(genericMap, fieldName))
502-
case "Proto":
503-
row = append(row, toProto(genericMap, fieldName))
504-
case "Dscp":
505-
row = append(row, toDSCP(genericMap, fieldName))
506-
// bytes count
507-
case "Bytes":
508-
row = append(row, toCount(genericMap, "Bytes"))
509-
case "PktDropBytes":
510-
row = append(row, toCount(genericMap, "PktDropBytes"))
511-
// duration parsing
512-
case "DNSLatency":
513-
row = append(row, toDuration(genericMap, fieldName, time.Millisecond))
514-
case "TimeFlowRttMs":
515-
row = append(row, toDuration(genericMap, fieldName, time.Nanosecond))
516-
case "NetworkEvents":
517-
events := ovnutils.NetworkEventsToStrings(genericMap)
518-
row = append(row, strings.Join(events, ", "))
519-
default:
520-
// else simply pick field value as text from column name
521-
row = append(row, toValue(genericMap, fieldName))
522-
}
532+
// append value to row
533+
row = append(row, toDisplayValue(genericMap, colID, fieldName))
523534
}
524-
525535
return row
526536
}

‎cmd/packet_capture.go‎

Lines changed: 59 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,19 @@ import (
55
"encoding/json"
66
"fmt"
77
"os"
8+
"sort"
89
"strings"
910
"sync"
1011
"time"
1112

12-
"github.com/google/gopacket/layers"
13+
"github.com/gopacket/gopacket"
14+
"github.com/gopacket/gopacket/layers"
15+
"github.com/gopacket/gopacket/pcapgo"
1316
"github.com/jpillora/sizestr"
1417
"github.com/netobserv/flowlogs-pipeline/pkg/config"
1518
"github.com/netobserv/flowlogs-pipeline/pkg/pipeline/utils"
1619
"github.com/netobserv/flowlogs-pipeline/pkg/pipeline/write/grpc"
1720
"github.com/netobserv/flowlogs-pipeline/pkg/pipeline/write/grpc/genericmap"
18-
"github.com/ryankurte/go-pcapng"
19-
"github.com/ryankurte/go-pcapng/types"
2021
"github.com/spf13/cobra"
2122
)
2223

@@ -56,32 +57,27 @@ func runPacketCaptureOnAddr(port int, filename string) error {
5657
":", "") // get rid of offensive colons
5758
}
5859

59-
var f *os.File
6060
err := os.MkdirAll("./output/pcap/", 0700)
6161
if err != nil {
6262
log.Errorf("Create directory failed: %v", err.Error())
6363
log.Fatal(err)
6464
}
6565
log.Trace("Created pcap folder")
6666

67-
pw, err := pcapng.NewFileWriter("./output/pcap/" + filename + ".pcapng")
67+
f, err := os.Create("./output/pcap/" + filename + ".pcapng")
6868
if err != nil {
69-
log.Errorf("Create file %s failed: %v", filename, err.Error())
7069
log.Fatal(err)
7170
}
71+
defer f.Close()
7272
log.Trace("Created pcapng file")
7373

74-
// write pcap file header
75-
so := types.SectionHeaderOptions{
76-
Comment: filename,
77-
Application: "netobserv-cli",
78-
}
79-
err = pw.WriteSectionHeader(so)
74+
ngw, err := pcapgo.NewNgWriter(f, layers.LinkTypeEthernet)
8075
if err != nil {
81-
log.Fatal(err)
76+
log.Error("Error while creating writer", err)
77+
return nil
8278
}
83-
defer f.Close()
84-
log.Trace("Wrote pcap section header")
79+
defer ngw.Flush()
80+
log.Trace("Wrote pcap section header & interface")
8581

8682
flowPackets := make(chan *genericmap.Flow, 100)
8783
collector, err := grpc.StartCollector(port, flowPackets)
@@ -99,6 +95,10 @@ func runPacketCaptureOnAddr(port int, filename string) error {
9995
log.Trace("Done")
10096
}()
10197

98+
var srcComment strings.Builder
99+
var dstComment strings.Builder
100+
var commonComment strings.Builder
101+
102102
log.Trace("Ready ! Waiting for packets...")
103103
go hearbeat()
104104
for fp := range flowPackets {
@@ -141,15 +141,53 @@ func runPacketCaptureOnAddr(port int, filename string) error {
141141
log.Error("Error while decoding data", err)
142142
return nil
143143
}
144+
// sort generic map keys to keep comments ordered
145+
keys := make([]string, 0, len(genericMap))
146+
for k := range genericMap {
147+
// ignore time field
148+
if k == "Time" {
149+
continue
150+
}
151+
keys = append(keys, k)
144152

145-
// write enriched data as interface
146-
writeEnrichedData(pw, &genericMap)
153+
}
154+
sort.Strings(keys)
155+
156+
// generate comments per category
157+
srcComment.WriteString("Source\n")
158+
dstComment.WriteString("Destination\n")
159+
commonComment.WriteString("Common\n")
160+
for _, k := range keys {
161+
id := toColID(k)
162+
str := fmt.Sprintf("%s: %v\n", ToTableColName(id), toDisplayValue(genericMap, id, k))
163+
if strings.HasPrefix(k, "Src") {
164+
srcComment.WriteString(str)
165+
} else if strings.HasPrefix(k, "Dst") {
166+
dstComment.WriteString(str)
167+
} else {
168+
commonComment.WriteString(str)
169+
}
170+
}
147171

148-
// then append packet to file using totalPackets as unique id
149-
err = pw.WriteEnhancedPacketBlock(totalPackets, ts, b, types.EnhancedPacketOptions{})
150-
if err != nil {
151-
return err
172+
// write enriched data as interface
173+
if err := ngw.WritePacketWithOptions(gopacket.CaptureInfo{
174+
Timestamp: ts,
175+
Length: len(b),
176+
CaptureLength: len(b),
177+
}, b, pcapgo.NgPacketOptions{
178+
Comments: []string{
179+
srcComment.String(),
180+
dstComment.String(),
181+
commonComment.String(),
182+
},
183+
}); err != nil {
184+
log.Error("Error while writing packet", err)
185+
return nil
152186
}
187+
188+
srcComment.Reset()
189+
dstComment.Reset()
190+
commonComment.Reset()
153191
} else {
154192
if !captureStarted {
155193
log.Trace("Data is missing")
@@ -167,7 +205,6 @@ func runPacketCaptureOnAddr(port int, filename string) error {
167205
return nil
168206
}
169207
}
170-
totalPackets++
171208

172209
// terminate capture if max time reached
173210
now := currentTime()
@@ -183,36 +220,3 @@ func runPacketCaptureOnAddr(port int, filename string) error {
183220
}
184221
return nil
185222
}
186-
187-
func writeEnrichedData(pw *pcapng.FileWriter, genericMap *config.GenericMap) {
188-
var io types.InterfaceOptions
189-
srcType := toValue(*genericMap, "SrcK8S_Type").(string)
190-
if srcType != emptyText {
191-
io = types.InterfaceOptions{
192-
Name: fmt.Sprintf(
193-
"%s: %s -> %s: %s",
194-
srcType,
195-
toValue(*genericMap, "SrcK8S_Name"),
196-
toValue(*genericMap, "DstK8S_Type"),
197-
toValue(*genericMap, "DstK8S_Name")),
198-
Description: fmt.Sprintf(
199-
"%s: %s Namespace: %s -> %s: %s Namespace: %s",
200-
toValue(*genericMap, "SrcK8S_OwnerType"),
201-
toValue(*genericMap, "SrcK8S_OwnerName"),
202-
toValue(*genericMap, "SrcK8S_Namespace"),
203-
toValue(*genericMap, "DstK8S_OwnerType"),
204-
toValue(*genericMap, "DstK8S_OwnerName"),
205-
toValue(*genericMap, "DstK8S_Namespace"),
206-
),
207-
}
208-
} else {
209-
io.Name = "Unknown resource"
210-
io = types.InterfaceOptions{
211-
Name: "Unknown kubernetes resource",
212-
}
213-
}
214-
err := pw.WriteInterfaceDescription(uint16(layers.LinkTypeEthernet), io)
215-
if err != nil {
216-
log.Fatal(err)
217-
}
218-
}

‎cmd/root.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,7 @@ var (
2828

2929
mutex = sync.Mutex{}
3030

31-
totalBytes = int64(0)
32-
totalPackets = uint32(0)
31+
totalBytes = int64(0)
3332

3433
rootCmd = &cobra.Command{
3534
Use: "network-observability-cli",

‎go.mod‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,14 @@ toolchain go1.23.5
77
require (
88
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203
99
github.com/fatih/color v1.18.0
10-
github.com/google/gopacket v1.1.19
10+
github.com/gopacket/gopacket v1.3.2-0.20250513021322-b23ac340366c
1111
github.com/jpillora/sizestr v1.0.0
1212
github.com/mattn/go-sqlite3 v1.14.28
1313
github.com/netobserv/flowlogs-pipeline v1.9.0-crc1
1414
github.com/netobserv/netobserv-ebpf-agent v1.9.0-crc0
1515
github.com/onsi/ginkgo/v2 v2.23.4
1616
github.com/onsi/gomega v1.37.0
1717
github.com/rodaine/table v1.3.0
18-
github.com/ryankurte/go-pcapng v0.0.0-20170712041429-73fd1a63fab4
1918
github.com/sirupsen/logrus v1.9.3
2019
github.com/spf13/cobra v1.9.1
2120
github.com/stretchr/testify v1.10.0

‎go.sum‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,12 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX
6565
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
6666
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
6767
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
68-
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
69-
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
7068
github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4 h1:gD0vax+4I+mAj+jEChEf25Ia07Jq7kYOFO5PPhAxFl4=
7169
github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA=
7270
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
7371
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
72+
github.com/gopacket/gopacket v1.3.2-0.20250513021322-b23ac340366c h1:8f3IKyIbp1D5VM0g/LGuHFyMzcCcr5FOyp9VmQniWWk=
73+
github.com/gopacket/gopacket v1.3.2-0.20250513021322-b23ac340366c/go.mod h1:EpvsxINeehp5qj4YMKMLf2/dekdhKn2IIAO/ZOifS7o=
7474
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
7575
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
7676
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -159,8 +159,6 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
159159
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
160160
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
161161
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
162-
github.com/ryankurte/go-pcapng v0.0.0-20170712041429-73fd1a63fab4 h1:bt4n2XpGDTrbVf1ahF8n3b2hRaq2DcRhtU+ELSyuYRk=
163-
github.com/ryankurte/go-pcapng v0.0.0-20170712041429-73fd1a63fab4/go.mod h1:pIeU5V4EYPgLckKEp7pQ9QuCl2y0tyJdRjxZKd49q3A=
164162
github.com/safchain/ethtool v0.5.10 h1:Im294gZtuf4pSGJRAOGKaASNi3wMeFaGaWuSaomedpc=
165163
github.com/safchain/ethtool v0.5.10/go.mod h1:w9jh2Lx7YBR4UwzLkzCmWl85UY0W2uZdd7/DckVE5+c=
166164
github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k=
@@ -239,8 +237,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
239237
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
240238
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
241239
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
242-
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
243-
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
244240
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
245241
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
246242
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -297,7 +293,6 @@ golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
297293
golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
298294
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
299295
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
300-
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
301296
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
302297
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
303298
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

‎vendor/github.com/google/gopacket/.travis.gofmt.sh‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

0 commit comments

Comments
 (0)