Skip to content

Commit f08f041

Browse files
hslatmanhslatman
committed
Add support for dumping to SQLite
1 parent 8f7839f commit f08f041

File tree

8 files changed

+362
-13
lines changed

8 files changed

+362
-13
lines changed

‎cmd/dump/flags.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ var (
4040
flagCSV = fs.Bool("csv", false, "print output data as csv with header line")
4141
flagPrintStructured = fs.Bool("struc", true, "print output as structured objects")
4242
flagTSV = fs.Bool("tsv", false, "print output as tab separated values")
43+
flagSQLite = fs.String("sqlite", "", "path of SQLite database to write to")
4344
flagHeader = fs.Bool("header", false, "print audit record file header and exit")
4445
flagTable = fs.Bool("table", false, "print output as table view (thanks @evilsocket)")
4546
flagBegin = fs.String("begin", "(", "begin character for a structure in CSV output")
@@ -50,4 +51,5 @@ var (
5051
flagJSON = fs.Bool("json", false, "print as JSON")
5152
flagMemBufferSize = fs.Int("membuf-size", defaults.BufferSize, "set size for membuf")
5253
flagForceColors = fs.Bool("c", false, "force colors")
54+
flagDebug = fs.Bool("debug", false, "display debug information")
5355
)

‎cmd/dump/main.go‎

Lines changed: 71 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
package dump
1515

1616
import (
17+
"context"
18+
"errors"
1719
"fmt"
1820
"log"
1921
"os"
@@ -28,8 +30,12 @@ import (
2830
"github.com/dreadl0ck/netcap/io"
2931
"github.com/dreadl0ck/netcap/types"
3032
"github.com/dreadl0ck/netcap/utils"
33+
34+
"github.com/dreadl0ck/netcap/internal/sqlite"
3135
)
3236

37+
var errAborted = errors.New("operation aborted by user")
38+
3339
// Run parses the subcommand flags and handles the arguments.
3440
func Run() {
3541
// parse commandline flags
@@ -84,13 +90,49 @@ func Run() {
8490
os.Exit(0) // bye bye
8591
}
8692

87-
// set separators for sub structures in CSV
88-
types.StructureBegin = *flagBegin
89-
types.StructureEnd = *flagEnd
90-
types.FieldSeparator = *flagStructSeparator
93+
switch {
94+
case isAuditRecordDirectory(*flagInput):
95+
if *flagSelect != "" {
96+
fmt.Println(ansi.Red + "field selection is (currently) not supported when dumping to SQLite" + ansi.Reset)
97+
os.Exit(1)
98+
}
99+
100+
// ensure we start with a fresh database at all times, so that results
101+
// from different directories don't get merged into a single database.
102+
if _, err := os.Stat(*flagSQLite); err == nil {
103+
shouldPrompt := true // TODO: make this depend on flags, similar to collector init
104+
if shouldPrompt {
105+
msg := "database output path already exists! Overwrite?"
106+
if !utils.Confirm(msg) {
107+
fmt.Println(ansi.Red + fmt.Sprintf("> %v", errAborted) + ansi.Reset)
108+
os.Exit(1)
109+
}
110+
111+
if err := os.Remove(*flagSQLite); err != nil {
112+
fmt.Println(ansi.Red + fmt.Sprintf("> failed removing existing database %q: %v", *flagSQLite, err) + ansi.Reset)
113+
os.Exit(1)
114+
}
115+
}
116+
}
91117

92-
// read ncap file and print to stdout
93-
if filepath.Ext(*flagInput) == defaults.FileExtension || filepath.Ext(*flagInput) == ".gz" {
118+
// read all ncap files, and insert into SQLite database
119+
err = sqlite.Dump(
120+
context.Background(),
121+
os.Stdout,
122+
sqlite.DumpConfig{
123+
Paths: []string{*flagInput}, // TODO: support multiple (non-directory) paths?
124+
Output: *flagSQLite,
125+
MemBufferSize: *flagMemBufferSize,
126+
Selection: *flagSelect,
127+
Debug: *flagDebug,
128+
})
129+
case isAuditRecordFile(*flagInput):
130+
// set separators for sub structures in CSV
131+
types.StructureBegin = *flagBegin
132+
types.StructureEnd = *flagEnd
133+
types.FieldSeparator = *flagStructSeparator
134+
135+
// read ncap file and print to stdout
94136
err = io.Dump(
95137
os.Stdout,
96138
io.DumpConfig{
@@ -107,10 +149,31 @@ func Run() {
107149
ForceColors: *flagForceColors,
108150
},
109151
)
152+
default:
153+
fi, err := os.Open(*flagInput)
110154
if err != nil {
111-
log.Fatal(err)
155+
fmt.Println(ansi.Red + fmt.Sprintf("> failed opening %q: %v", *flagInput, errors.Unwrap(err)) + ansi.Reset)
156+
os.Exit(1)
112157
}
113158

114-
return
159+
fmt.Println(ansi.Red + fmt.Sprintf("> input %q doesn't contain audit record file(s)", fi.Name()) + ansi.Reset)
160+
os.Exit(1)
161+
}
162+
163+
if err != nil {
164+
log.Fatal(err)
115165
}
116166
}
167+
168+
func isAuditRecordDirectory(input string) bool {
169+
fi, err := os.Stat(input)
170+
if err == nil && fi.IsDir() {
171+
return true
172+
}
173+
174+
return false
175+
}
176+
177+
func isAuditRecordFile(input string) bool {
178+
return filepath.Ext(input) == defaults.FileExtension || filepath.Ext(input) == ".gz"
179+
}

‎go.mod‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ require (
4545
golang.org/x/net v0.41.0
4646
gonum.org/v1/gonum v0.16.0
4747
gopkg.in/yaml.v2 v2.4.0
48+
modernc.org/sqlite v1.38.2
4849
mvdan.cc/xurls/v2 v2.6.0
4950
)
5051

@@ -82,6 +83,7 @@ require (
8283
github.com/golang/protobuf v1.5.4 // indirect
8384
github.com/golang/snappy v1.0.0 // indirect
8485
github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect
86+
github.com/google/uuid v1.6.0 // indirect
8587
github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c // indirect
8688
github.com/hashicorp/golang-lru v1.0.2 // indirect
8789
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -92,12 +94,14 @@ require (
9294
github.com/mattn/go-runewidth v0.0.16 // indirect
9395
github.com/mschoch/smat v0.2.0 // indirect
9496
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
97+
github.com/ncruces/go-strftime v0.1.9 // indirect
9598
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
9699
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
97100
github.com/pjbgf/sha1cd v0.3.0 // indirect
98101
github.com/prometheus/client_model v0.6.2 // indirect
99102
github.com/prometheus/common v0.65.0 // indirect
100103
github.com/prometheus/procfs v0.16.1 // indirect
104+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
101105
github.com/rivo/uniseg v0.4.7 // indirect
102106
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
103107
github.com/skeema/knownhosts v1.3.0 // indirect
@@ -110,13 +114,16 @@ require (
110114
golang.org/x/image v0.25.0 // indirect
111115
golang.org/x/mod v0.25.0 // indirect
112116
golang.org/x/sync v0.15.0 // indirect
113-
golang.org/x/sys v0.33.0 // indirect
117+
golang.org/x/sys v0.34.0 // indirect
114118
golang.org/x/term v0.32.0 // indirect
115119
golang.org/x/text v0.26.0 // indirect
116120
golang.org/x/tools v0.34.0 // indirect
117121
google.golang.org/protobuf v1.36.6 // indirect
118122
gopkg.in/warnings.v0 v0.1.2 // indirect
119123
gopkg.in/yaml.v3 v3.0.1 // indirect
124+
modernc.org/libc v1.66.3 // indirect
125+
modernc.org/mathutil v1.7.1 // indirect
126+
modernc.org/memory v1.11.0 // indirect
120127
)
121128

122129
//replace github.com/dreadl0ck/maltego => ../maltego

‎go.sum‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,8 @@ github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8v
188188
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
189189
github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 h1:xhMrHhTJ6zxu3gA4enFM9MLn9AY7613teCdFnlUVbSQ=
190190
github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA=
191+
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
192+
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
191193
github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
192194
github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc=
193195
github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE=
@@ -253,6 +255,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
253255
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
254256
github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs=
255257
github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo=
258+
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
259+
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
256260
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
257261
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
258262
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -288,6 +292,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM
288292
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
289293
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
290294
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
295+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
296+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
291297
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
292298
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
293299
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -416,8 +422,8 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc
416422
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
417423
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
418424
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
419-
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
420-
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
425+
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
426+
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
421427
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
422428
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
423429
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
@@ -460,5 +466,31 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
460466
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
461467
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
462468
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
469+
modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM=
470+
modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
471+
modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU=
472+
modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE=
473+
modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM=
474+
modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
475+
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
476+
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
477+
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
478+
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
479+
modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ=
480+
modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8=
481+
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
482+
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
483+
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
484+
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
485+
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
486+
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
487+
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
488+
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
489+
modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek=
490+
modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E=
491+
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
492+
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
493+
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
494+
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
463495
mvdan.cc/xurls/v2 v2.6.0 h1:3NTZpeTxYVWNSokW3MKeyVkz/j7uYXYiMtXRUfmjbgI=
464496
mvdan.cc/xurls/v2 v2.6.0/go.mod h1:bCvEZ1XvdA6wDnxY7jPPjEmigDtvtvPXAD/Exa9IMSk=

0 commit comments

Comments
 (0)