Add support for dumping to SQLite

Author: hslatman

cmd/dump/flags.go

@@ -40,6 +40,7 @@ var (
 	flagCSV             = fs.Bool("csv", false, "print output data as csv with header line")
 	flagPrintStructured = fs.Bool("struc", true, "print output as structured objects")
 	flagTSV             = fs.Bool("tsv", false, "print output as tab separated values")
+	flagSQLite          = fs.String("sqlite", "", "path of SQLite database to write to")
 	flagHeader          = fs.Bool("header", false, "print audit record file header and exit")
 	flagTable           = fs.Bool("table", false, "print output as table view (thanks @evilsocket)")
 	flagBegin           = fs.String("begin", "(", "begin character for a structure in CSV output")
@@ -50,4 +51,5 @@ var (
 	flagJSON            = fs.Bool("json", false, "print as JSON")
 	flagMemBufferSize   = fs.Int("membuf-size", defaults.BufferSize, "set size for membuf")
 	flagForceColors     = fs.Bool("c", false, "force colors")
+	flagDebug           = fs.Bool("debug", false, "display debug information")
 )

cmd/dump/main.go

@@ -14,6 +14,8 @@
 package dump
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"log"
 	"os"
@@ -28,8 +30,12 @@ import (
 	"github.com/dreadl0ck/netcap/io"
 	"github.com/dreadl0ck/netcap/types"
 	"github.com/dreadl0ck/netcap/utils"
+
+	"github.com/dreadl0ck/netcap/internal/sqlite"
 )
 
+var errAborted = errors.New("operation aborted by user")
+
 // Run parses the subcommand flags and handles the arguments.
 func Run() {
 	// parse commandline flags
@@ -84,13 +90,44 @@ func Run() {
 		os.Exit(0) // bye bye
 	}
 
-	// set separators for sub structures in CSV
-	types.StructureBegin = *flagBegin
-	types.StructureEnd = *flagEnd
-	types.FieldSeparator = *flagStructSeparator
+	switch {
+	case isAuditRecordDirectory(*flagInput):
+		// ensure we start with a fresh database at all times, so that results
+		// from different directories don't get merged into a single database.
+		if _, err := os.Stat(*flagSQLite); err == nil {
+			shouldPrompt := true // TODO: make this depend on flags, similar to collector init
+			if shouldPrompt {
+				msg := "database output path already exists! Overwrite?"
+				if !utils.Confirm(msg) {
+					fmt.Println(ansi.Red + fmt.Sprintf("> %v", errAborted) + ansi.Reset)
+					os.Exit(1)
+				}
+
+				if err := os.Remove(*flagSQLite); err != nil {
+					fmt.Println(ansi.Red + fmt.Sprintf("> failed removing existing database %q: %v", *flagSQLite, err) + ansi.Reset)
+					os.Exit(1)
+				}
+			}
+		}
 
-	// read ncap file and print to stdout
-	if filepath.Ext(*flagInput) == defaults.FileExtension || filepath.Ext(*flagInput) == ".gz" {
+		// read all ncap files, and insert into SQLite database
+		err = sqlite.Dump(
+			context.Background(),
+			os.Stdout,
+			sqlite.DumpConfig{
+				Paths:         []string{*flagInput}, // TODO: support multiple (non-directory) paths?
+				Output:        *flagSQLite,
+				MemBufferSize: *flagMemBufferSize,
+				Selection:     *flagSelect,
+				Debug:         *flagDebug,
+			})
+	case isAuditRecordFile(*flagInput):
+		// set separators for sub structures in CSV
+		types.StructureBegin = *flagBegin
+		types.StructureEnd = *flagEnd
+		types.FieldSeparator = *flagStructSeparator
+
+		// read ncap file and print to stdout
 		err = io.Dump(
 			os.Stdout,
 			io.DumpConfig{
@@ -107,10 +144,31 @@ func Run() {
 				ForceColors:  *flagForceColors,
 			},
 		)
+	default:
+		fi, err := os.Open(*flagInput)
 		if err != nil {
-			log.Fatal(err)
+			fmt.Println(ansi.Red + fmt.Sprintf("> failed opening %q: %v", *flagInput, errors.Unwrap(err)) + ansi.Reset)
+			os.Exit(1)
 		}
 
-		return
+		fmt.Println(ansi.Red + fmt.Sprintf("> input %q doesn't contain audit record file(s)", fi.Name()) + ansi.Reset)
+		os.Exit(1)
+	}
+
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func isAuditRecordDirectory(input string) bool {
+	fi, err := os.Stat(input)
+	if err == nil && fi.IsDir() {
+		return true
 	}
+
+	return false
+}
+
+func isAuditRecordFile(input string) bool {
+	return filepath.Ext(input) == defaults.FileExtension || filepath.Ext(input) == ".gz"
 }

go.mod

@@ -45,6 +45,7 @@ require (
 	golang.org/x/net v0.41.0
 	gonum.org/v1/gonum v0.16.0
 	gopkg.in/yaml.v2 v2.4.0
+	modernc.org/sqlite v1.38.2
 	mvdan.cc/xurls/v2 v2.6.0
 )
 
@@ -82,6 +83,7 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v1.0.0 // indirect
 	github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -92,12 +94,14 @@ require (
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.65.0 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/skeema/knownhosts v1.3.0 // indirect
@@ -110,13 +114,16 @@ require (
 	golang.org/x/image v0.25.0 // indirect
 	golang.org/x/mod v0.25.0 // indirect
 	golang.org/x/sync v0.15.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/sys v0.34.0 // indirect
 	golang.org/x/term v0.32.0 // indirect
 	golang.org/x/text v0.26.0 // indirect
 	golang.org/x/tools v0.34.0 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.66.3 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
 )
 
 //replace github.com/dreadl0ck/maltego => ../maltego

go.sum

@@ -188,6 +188,8 @@ github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8v
 github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 h1:xhMrHhTJ6zxu3gA4enFM9MLn9AY7613teCdFnlUVbSQ=
 github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc=
 github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE=
@@ -253,6 +255,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs=
 github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -288,6 +292,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM
 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -416,8 +422,8 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
 golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
@@ -460,5 +466,31 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM=
+modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU=
+modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE=
+modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM=
+modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ=
+modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek=
+modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 mvdan.cc/xurls/v2 v2.6.0 h1:3NTZpeTxYVWNSokW3MKeyVkz/j7uYXYiMtXRUfmjbgI=
 mvdan.cc/xurls/v2 v2.6.0/go.mod h1:bCvEZ1XvdA6wDnxY7jPPjEmigDtvtvPXAD/Exa9IMSk=

internal/sqlite/sqlite.go

@@ -0,0 +1,157 @@
+package sqlite
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	_ "modernc.org/sqlite"
+
+	"github.com/dreadl0ck/netcap/defaults"
+	netcapio "github.com/dreadl0ck/netcap/io"
+	"github.com/dreadl0ck/netcap/types"
+)
+
+const newline = "\n"
+
+// DumpConfig contains all settings for writing audit records to
+// an SQLite database.
+type DumpConfig struct {
+	Paths         []string
+	Output        string
+	MemBufferSize int
+	Selection     string
+	Debug         bool
+}
+
+func Dump(ctx context.Context, w *os.File, c DumpConfig) error {
+	if len(c.Paths) > 1 {
+		return errors.New("multiple files not yet supported")
+	}
+
+	p := c.Paths[0]
+	fi, err := os.Stat(p)
+	if err != nil {
+		return fmt.Errorf("failed statting file %q: %w", p, err)
+	}
+
+	if !fi.IsDir() {
+		return fmt.Errorf("%q is not a directory", p)
+	}
+
+	db, err := sql.Open("sqlite", c.Output)
+	if err != nil {
+		return fmt.Errorf("failed opening SQLite database %q: %w", c.Output, err)
+	}
+
+	defer func() {
+		if err := db.Close(); err != nil {
+			_, _ = w.WriteString(fmt.Sprintf("failed to close database: %v\n", err))
+		}
+	}()
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("failed starting database transaction: %w", err)
+	}
+
+	entries, err := os.ReadDir(p)
+	if err != nil {
+		return fmt.Errorf("failed reading directory %q: %w", p, err)
+	}
+
+	for _, entry := range entries {
+		if entry.IsDir() || (filepath.Ext(entry.Name()) != defaults.FileExtension && filepath.Ext(entry.Name()) != ".gz") {
+			continue // skip directories and files that don't look like audit record files
+		}
+
+		fp := filepath.Join(p, entry.Name())
+		r, err := netcapio.Open(fp, c.MemBufferSize)
+		if err != nil {
+			return fmt.Errorf("failed to open audit record file: %w", err)
+		}
+
+		header, err := r.ReadHeader()
+		if err != nil {
+			return fmt.Errorf("failed reading record file header: %w", err)
+		}
+
+		var (
+			record           = netcapio.InitRecord(header.Type)
+			isFirstIteration = true
+		)
+
+		types.Select(record, c.Selection)
+		types.UTC = true // always use UTC
+
+		for {
+			err = r.Next(record)
+			if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+				break
+			} else if err != nil {
+				return fmt.Errorf("failed to read next audit record: %w", err)
+			}
+
+			if p, ok := record.(types.SQLCapableAuditRecord); ok {
+				if isFirstIteration {
+					if c.Debug {
+						_, _ = w.WriteString(p.SQLTable())
+						_, _ = w.WriteString(newline)
+					}
+
+					if _, err = tx.ExecContext(ctx, p.SQLTable()); err != nil {
+						return fmt.Errorf("failed creating table: %w", err)
+					}
+				}
+
+				query, values := p.SQLInsert()
+				if c.Debug {
+					_, _ = w.WriteString(fmt.Sprintf("%s, (%s)", query, join(values, ",")))
+					_, _ = w.WriteString(newline)
+				}
+
+				if _, err = tx.ExecContext(ctx, query, values...); err != nil {
+					return fmt.Errorf("failed inserting audit record: %w", err)
+				}
+			} else {
+				_, _ = w.WriteString(fmt.Sprintf("skipped processing %q containing %q audit records; dumping to SQLite not yet supported for this audit record type", fp, header.Type.String()))
+				_, _ = w.WriteString(newline)
+
+				// exit from the inner loop; continues with next audit record file
+				break
+			}
+
+			isFirstIteration = false
+		}
+
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func join(values []any, sep string) string {
+	if len(values) == 0 {
+		return ""
+	}
+
+	if len(values) == 1 {
+		return fmt.Sprintf("%v", values[0])
+	}
+
+	var buffer bytes.Buffer
+	buffer.WriteString(fmt.Sprintf("%v", values[0]))
+	for _, s := range values[1:] {
+		buffer.WriteString(fmt.Sprintf("%s%v", sep, s))
+	}
+
+	return buffer.String()
+}