Merge pull request #3 from ncode/juliano/messaging

Support for notification

Author: ncode

.gitignore

@@ -27,6 +27,8 @@ go.work.sum
 # idea
 .idea
 
-# ignore the log directory
+# .DS_Store
+.DS_Store
 
+# ignore the log directory
 configs/development/logs/

README.md

@@ -13,6 +13,8 @@
 - **Dynamic Logging**: Log audit events to specified files with log rotation and size limits.
 - **Supports Multiple Operations**: Filters common Vault operations, including KV operations, metadata updates, and deletion events.
 - **Performance-Oriented**: Built with `gnet` to handle high concurrency.
+- **Flexible Forwarding**: Forward filtered audit logs to specified UDP addresses for further processing or monitoring.
+- **Messaging Integration**: Send notifications about matched audit logs to messaging platforms like Mattermost.
 
 ## Table of Contents
 
@@ -30,7 +32,7 @@ These instructions will help you set up and run `vault-audit-filter` on your loc
 
 ### Prerequisites
 
-- **Go**: Ensure you have Go 1.21 or later installed. You can download it here: <https://golang.org/dl/>
+- **Go**: Ensure you have Go 1.22.3 or later installed. You can download it here: <https://golang.org/dl/>
 - **Vault**: You should have HashiCorp Vault installed and configured. Instructions can be found here: <https://www.vaultproject.io/docs/install>
 
 ### Installation
@@ -54,7 +56,7 @@ Once you have built the project, you can run the `vault-audit-filter` executable
 
 ## Configuration
 
-`vault-audit-filter` uses a YAML-based configuration file that allows you to define rule groups, specify logging files, and configure Vault settings.
+`vault-audit-filter` uses a YAML-based configuration file that allows you to define rule groups, specify logging files, configure Vault settings, set up forwarding options, and configure messaging integration.
 
 ### Sample Configuration (`config.yaml`)
 
@@ -75,6 +77,12 @@ Once you have built the project, you can run the `vault-audit-filter` executable
           max_backups: 5     # Max number of backup files
           max_age: 30        # Max age in days
           compress: true     # Compress rotated files
+        forwarding:
+          enabled: true
+          address: "127.0.0.1:9001"
+        messaging:
+          type: "mattermost_webhook"
+          webhook_url: "https://your-mattermost-instance.com/hooks/your-webhook-id"
 
       - name: "critical_events"
         rules:
@@ -86,6 +94,14 @@ Once you have built the project, you can run the `vault-audit-filter` executable
           max_backups: 5
           max_age: 30
           compress: true
+        forwarding:
+          enabled: true
+          address: "127.0.0.1:9002"
+        messaging:
+          type: "mattermost"
+          url: "https://your-mattermost-instance.com"
+          token: "your-bot-token"
+          channel: "your-channel-id"
 
 ### Configuration Parameters
 
@@ -104,6 +120,13 @@ Once you have built the project, you can run the `vault-audit-filter` executable
   - `log_file.max_backups`: The number of backup logs to keep.
   - `log_file.max_age`: The maximum number of days to retain logs.
   - `log_file.compress`: Whether to compress the old log files.
+  - `forwarding.enabled`: Whether to enable forwarding for this rule group.
+  - `forwarding.address`: The UDP address to forward matching audit logs to.
+  - `messaging.type`: The type of messaging integration ("mattermost" or "mattermost_webhook").
+  - `messaging.webhook_url`: The webhook URL for Mattermost (when using "mattermost_webhook" type).
+  - `messaging.url`: The Mattermost server URL (when using "mattermost" type).
+  - `messaging.token`: The bot token for Mattermost (when using "mattermost" type).
+  - `messaging.channel`: The channel ID for Mattermost messages (when using "mattermost" type).
 
 ### Rule Syntax
 
@@ -139,6 +162,33 @@ $ export VAULT_ADDRESS="http://127.0.0.1:8200"
 $ export VAULT_TOKEN="your-vault-token"
 ```
 
+## Testing
+
+To run the test suite for `vault-audit-filter`, use the following command:
+
+```bash
+go test -v ./...
+```
+
+For running tests with race condition detection:
+
+```bash
+go test -race -v ./...
+```
+
+To run a specific test, such as the concurrent forwarding test:
+
+```bash
+go test -v -run TestUDPForwarder_ConcurrentForwarding ./pkg/forwarder
+```
+
+To generate a test coverage report:
+
+```bash
+go test -v -coverprofile=coverage.out ./...
+go tool cover -html=coverage.out
+```
+
 ### Development
 
 For development purposes, you can use the provided Makefile located at `configs/development/Makefile` to build and run the project using Docker and Docker Compose. This is how I test my changes and have a playground of sorts.
@@ -151,6 +201,7 @@ Before submitting a pull request, ensure that:
 - The code compiles without errors.
 - All tests pass.
 - Your changes are well-documented.
+- You've added or updated tests to cover your changes.
 
 ## License
 

cmd/auditServer.go

@@ -17,10 +17,11 @@ package cmd
 
 import (
 	"fmt"
+	"log"
+
 	"github.com/ncode/vault-audit-filter/pkg/auditserver"
 	"github.com/panjf2000/gnet"
 	"github.com/spf13/viper"
-	"log"
 
 	"github.com/spf13/cobra"
 )
@@ -37,7 +38,10 @@ This application is a tool to generate the needed files
 to quickly create a Cobra application.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		addr := fmt.Sprintf("udp://%s", viper.GetString("vault.audit_address"))
-		server := auditserver.New(nil)
+		server, err := auditserver.New(nil)
+		if err != nil {
+			logger.Error(err.Error())
+		}
 		log.Fatal(gnet.Serve(server, addr, gnet.WithMulticore(true)))
 	},
 }

cmd/root.go

@@ -104,11 +104,6 @@ func initConfig() {
 		fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
 	}
 
-	if viper.GetString("vault.token") == "" {
-		logger.Error("vault.token is required")
-		os.Exit(1)
-	}
-
 	if !viper.IsSet("rule_groups") || len(viper.GetStringSlice("rule_groups")) == 0 {
 		logger.Info("No rules defined in configuration; all audit logs will be printed")
 	}

cmd/setup.go

@@ -29,6 +29,11 @@ var setupCmd = &cobra.Command{
 	Short: "Setup vault audit device",
 	Long:  ``,
 	Run: func(cmd *cobra.Command, args []string) {
+		if viper.GetString("vault.token") == "" {
+			logger.Error("vault.token is required")
+			os.Exit(1)
+		}
+
 		client, err := vault.NewVaultClient(viper.GetString("vault.address"), vault.TokenAuth{Token: viper.GetString("vault.token")})
 		if err != nil {
 			logger.Error("setup", "unable to setup vault client", err.Error())

configs/development/config/config.yaml

@@ -20,3 +20,6 @@ rule_groups:
       max_backups: 5
       max_age: 30
       compress: true
+    forwarding:
+      enabled: true
+      address: "udp_receiver:9001"

configs/development/docker-compose.yaml

@@ -42,3 +42,11 @@ services:
       - IPC_LOCK
     entrypoint: "/scripts/writer.sh"
 
+  udp_receiver:
+    image: oraclelinux:9
+    container_name: udp_receiver
+    volumes:
+      - ./scripts:/scripts:ro
+    entrypoint: "/scripts/udp-receiver.sh"
+    ports:
+      - "9001:9001/udp"

configs/development/scripts/udp-receiver.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -x
+
+dnf -y install nc
+
+while true ; do
+    nc -lu -p 9001
+done
+

configs/development/scripts/writer.sh

@@ -1,19 +1,19 @@
 #!/bin/sh
 
-sleep 5
-
 set -x
+while true ; do
+    sleep 5
 
-vault kv put secret/data/myapp/config api_key=12345 environment=production
-vault kv metadata put -custom-metadata="replicate_to=vault_replica" secret/metadata/myapp/config
-vault kv metadata get secret/metadata/myapp/config
-
-vault kv put secret/data/myapp/database username=dbuser password=supersecret host=db.example.com port=5432
+    vault kv put secret/data/myapp/config api_key=12345 environment=production
+    vault kv metadata put -custom-metadata="replicate_to=vault_replica" secret/metadata/myapp/config
+    vault kv metadata get secret/metadata/myapp/config
 
-vault kv get -field=api_key secret/data/myapp/config
-vault kv get -format=json secret/data/myapp/database
+    vault kv put secret/data/myapp/database username=dbuser password=supersecret host=db.example.com port=5432
 
-vault kv list secret/metadata/myapp/
-vault kv delete secret/data/myapp/config
+    vault kv get -field=api_key secret/data/myapp/config
+    vault kv get -format=json secret/data/myapp/database
 
+    vault kv list secret/metadata/myapp/
+    vault kv delete secret/data/myapp/config
+done
 

go.mod

@@ -5,26 +5,28 @@ go 1.22.3
 require (
 	github.com/expr-lang/expr v1.16.9
 	github.com/hashicorp/vault/api v1.14.0
-	github.com/ncode/courier v0.0.0-20240805115708-0d51f4845b17
+	github.com/mattermost/mattermost-server/v6 v6.7.2
 	github.com/panjf2000/gnet v1.6.7
-	github.com/redis/go-redis/v9 v9.6.1
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.18.2
 	github.com/stretchr/testify v1.8.4
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
 require (
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/tidwall/btree v1.1.0 // indirect
-	github.com/tidwall/match v1.1.1 // indirect
-)
-
-require (
+	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cenkalti/backoff/v3 v3.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/dyatlov/go-opengraph v0.0.0-20210112100619-dae8665a5b09 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.3 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
+	github.com/go-test/deep v1.0.4 // indirect
+	github.com/google/uuid v1.4.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/graph-gophers/graphql-go v1.3.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -35,21 +37,44 @@ require (
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect
+	github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d // indirect
+	github.com/mattermost/logr/v2 v2.0.15 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/minio-go/v7 v7.0.24 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pborman/uuid v1.2.1 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/philhofer/fwd v1.1.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/rs/xid v1.4.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/tidwall/redcon v1.6.2 // indirect
+	github.com/tinylib/msgp v1.1.6 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/wiggin77/merror v1.0.3 // indirect
+	github.com/wiggin77/srslog v1.0.1 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
@@ -60,6 +85,6 @@ require (
 	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )