Update log writing with correct use of 'catcher' in the correlation engine.

Author: JocLRojas

correlation/api/newLogHandler.go

@@ -3,12 +3,13 @@ package api
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/utmstack/UTMStack/correlation/ti"
 	"io"
-	"log"
 	"net/http"
 	"time"
 
+	"github.com/threatwinds/go-sdk/catcher"
+	"github.com/utmstack/UTMStack/correlation/ti"
+
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
 	"github.com/tidwall/gjson"
@@ -29,7 +30,7 @@ func NewLog(c *gin.Context) {
 	if err != nil {
 		response["status"] = "error"
 		response["error"] = fmt.Sprintf("%v", err)
-		log.Println(response["error"])
+		catcher.Error("Failed to read request body", err, map[string]any{"status": http.StatusBadRequest})
 		c.JSON(http.StatusBadRequest, response)
 		return
 	}
@@ -38,7 +39,7 @@ func NewLog(c *gin.Context) {
 	if err := json.Unmarshal(body, &lo); err != nil {
 		response["status"] = "error"
 		response["error"] = fmt.Sprintf("%v", err)
-		log.Println(response["error"])
+		catcher.Error("Failed to read request body", err, map[string]any{"status": http.StatusBadRequest})
 		c.JSON(http.StatusBadRequest, response)
 		return
 	}
@@ -69,7 +70,10 @@ func NewLog(c *gin.Context) {
 		!gjson.Get(l, "dataSource").Exists() {
 		response["status"] = "error"
 		response["error"] = "The log doesn't have the required fields. Please be sure that you are sending the @timestamp in RFC3339Nano format, the dataType that could be windows, linux, iis, macos, ... and the dataSource that could be the Hostname or IP of the log source."
-		log.Printf("%s LOG: %s", response["error"], l)
+		catcher.Error("Log validation failed - missing required fields", nil, map[string]any{
+			"status":     http.StatusBadRequest,
+			"log_sample": l,
+		})
 		c.JSON(http.StatusBadRequest, response)
 		return
 	}

correlation/cache/cache.go

@@ -1,11 +1,11 @@
 package cache
 
 import (
-	"log"
 	"runtime"
 	"sync"
 	"time"
 
+	"github.com/threatwinds/go-sdk/catcher"
 	"github.com/tidwall/gjson"
 	"github.com/utmstack/UTMStack/correlation/rules"
 	"github.com/utmstack/UTMStack/correlation/utils"
@@ -19,10 +19,10 @@ var storage []string
 
 func Status() {
 	for {
-		log.Printf("Logs in cache: %v", len(storage))
+		catcher.Info("Logs in cache", map[string]any{"count": len(storage)})
 		if len(storage) != 0 {
 			est := gjson.Get(storage[0], "@timestamp").String()
-			log.Printf("Old document in cache: %s", est)
+			catcher.Info("Old document in cache", map[string]any{"timestamp": est})
 		}
 		time.Sleep(60 * time.Second)
 	}
@@ -47,7 +47,7 @@ func Search(allOf []rules.AllOf, oneOf []rules.OneOf, seconds int64) []string {
 		est := gjson.Get(storage[i], "@timestamp").String()
 		eit, err := time.Parse(time.RFC3339Nano, est)
 		if err != nil {
-			log.Printf("Could not parse @timestamp: %v", err)
+			catcher.Error("Could not parse @timestamp:", err, nil)
 			continue
 		}
 		if eit.Unix() < ait {
@@ -85,7 +85,7 @@ var logs = make(chan string, bufferSize)
 
 func AddToCache(l string) {
 	if len(logs) == bufferSize {
-		log.Printf("Buffer is full, you could be lossing events")
+		catcher.Info("Buffer is full, you could be lossing events", nil)
 		return
 	}
 	logs <- l
@@ -116,7 +116,7 @@ func Clean() {
 				old := gjson.Get(storage[0], "@timestamp").String()
 				oldTime, err := time.Parse(time.RFC3339Nano, old)
 				if err != nil {
-					log.Printf("Could not parse old log timestamp. Cleaning up")
+					catcher.Error("Could not parse old log timestamp. Cleaning up", err, nil)
 					clean = true
 				}
 

correlation/correlation/finder.go

@@ -4,9 +4,9 @@ import (
 	"bytes"
 	"fmt"
 	"html/template"
-	"log"
 	"time"
 
+	"github.com/threatwinds/go-sdk/catcher"
 	"github.com/utmstack/UTMStack/correlation/cache"
 	"github.com/utmstack/UTMStack/correlation/rules"
 	"github.com/utmstack/UTMStack/correlation/search"
@@ -15,13 +15,13 @@ import (
 
 func Finder(rule rules.Rule) {
 	if len(rule.DataTypes) == 0 {
-		log.Printf("Disabling rule '%s', because dataTypes is empty", rule.Name)
+		catcher.Info("Disabling rule, because dataTypes is empty", map[string]any{"name": rule.Name})
 		return
 	}
 
 	sleep, err := time.ParseDuration(fmt.Sprintf("%ds", rule.Frequency))
 	if err != nil {
-		log.Printf("Disabling rule '%s', because of error: '%v", rule.Name, err)
+		catcher.Error("Disabling rule", err, map[string]any{"name": rule.Name})
 		return
 	}
 
@@ -52,15 +52,15 @@ func Finder(rule rules.Rule) {
 			continue
 		}
 
-		log.Printf("Executing rule: %s", rule.Name)
+		catcher.Info("Executing rule", map[string]any{"name": rule.Name})
 
 		if len(rule.Cache) != 0 {
 			findInCache(rule)
 		} else if len(rule.Search) != 0 {
 			findInSearch(rule)
 		}
 
-		log.Printf("Execution of rule '%s' finished", rule.Name)
+		catcher.Info("Execution of rule finished", map[string]any{"name": rule.Name})
 
 		switch sleep {
 		case 0:
@@ -84,7 +84,7 @@ func findInSearch(rule rules.Rule) {
 				t := template.Must(template.New("query").Parse(query.Query))
 				err := t.Execute(&q, fields)
 				if err != nil {
-					log.Printf("Error while trying to process the query %v of the rule %s: %v", step+1, rule.Name, err)
+					catcher.Error("Error while trying to process the query", err, map[string]any{"step": step + 1, "rule": rule.Name})
 				} else {
 					l := search.Search(q.String())
 					processResponse(l, rule, query.Save, &tmpLogs, len(rule.Search), step, query.MinCount)
@@ -108,7 +108,7 @@ func findInCache(rule rules.Rule) {
 					t := template.Must(template.New("allOf").Parse(allOf.Value))
 					err := t.Execute(&value, fields)
 					if err != nil {
-						log.Printf("Error while trying to process the query %v of the rule %s: %v", step+1, rule.Name, err)
+						catcher.Error("Error while trying to process the query", err, map[string]any{"step": step + 1, "rule": rule.Name})
 					} else {
 						allOfList = append(allOfList, rules.AllOf{Field: allOf.Field, Operator: allOf.Operator, Value: value.String()})
 					}
@@ -120,7 +120,7 @@ func findInCache(rule rules.Rule) {
 					t := template.Must(template.New("oneOf").Parse(oneOf.Value))
 					err := t.Execute(&value, fields)
 					if err != nil {
-						log.Printf("Error while trying to process the query %v of the rule %s: %v", step+1, rule.Name, err)
+						catcher.Error("Error while trying to process the query", err, map[string]any{"step": step + 1, "rule": rule.Name})
 					} else {
 						oneOfList = append(oneOfList, rules.OneOf{Field: oneOf.Field, Operator: oneOf.Operator, Value: value.String()})
 					}

correlation/correlation/reporter.go

@@ -2,15 +2,16 @@ package correlation
 
 import (
 	"encoding/json"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/google/uuid"
 	"github.com/levigross/grequests"
+	"github.com/threatwinds/go-sdk/catcher"
 	"github.com/utmstack/UTMStack/correlation/geo"
 	"github.com/utmstack/UTMStack/correlation/search"
 	"github.com/utmstack/UTMStack/correlation/utils"
-	"log"
-	"strconv"
-	"strings"
-	"time"
 )
 
 type Host struct {
@@ -100,7 +101,7 @@ func Alert(name, severity, description, solution, category, tactic string, refer
 		}
 	}
 
-	log.Printf("Reporting alert: %s", name)
+	catcher.Info("Reporting alert", map[string]any{"name": name})
 
 	if !UpdateAlert(name, severity, fields) {
 		NewAlert(name, severity, description, solution, category, tactic, reference, dataType, dataSource,
@@ -113,7 +114,7 @@ func UpdateAlert(name, severity string, details map[string]string) bool {
 
 	index, err := search.IndexBuilder("alert", time.Now().UTC().Format(time.RFC3339Nano))
 	if err != nil {
-		log.Printf("Could not build index name: %v", err)
+		catcher.Error("Could not build index name", err, nil)
 		return true
 	}
 
@@ -208,7 +209,7 @@ func UpdateAlert(name, severity string, details map[string]string) bool {
 		JSON: request,
 	})
 	if err != nil {
-		log.Printf("Could not check existent alert: %v", err)
+		catcher.Error("Could not check existent alert", err, nil)
 		return false
 	}
 
@@ -221,7 +222,7 @@ func UpdateAlert(name, severity string, details map[string]string) bool {
 	err = json.Unmarshal([]byte(resultStr), &resultObj)
 
 	if err != nil {
-		log.Printf("Could not check existent alert: %v", err)
+		catcher.Error("Could not check existent alert", err, nil)
 		return false
 	}
 
@@ -242,7 +243,7 @@ func UpdateAlert(name, severity string, details map[string]string) bool {
 						},
 					})
 					if err != nil {
-						log.Printf("Could not update existent alert: %v", err)
+						catcher.Error("Could not update existent alert", err, nil)
 						return false
 					}
 
@@ -362,13 +363,13 @@ func NewAlert(name, severity, description, solution, category, tactic string, re
 			url := cnf.Elasticsearch + "/" + index + "/_doc"
 			_, err := utils.DoPost(url, "application/json", body)
 			if err != nil {
-				log.Printf("Could not send alert to Elasticsearch: %v", err)
+				catcher.Error("Could not send alert to Elasticsearch", err, nil)
 			}
 		} else {
-			log.Printf("Could not build index name: %v", err)
+			catcher.Error("Could not build index name", err, nil)
 		}
 	} else {
-		log.Printf("Could not encode alert in JSON: %v", err)
+		catcher.Error("Could not encode alert in JSON", err, nil)
 	}
 	time.Sleep(3 * time.Second)
 }

correlation/geo/bases.go

@@ -1,15 +1,16 @@
 package geo
 
 import (
-	"github.com/utmstack/UTMStack/correlation/utils"
-	"log"
 	"net"
 	"path/filepath"
 	"strconv"
+
+	"github.com/threatwinds/go-sdk/catcher"
+	"github.com/utmstack/UTMStack/correlation/utils"
 )
 
 func Load() {
-	log.Printf("Loading GeoIP databases")
+	catcher.Info("Loading GeoIP databases", nil)
 
 	var files = []string{
 		"asn-blocks-v4.csv",
@@ -38,10 +39,10 @@ func Load() {
 		}
 	}
 
-	log.Printf("asnBlocks rows: %v", len(asnBlocks))
-	log.Printf("cityBlocks rows: %v", len(cityBlocks))
-	log.Printf("cityLocations rows: %v", len(cityLocations))
-	log.Printf("GeoIP databases loaded")
+	catcher.Info("asnBlocks rows", map[string]any{"count": len(asnBlocks)})
+	catcher.Info("cityBlocks rows", map[string]any{"count": len(cityBlocks)})
+	catcher.Info("cityLocations rows", map[string]any{"count": len(cityLocations)})
+	catcher.Info("GeoIP databases loaded", map[string]any{})
 }
 
 func populateASNBlocks(csv [][]string) {
@@ -51,13 +52,13 @@ func populateASNBlocks(csv [][]string) {
 		}
 		_, n, err := net.ParseCIDR(line[0])
 		if err != nil {
-			log.Printf("Could not get CIDR in populateASNBlocks: %v", err)
+			catcher.Error("Could not get CIDR in populateASNBlocks", err, nil)
 			continue
 		}
 
 		asn, err := strconv.Atoi(line[1])
 		if err != nil {
-			log.Printf("Could not get ASN in populateASNBlocks: %v", err)
+			catcher.Error("Could not get ASN in populateASNBlocks", err, nil)
 			continue
 		}
 
@@ -78,7 +79,7 @@ func populateCityBlocks(csv [][]string) {
 		}
 		_, n, err := net.ParseCIDR(line[0])
 		if err != nil {
-			log.Printf("Could not parse CIDR in populateCityBlocks: %v", err)
+			catcher.Error("Could not parse CIDR in populateCityBlocks", err, nil)
 			continue
 		}
 
@@ -88,13 +89,13 @@ func populateCityBlocks(csv [][]string) {
 
 		geonameID, err := strconv.Atoi(line[1])
 		if err != nil {
-			log.Printf("Could not parse geonameID in populateCityBlocks: %v", err)
+			catcher.Error("Could not parse geonameID in populateCityBlocks", err, nil)
 			continue
 		}
 
 		isAnonymousProxy, err := strconv.Atoi(line[4])
 		if err != nil {
-			log.Printf("Could not parse isAnonymousProxy in populateCityBlocks: %v", err)
+			catcher.Error("Could not parse isAnonymousProxy in populateCityBlocks", err, nil)
 			continue
 		}
 
@@ -105,7 +106,7 @@ func populateCityBlocks(csv [][]string) {
 
 		isSatelliteProvider, err := strconv.Atoi(line[5])
 		if err != nil {
-			log.Printf("Could not parse isSatelliteProvider in populateCityBlocks: %v", err)
+			catcher.Error("Could not parse isSatelliteProvider in populateCityBlocks", err, nil)
 			continue
 		}
 
@@ -116,19 +117,19 @@ func populateCityBlocks(csv [][]string) {
 
 		latitude, err := strconv.ParseFloat(line[7], 64)
 		if err != nil {
-			log.Printf("Could not parse latitude in populateCityBlocks: %v", err)
+			catcher.Error("Could not parse latitude in populateCityBlocks", err, nil)
 			continue
 		}
 
 		longitude, err := strconv.ParseFloat(line[8], 64)
 		if err != nil {
-			log.Printf("Could not parse longitude in populateCityBlocks: %v", err)
+			catcher.Error("Could not parse longitude in populateCityBlocks", err, nil)
 			continue
 		}
 
 		accuracyRadius, err := strconv.Atoi(line[9])
 		if err != nil {
-			log.Printf("Could not parse accuracyRadius in populateCityBlocks: %v", err)
+			catcher.Error("Could not parse accuracyRadius in populateCityBlocks", err, nil)
 			continue
 		}
 
@@ -153,13 +154,13 @@ func populateCityLocations(csv [][]string) {
 		}
 		geonameID, err := strconv.Atoi(line[0])
 		if err != nil {
-			log.Printf("Could not parse geonameID in populateCityLocations: %v", err)
+			catcher.Error("Could not parse geonameID in populateCityLocations", err, nil)
 			continue
 		}
 
 		isInEuropeanUnion, err := strconv.Atoi(line[13])
 		if err != nil {
-			log.Printf("Could not parse isInEuropeanUnion in populateCityLocations: %v", err)
+			catcher.Error("Could not parse isInEuropeanUnion in populateCityLocations", err, nil)
 			continue
 		}