add: [init] parser logic - not functional

Author: gallypette

logparser/parser.go

@@ -1 +1,13 @@
 package logparser
+
+type (
+	// Parser provides the interface for a Parser
+	// It should provide:
+	//  Parse to parse a line of log
+	//  GetAttributes to get list of attributes (map keys)
+	Parser interface {
+		Parse() error
+		Push() error
+		Pop() map[string]string
+	}
+)

logparser/sshd.go

@@ -0,0 +1,44 @@
+package logparser
+
+import "github.com/gomodule/redigo/redis"
+
+// Sshd is a struct that corresponds to a line
+type Sshd struct {
+	Date string
+	Host string
+	User string
+	Src  string
+}
+
+// SshdParser Holds a struct that corresponds to a sshd log line
+// and the redis connection
+type SshdParser struct {
+	logs Sshd
+	r    *redis.Conn
+}
+
+// New Creates a new sshd parser
+func New(rconn *redis.Conn) *SshdParser {
+	return &SshdParser{
+		logs: Sshd{},
+		r:    rconn,
+	}
+}
+
+// Parse parses a line of sshd log
+func (s *SshdParser) Parse() error {
+	//TODO
+	return nil
+}
+
+// Push pushed the parsed line into redis
+func (s *SshdParser) Push() error {
+	//TODO
+	return nil
+}
+
+// Pop returns the list of attributes
+func (s *SshdParser) Pop() map[string]string {
+	//TODO
+	return nil
+}

main.go

@@ -1,34 +1,46 @@
 package main
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	"log"
 	"os"
 	"os/signal"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/D4-project/analyzer-d4-log/logparser"
 	config "github.com/D4-project/d4-golang-utils/config"
 	"github.com/gomodule/redigo/redis"
 )
 
 type (
-	conf struct {
+	redisconfD4 struct {
 		redisHost  string
 		redisPort  string
 		redisDB    int
 		redisQueue string
-		httpHost   string
-		httpPort   string
+	}
+	redisconfParsers struct {
+		redisHost    string
+		redisPort    string
+		redisDBCount int
+	}
+	conf struct {
+		httpHost string
+		httpPort string
 	}
 )
 
 // Setting up flags
 var (
-	confdir = flag.String("c", "conf.sample", "configuration directory")
-	cr      redis.Conn
+	confdir      = flag.String("c", "conf.sample", "configuration directory")
+	all          = flag.Bool("a", true, "run all parsers when set. Set by default")
+	specific     = flag.String("o", "", "run only a specific parser [sshd]")
+	redisD4      redis.Conn
+	redisParsers *redis.Pool
+	parsers      = [1]string{"sshd"}
 )
 
 func main() {
@@ -61,14 +73,17 @@ func main() {
 		fmt.Printf("\n")
 		fmt.Printf("The configuration directory should hold the following files\n")
 		fmt.Printf("to specify the settings to use:\n\n")
-		fmt.Printf(" mandatory: redis - host:port/db\n")
+		fmt.Printf(" mandatory: redis_d4 - host:port/db\n")
 		fmt.Printf(" mandatory: redis_queue - uuid\n")
+		fmt.Printf(" mandatory: redis_parsers - host:port/maxdb\n")
 		fmt.Printf(" optional: http_server - host:port\n\n")
 		fmt.Printf("See conf.sample for an example.\n")
 	}
 
 	// Config
-	c := conf{}
+	// c := conf{}
+	rd4 := redisconfD4{}
+	rp := redisconfParsers{}
 	flag.Parse()
 	if flag.NFlag() == 0 || *confdir == "" {
 		flag.Usage()
@@ -78,31 +93,76 @@ func main() {
 		*confdir = strings.TrimSuffix(*confdir, "\\")
 	}
 
-	// Parse Redis Config
-	tmp := config.ReadConfigFile(*confdir, "redis")
+	// Parse Redis D4 Config
+	tmp := config.ReadConfigFile(*confdir, "redis_d4")
 	ss := strings.Split(string(tmp), "/")
 	if len(ss) <= 1 {
-		log.Fatal("Missing Database in Redis config: should be host:port/database_name")
+		log.Fatal("Missing Database in Redis D4 config: should be host:port/database_name")
 	}
-	c.redisDB, _ = strconv.Atoi(ss[1])
+	rd4.redisDB, _ = strconv.Atoi(ss[1])
 	var ret bool
 	ret, ss[0] = config.IsNet(ss[0])
 	if !ret {
 		sss := strings.Split(string(ss[0]), ":")
-		c.redisHost = sss[0]
-		c.redisPort = sss[1]
+		rd4.redisHost = sss[0]
+		rd4.redisPort = sss[1]
+	}
+	rd4.redisQueue = string(config.ReadConfigFile(*confdir, "redis_queue"))
+	// Connect to D4 Redis
+	redisD4, err = redis.Dial("tcp", rd4.redisHost+":"+rd4.redisPort, redis.DialDatabase(rd4.redisDB))
+	if err != nil {
+		log.Fatal(err)
 	}
-	c.redisQueue = string(config.ReadConfigFile(*confdir, "redis_queue"))
-	initRedis(c.redisHost, c.redisPort, c.redisDB)
-	defer cr.Close()
+	defer redisD4.Close()
+
+	// Parse Redis Parsers Config
+	tmp = config.ReadConfigFile(*confdir, "redis_parsers")
+	ss = strings.Split(string(tmp), "/")
+	if len(ss) <= 1 {
+		log.Fatal("Missing Database Count in Redis config: should be host:port/max number of DB")
+	}
+	rp.redisDBCount, _ = strconv.Atoi(ss[1])
+	ret, ss[0] = config.IsNet(string(tmp))
+	if !ret {
+		sss := strings.Split(string(ss[0]), ":")
+		rp.redisHost = sss[0]
+		rp.redisPort = sss[1]
+	}
+
+	// Create a connection Pool
+	redisParsers = newPool(rp.redisHost+":"+rp.redisPort, rp.redisDBCount)
+
+	// Init parser depending on the parser flags:
+	if *all {
+		// Init all parsers
+		var torun = []logparser.Parser{}
+		for _, v := range parsers {
+			switch v {
+			case "sshd":
+				var sshdrcon, err = redisParsers.Dial()
+				if err != nil {
+					log.Fatal("Could not connect to Parser Redis")
+				}
+				sshd := logparser.New(&sshdrcon)
+				torun = append(torun, sshd)
+			}
+		}
+	} else if *specific != "" {
+		log.Println("TODO should run specific parser here")
+	}
+
+	// Run the parsers
+	log.Println("TODO should run the parsers here")
 
 	log.Println("Exit")
 }
 
-func initRedis(host string, port string, d int) {
-	err := errors.New("")
-	cr, err = redis.Dial("tcp", host+":"+port, redis.DialDatabase(d))
-	if err != nil {
-		log.Fatal(err)
+func newPool(addr string, maxconn int) *redis.Pool {
+	return &redis.Pool{
+		MaxActive:   maxconn,
+		MaxIdle:     3,
+		IdleTimeout: 240 * time.Second,
+		// Dial or DialContext must be set. When both are set, DialContext takes precedence over Dial.
+		Dial: func() (redis.Conn, error) { return redis.Dial("tcp", addr) },
 	}
 }