Skip to content

Commit 182b8c1

Browse files
gallypettegallypette
committed
chg: [sshd] daily, monthly and yearly statistics and plots
1 parent 688ca71 commit 182b8c1

File tree

2 files changed

+135
-62
lines changed

2 files changed

+135
-62
lines changed

‎logparser/sshd.go‎

Lines changed: 134 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -68,34 +68,63 @@ func (s *SshdParser) Parse(logline string) error {
6868
r.Close()
6969
return err
7070
}
71-
_, err = redis.String(r.Do("ZINCRBY", fmt.Sprintf("%v%v%v:statssrc", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day()), 1, md["src"]))
71+
72+
// Daily
73+
dstr := fmt.Sprintf("%v%v%v", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day())
74+
err = compileStats(s, dstr, "daily", md["src"], md["username"], md["host"])
7275
if err != nil {
7376
r.Close()
7477
return err
7578
}
76-
_, err = redis.String(r.Do("ZINCRBY", fmt.Sprintf("%v%v%v:statsusername", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day()), 1, md["username"]))
79+
80+
// Monthly
81+
mstr := fmt.Sprintf("%v%v", parsedTime.Year(), int(parsedTime.Month()))
82+
err = compileStats(s, mstr, "daily", md["src"], md["username"], md["host"])
7783
if err != nil {
7884
r.Close()
7985
return err
8086
}
81-
_, err = redis.String(r.Do("ZINCRBY", fmt.Sprintf("%v%v%v:statshost", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day()), 1, md["host"]))
87+
88+
// Yearly
89+
ystr := fmt.Sprintf("%v", parsedTime.Year())
90+
err = compileStats(s, ystr, "daily", md["src"], md["username"], md["host"])
8291
if err != nil {
8392
r.Close()
8493
return err
8594
}
8695

87-
// Keeping track of which days we updated statistics for
88-
_, err = redis.Int(r.Do("SADD", "toupdate", fmt.Sprintf("%v%v%v:statssrc", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day())))
96+
return nil
97+
}
98+
99+
func compileStats(s *SshdParser, datestr string, mode string, src string, username string, host string) error {
100+
r := *s.r1
101+
_, err := redis.String(r.Do("ZINCRBY", fmt.Sprintf("%v:%v", datestr, "statssrc"), 1, src))
89102
if err != nil {
90103
r.Close()
91104
return err
92105
}
93-
_, err = redis.Int(r.Do("SADD", "toupdate", fmt.Sprintf("%v%v%v:statsusername", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day())))
106+
_, err = redis.String(r.Do("ZINCRBY", fmt.Sprintf("%v:%v", datestr, "statsusername"), 1, username))
94107
if err != nil {
95108
r.Close()
96109
return err
97110
}
98-
_, err = redis.Int(r.Do("SADD", "toupdate", fmt.Sprintf("%v%v%v:statshost", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day())))
111+
_, err = redis.String(r.Do("ZINCRBY", fmt.Sprintf("%v:%v", datestr, "statshost"), 1, host))
112+
if err != nil {
113+
r.Close()
114+
return err
115+
}
116+
117+
_, err = redis.Int(r.Do("SADD", fmt.Sprintf("toupdate:%v", mode), fmt.Sprintf("%v:%v", datestr, "statssrc")))
118+
if err != nil {
119+
r.Close()
120+
return err
121+
}
122+
_, err = redis.Int(r.Do("SADD", fmt.Sprintf("toupdate:%v", mode), fmt.Sprintf("%v:%v", datestr, "statsusername")))
123+
if err != nil {
124+
r.Close()
125+
return err
126+
}
127+
_, err = redis.Int(r.Do("SADD", fmt.Sprintf("toupdate:%v", mode), fmt.Sprintf("%v:%v", datestr, "statshost")))
99128
if err != nil {
100129
r.Close()
101130
return err
@@ -114,88 +143,132 @@ func (s *SshdParser) Compile() error {
114143
return err
115144
}
116145

117-
// List days for which we need to update statistic
118-
toupdate, err := redis.Strings(r.Do("SMEMBERS", "toupdate"))
146+
// List days for which we need to update statistics
147+
toupdateD, err := redis.Strings(r.Do("SMEMBERS", "toupdate:daily"))
119148
if err != nil {
120149
r.Close()
121150
return err
122151
}
123152

124-
// Query statistics dor each day to update
125-
for _, v := range toupdate {
126-
zrank, err := redis.Strings(r.Do("ZRANGEBYSCORE", v, "-inf", "+inf", "WITHSCORES"))
153+
// Plot statistics for each day to update
154+
for _, v := range toupdateD {
155+
err = plotStats(s, v)
127156
if err != nil {
128157
r.Close()
129158
return err
130159
}
160+
}
131161

132-
// Split keys and values - keep these ordered
133-
values := plotter.Values{}
134-
keys := make([]string, 0, len(zrank)/2)
135-
136-
for k, v := range zrank {
137-
// keys
138-
if (k % 2) == 0 {
139-
keys = append(keys, zrank[k])
140-
// values
141-
} else {
142-
fv, _ := strconv.ParseFloat(v, 64)
143-
values = append(values, fv)
144-
}
145-
}
162+
// List months for which we need to update statistics
163+
toupdateM, err := redis.Strings(r.Do("SMEMBERS", "toupdate:monthly"))
164+
if err != nil {
165+
r.Close()
166+
return err
167+
}
146168

147-
p, err := plot.New()
169+
// Plot statistics for each month to update
170+
for _, v := range toupdateM {
171+
err = plotStats(s, v)
148172
if err != nil {
149-
panic(err)
173+
r.Close()
174+
return err
150175
}
176+
}
151177

152-
stype := strings.Split(v, ":")
153-
switch stype[1] {
154-
case "statsusername":
155-
p.Title.Text = "Usernames"
156-
case "statssrc":
157-
p.Title.Text = "Source IP"
158-
case "statshost":
159-
p.Title.Text = "Host"
160-
default:
161-
p.Title.Text = ""
162-
log.Println("We should not reach this point, open an issue.")
163-
}
178+
// List years for which we need to update statistics
179+
toupdateY, err := redis.Strings(r.Do("SMEMBERS", "toupdate:yearly"))
180+
if err != nil {
181+
r.Close()
182+
return err
183+
}
164184

165-
p.Y.Label.Text = "Count"
166-
w := 0.5 * vg.Centimeter
167-
bc, err := plotter.NewBarChart(values, w)
168-
bc.Horizontal = true
185+
// Plot statistics for each year to update
186+
for _, v := range toupdateY {
187+
err = plotStats(s, v)
169188
if err != nil {
189+
r.Close()
170190
return err
171191
}
172-
bc.LineStyle.Width = vg.Length(0)
173-
bc.Color = plotutil.Color(0)
192+
}
193+
194+
return nil
195+
}
174196

175-
p.Add(bc)
176-
p.NominalY(keys...)
197+
func plotStats(s *SshdParser, v string) error {
198+
r := *s.r2
199+
zrank, err := redis.Strings(r.Do("ZRANGEBYSCORE", v, "-inf", "+inf", "WITHSCORES"))
200+
if err != nil {
201+
r.Close()
202+
return err
203+
}
177204

178-
// Create folder to store plots
205+
// Split keys and values - keep these ordered
206+
values := plotter.Values{}
207+
keys := make([]string, 0, len(zrank)/2)
179208

180-
if _, err := os.Stat("data"); os.IsNotExist(err) {
181-
err := os.Mkdir("data", 0700)
182-
if err != nil {
183-
return err
184-
}
209+
for k, v := range zrank {
210+
// keys
211+
if (k % 2) == 0 {
212+
keys = append(keys, zrank[k])
213+
// values
214+
} else {
215+
fv, _ := strconv.ParseFloat(v, 64)
216+
values = append(values, fv)
185217
}
218+
}
186219

187-
if _, err := os.Stat(filepath.Join("data", stype[0])); os.IsNotExist(err) {
188-
err := os.Mkdir(filepath.Join("data", stype[0]), 0700)
189-
if err != nil {
190-
return err
191-
}
220+
p, err := plot.New()
221+
if err != nil {
222+
panic(err)
223+
}
224+
225+
stype := strings.Split(v, ":")
226+
fmt.Println(stype[0])
227+
fmt.Println(stype[1])
228+
switch stype[1] {
229+
case "statsusername":
230+
p.Title.Text = "Usernames"
231+
case "statssrc":
232+
p.Title.Text = "Source IP"
233+
case "statshost":
234+
p.Title.Text = "Host"
235+
default:
236+
p.Title.Text = ""
237+
log.Println("We should not reach this point, open an issue.")
238+
}
239+
240+
p.Y.Label.Text = "Count"
241+
w := 0.5 * vg.Centimeter
242+
bc, err := plotter.NewBarChart(values, w)
243+
bc.Horizontal = true
244+
if err != nil {
245+
return err
246+
}
247+
bc.LineStyle.Width = vg.Length(0)
248+
bc.Color = plotutil.Color(0)
249+
250+
p.Add(bc)
251+
p.NominalY(keys...)
252+
253+
// Create folder to store plots
254+
255+
if _, err := os.Stat("data"); os.IsNotExist(err) {
256+
err := os.Mkdir("data", 0700)
257+
if err != nil {
258+
return err
192259
}
260+
}
193261

194-
xsize := 3 + vg.Length(math.Round(float64(len(keys)/2)))
195-
if err := p.Save(15*vg.Centimeter, xsize*vg.Centimeter, filepath.Join("data", stype[0], fmt.Sprintf("%v.svg", v))); err != nil {
262+
if _, err := os.Stat(filepath.Join("data", stype[0])); os.IsNotExist(err) {
263+
err := os.Mkdir(filepath.Join("data", stype[0]), 0700)
264+
if err != nil {
196265
return err
197266
}
267+
}
198268

269+
xsize := 3 + vg.Length(math.Round(float64(len(keys)/2)))
270+
if err := p.Save(15*vg.Centimeter, xsize*vg.Centimeter, filepath.Join("data", stype[0], fmt.Sprintf("%v.svg", v))); err != nil {
271+
return err
199272
}
200273

201274
return nil

‎main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ var (
5050
redisD4 redis.Conn
5151
redisParsers *redis.Pool
5252
parsers = [1]string{"sshd"}
53-
compilationTrigger = 200
53+
compilationTrigger = 20
5454
wg sync.WaitGroup
5555
compiling comutex
5656
torun = []logparser.Parser{}

0 commit comments

Comments
 (0)