Skip to content

Commit 55bc7bc

Browse files
authored
Merge pull request #1 from SBOsoft/serkan-dev
Initial version. Experimental and WIP
2 parents 92e47dd + 0071bcc commit 55bc7bc

File tree

16 files changed

+3072
-0
lines changed

16 files changed

+3072
-0
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,4 @@ go.work.sum
2323

2424
# env file
2525
.env
26+
test-data

README.md

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,32 @@
11
# SBOLogProcessor
22
Log processor for SBOanalytics
3+
4+
The purpose of this tool is to monitor web server to:
5+
6+
1. Generate metrics
7+
2. Allow users to view realtime metrics by running the tool on the command line
8+
3. Push metrics to a SBOanalytics database
9+
4. Report on web site visitors
10+
5. Report on non-human visitors, e.g google bot, scanners, malicious actors
11+
12+
13+
The primary goal is to provide realtime metrics from the command line by following realtime changes to log files,
14+
e.g top offending IPs (e.g so you can block them).
15+
The secondary goal of this tool is to generate insights about web site visitors, to replace Google analytics.
16+
17+
18+
# Build and Run
19+
20+
## Development
21+
22+
### Build
23+
24+
Build
25+
```go build -o ./output/bin/sbologc```
26+
27+
Clean
28+
```go clean```
29+
30+
### Run
31+
32+
```go run . -f=true ./test-data/testfile.txt```

db/mysqldb.go

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package db
2+
3+
import (
4+
"database/sql"
5+
"log/slog"
6+
7+
"github.com/go-sql-driver/mysql"
8+
9+
"github.com/SBOsoft/SBOLogProcessor/metrics"
10+
)
11+
12+
type SBOAnalyticsDB struct {
13+
DbInstance *sql.DB
14+
}
15+
16+
func NewSBOAnalyticsDB() *SBOAnalyticsDB {
17+
rv := SBOAnalyticsDB{}
18+
return &rv
19+
}
20+
21+
func (sboadb *SBOAnalyticsDB) Init(dbUser string, dbPassword string, dbAddress string, databaseName string) (bool, error) {
22+
cfg := mysql.NewConfig()
23+
cfg.User = dbUser
24+
cfg.Passwd = dbPassword
25+
cfg.Net = "tcp"
26+
cfg.Addr = dbAddress
27+
cfg.DBName = databaseName
28+
29+
var err error
30+
sboadb.DbInstance, err = sql.Open("mysql", cfg.FormatDSN())
31+
if err != nil {
32+
slog.Error("Failed to connect to mysql db", "error", err)
33+
return false, err
34+
}
35+
36+
pingErr := sboadb.DbInstance.Ping()
37+
if pingErr != nil {
38+
slog.Error("Failed to ping the db after connection", "error", pingErr)
39+
return false, err
40+
}
41+
return true, nil
42+
}
43+
44+
func (sboadb *SBOAnalyticsDB) Close() (bool, error) {
45+
err := sboadb.DbInstance.Close()
46+
if err != nil {
47+
return false, err
48+
}
49+
return true, nil
50+
}
51+
52+
func (sboadb *SBOAnalyticsDB) GetDomainId(domainName string) (int, error) {
53+
result, err := sboadb.DbInstance.Exec("INSERT INTO sbo_domains (domain_name, created) VALUES (?, now())", domainName)
54+
if err != nil {
55+
//exists? try to select
56+
var domainId int
57+
row := sboadb.DbInstance.QueryRow("SELECT domain_id FROM sbo_domains WHERE domain_name = ?", domainName)
58+
err := row.Scan(&domainId)
59+
if err != nil {
60+
if err == sql.ErrNoRows {
61+
slog.Error("Unexpected error. db.GetDomainId could not create a new domain but selecting the existing record also failed", "domainName", domainName)
62+
}
63+
return -1, err
64+
} else {
65+
return domainId, nil
66+
}
67+
} else {
68+
newDomainId, err := result.LastInsertId()
69+
return int(newDomainId), err
70+
}
71+
}
72+
73+
func (sboadb *SBOAnalyticsDB) GetFileId(domainId int, hostname string, filePath string) (int, error) {
74+
result, err := sboadb.DbInstance.Exec("INSERT INTO sbo_log_files (domain_id, host_name, file_path, created) VALUES (?, ?, ?, now())", domainId, hostname, filePath)
75+
if err != nil {
76+
//exists? try to select
77+
var fileId int
78+
row := sboadb.DbInstance.QueryRow("SELECT file_id FROM sbo_log_files WHERE domain_id = ? AND host_name=? AND file_path=?", domainId, hostname, filePath)
79+
err := row.Scan(&fileId)
80+
if err != nil {
81+
if err == sql.ErrNoRows {
82+
slog.Error("Unexpected error. db.GetFileId could not create a new log file entry but selecting the existing record also failed", "domainId", domainId, "hostname", hostname, "filePath", filePath)
83+
}
84+
return -1, err
85+
} else {
86+
return fileId, nil
87+
}
88+
} else {
89+
newFileId, err := result.LastInsertId()
90+
return int(newFileId), err
91+
}
92+
}
93+
94+
func (sboadb *SBOAnalyticsDB) SaveMetricData(data *metrics.SBOMetricWindowDataToBeSaved, domainId int, replaceIfExists bool) (bool, error) {
95+
var sql string = "INSERT INTO sbo_metrics (domain_id, metric_type, key_value, time_window, metric_value, created) " +
96+
" VALUES (?, ?, ?, ?, ?, now()) "
97+
if replaceIfExists {
98+
sql += "ON DUPLICATE KEY UPDATE metric_value=VALUES(metric_value)"
99+
} else {
100+
sql += "ON DUPLICATE KEY UPDATE metric_value=metric_value+VALUES(metric_value)"
101+
}
102+
_, err := sboadb.DbInstance.Exec(sql, domainId, data.MetricType, data.KeyValue, data.TimeWindow, data.MetricValue)
103+
if err != nil {
104+
slog.Error("SaveMetricData failed", "domainId", domainId, "data.FilePath", data.FilePath, "error", err)
105+
return false, err
106+
} else {
107+
return true, nil
108+
}
109+
}

go.mod

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
module github.com/SBOsoft/SBOLogProcessor
2+
3+
go 1.23.9
4+
5+
toolchain go1.24.3
6+
7+
require github.com/fsnotify/fsnotify v1.9.0
8+
9+
require (
10+
filippo.io/edwards25519 v1.1.0 // indirect
11+
github.com/go-sql-driver/mysql v1.9.3 // indirect
12+
golang.org/x/sys v0.33.0 // indirect
13+
)

go.sum

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
2+
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
3+
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
4+
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
5+
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
6+
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
7+
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
8+
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
9+
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
10+
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11+
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
12+
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=

0 commit comments

Comments
 (0)