Skip to content
This repository was archived by the owner on Mar 4, 2025. It is now read-only.

Commit 28872d8

Browse files
justincliftjustinclift
committed
analysis: Add standalone utility to record disk space usage by users
Initial plan is to run this from cron periodically (ie every day), but we could turn this into a daemon if needed for some reason.
1 parent 46f4edf commit 28872d8

File tree

8 files changed

+190
-0
lines changed

8 files changed

+190
-0
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ _testmain.go
3535
request.log
3636
api/api
3737
db4s/db4s
38+
standalone/analysis/analysis
3839
webui/webui
3940

4041
# Cypress generated files

‎common/postgresql.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,54 @@ func AddUser(auth0ID, userName, password, email, displayName, avatarURL string)
103103
return nil
104104
}
105105

106+
// AnalysisRecordUserStorage adds a record to the backend database containing the amount of storage space used by a user
107+
func AnalysisRecordUserStorage(userName string, spaceUsedStandard, spaceUsedLive int64) (err error) {
108+
dbQuery := `
109+
WITH u AS (
110+
SELECT user_id
111+
FROM users
112+
WHERE lower(user_name) = lower($1)
113+
)
114+
INSERT INTO analysis_space_used (user_id, standard_databases_bytes, live_databases_bytes)
115+
VALUES ((SELECT user_id FROM u), $2, $3)`
116+
commandTag, err := pdb.Exec(context.Background(), dbQuery, userName, spaceUsedStandard, spaceUsedLive)
117+
if err != nil {
118+
log.Printf("Adding record of storage space used by '%s' failed: %s", userName, err)
119+
return
120+
}
121+
if numRows := commandTag.RowsAffected(); numRows != 1 {
122+
log.Printf("Wrong number of rows (%d) affected when recording the storage space used by '%s'", numRows, userName)
123+
}
124+
return
125+
}
126+
127+
// AnalysisUsersWithDBs returns the list of users with at least one database
128+
func AnalysisUsersWithDBs() (userList map[string]int, err error) {
129+
dbQuery := `
130+
SELECT u.user_name, count(*)
131+
FROM users u, sqlite_databases db
132+
WHERE u.user_id = db.user_id
133+
GROUP BY u.user_name`
134+
rows, err := pdb.Query(context.Background(), dbQuery)
135+
if err != nil {
136+
log.Printf("Database query failed in %s: %v", GetCurrentFunctionName(), err)
137+
return
138+
}
139+
defer rows.Close()
140+
userList = make(map[string]int)
141+
for rows.Next() {
142+
var user string
143+
var numDBs int
144+
err = rows.Scan(&user, &numDBs)
145+
if err != nil {
146+
log.Printf("Error in %s when getting the list of users with at least one database: %v", GetCurrentFunctionName(), err)
147+
return nil, err
148+
}
149+
userList[user] = numDBs
150+
}
151+
return
152+
}
153+
106154
// ApiCallLog records an API call operation. Database name is optional, as not all API calls operate on a
107155
// database. If a database name is provided however, then the database owner name *must* also be provided
108156
func ApiCallLog(loggedInUser, dbOwner, dbName, operation, callerSw string) {

‎database/migrations/000003_analysis_space_used.down.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
BEGIN;
2+
DROP TABLE IF EXISTS public.analysis_space_used;
3+
COMMIT;

‎database/migrations/000003_analysis_space_used.up.sql

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
BEGIN;
2+
CREATE TABLE IF NOT EXISTS public.analysis_space_used
3+
(
4+
id BIGSERIAL,
5+
user_id BIGINT NOT NULL
6+
CONSTRAINT analysis_space_used_users_user_id_fk REFERENCES public.users,
7+
analysis_date TIMESTAMPTZ DEFAULT now(),
8+
standard_databases_bytes BIGINT,
9+
live_databases_bytes BIGINT
10+
);
11+
COMMIT;

‎docker/Dockerfile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ RUN echo "cd ${DBHUB_SOURCE}" >> /usr/local/bin/compile.sh && \
106106
echo "PKG_CONFIG_PATH=/sqlite/lib/pkgconfig go build -gcflags \"all=-N -l\" -buildvcs=false -o /usr/local/bin/dbhub-db4s ." >> /usr/local/bin/compile.sh && \
107107
echo "cd ${DBHUB_SOURCE}/live" >> /usr/local/bin/compile.sh && \
108108
echo "PKG_CONFIG_PATH=/sqlite/lib/pkgconfig go build -gcflags \"all=-N -l\" -buildvcs=false -o /usr/local/bin/dbhub-live ." >> /usr/local/bin/compile.sh && \
109+
echo "cd ${DBHUB_SOURCE}/standalone/analysis" >> /usr/local/bin/compile.sh && \
110+
echo "PKG_CONFIG_PATH=/sqlite/lib/pkgconfig go build -gcflags \"all=-N -l\" -buildvcs=false -o /usr/local/bin/dbhub-analysis ." >> /usr/local/bin/compile.sh && \
111+
echo "ln --force -s /usr/local/bin/dbhub-analysis /etc/periodic/15min/" >> /usr/local/bin/compile.sh && \
109112
echo "cd ${DBHUB_SOURCE}/webui" >> /usr/local/bin/compile.sh && \
110113
echo "PKG_CONFIG_PATH=/sqlite/lib/pkgconfig go build -gcflags \"all=-N -l\" -buildvcs=false -o /usr/local/bin/dbhub-webui ." >> /usr/local/bin/compile.sh && \
111114
echo "/usr/local/bin/restart.sh" >> /usr/local/bin/compile.sh && \

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ require (
1616
github.com/aquilax/truncate v1.0.0
1717
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
1818
github.com/bradleypeabody/gorilla-sessions-memcache v0.0.0-20181103040241-659414f458e1
19+
github.com/docker/go-units v0.5.0
1920
github.com/go-playground/validator/v10 v10.3.0
2021
github.com/golang-migrate/migrate/v4 v4.15.3-0.20230407054901-84009cf2ab46
2122
github.com/gorilla/sessions v1.2.0

‎go.sum

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6
2222
github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE=
2323
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
2424
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
25+
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
2526
github.com/go-ini/ini v1.56.0 h1:6HjxSjqdmgnujDPhlzR4a44lxK3w03WPN8te0SoUSeM=
2627
github.com/go-ini/ini v1.56.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
2728
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=

‎standalone/analysis/main.go

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
package main
2+
3+
// Stand alone (non-daemon) utility to analyse various aspects of user resource usage
4+
// At some point this may be turned into a daemon, but for now it's likely just to be
5+
// run from cron on a periodic basis (ie every few hours)
6+
7+
import (
8+
"fmt"
9+
"log"
10+
11+
"github.com/docker/go-units"
12+
com "github.com/sqlitebrowser/dbhub.io/common"
13+
)
14+
15+
var (
16+
Debug = false
17+
)
18+
19+
func main() {
20+
// Read server configuration
21+
err := com.ReadConfig()
22+
if err != nil {
23+
log.Fatalf("Configuration file problem: '%s'", err)
24+
}
25+
26+
// Connect to PostgreSQL server
27+
err = com.ConnectPostgreSQL()
28+
if err != nil {
29+
log.Fatal(err)
30+
}
31+
32+
// Connect to MQ server
33+
com.Conf.Live.Nodename = "Usage Analysis"
34+
com.AmqpChan, err = com.ConnectMQ()
35+
if err != nil {
36+
log.Fatal(err)
37+
}
38+
39+
// Get the list of all users with at least one database
40+
userList, err := com.AnalysisUsersWithDBs()
41+
if err != nil {
42+
log.Fatalln(err)
43+
}
44+
45+
type dbSizes struct {
46+
Live int64
47+
Standard int64
48+
}
49+
userStorage := make(map[string]dbSizes)
50+
51+
// Loop through the users, calculating the total disk space used by each
52+
for user, numDBs := range userList {
53+
if Debug {
54+
fmt.Printf("User: %s, # databases: %d\n", user, numDBs)
55+
}
56+
57+
// Get the list of standard databases for a user
58+
dbList, err := com.UserDBs(user, com.DB_BOTH)
59+
if err != nil {
60+
log.Fatal(err)
61+
}
62+
63+
// For each standard database, count the list of commits and amount of space used
64+
var spaceUsedStandard int64
65+
for _, db := range dbList {
66+
commitList, err := com.GetCommitList(user, db.Database)
67+
if err != nil {
68+
log.Println(err)
69+
}
70+
71+
// Calculate space used by standard databases
72+
for _, commit := range commitList {
73+
tree := commit.Tree.Entries
74+
for _, j := range tree {
75+
spaceUsedStandard += j.Size
76+
}
77+
}
78+
79+
if Debug {
80+
fmt.Printf("User: %s, Standard database: %s, # Commits: %d, Space used: %s\n", user, db.Database, len(commitList), units.HumanSize(float64(spaceUsedStandard)))
81+
}
82+
}
83+
84+
// Get the list of live databases for a user
85+
liveList, err := com.LiveUserDBs(user, com.DB_BOTH)
86+
if err != nil {
87+
log.Fatal(err)
88+
}
89+
90+
// For each live database, get the amount of space used
91+
var spaceUsedLive int64
92+
for _, db := range liveList {
93+
_, liveNode, err := com.CheckDBLive(user, db.Database)
94+
if err != nil {
95+
log.Fatal(err)
96+
return
97+
}
98+
99+
// Ask our AMQP backend for the database size
100+
z, err := com.LiveSize(liveNode, user, user, db.Database)
101+
if err != nil {
102+
log.Fatal(err)
103+
}
104+
spaceUsedLive += z
105+
106+
if Debug {
107+
fmt.Printf("User: %s, Live database: %s, Space used: %s\n", user, db.Database, units.HumanSize(float64(spaceUsedLive)))
108+
}
109+
}
110+
userStorage[user] = dbSizes{Standard: spaceUsedStandard, Live: spaceUsedLive}
111+
}
112+
113+
// Store the information in our PostgreSQL backend
114+
for user, z := range userStorage {
115+
err = com.AnalysisRecordUserStorage(user, z.Standard, z.Live)
116+
if err != nil {
117+
log.Fatalln()
118+
}
119+
}
120+
121+
log.Printf("%s run complete", com.Conf.Live.Nodename)
122+
}

0 commit comments

Comments
 (0)