Skip to content
This repository was archived by the owner on Mar 4, 2025. It is now read-only.

Commit c57a409

Browse files
justincliftjustinclift
committed
api, common, live, webui: Use a generated bucket name and id for live databases
With this commit (which needs some database fields added) we move to using randonly generated strings for the Minio bucket name and object id when storing live databases. The randomly generated string used as the bucket name is kept in each users database record, so all of a users live databases will be stored in the same directory in the Minio backend. The object id is uniquely generated for each database. Manual PostgreSQL database changes needed for this commit: alter table public.users add live_minio_bucket_name text; alter table public.sqlite_databases add live_minio_object_id text;
1 parent cede96b commit c57a409

File tree

13 files changed

+295
-190
lines changed

13 files changed

+295
-190
lines changed

‎api/handlers.go

Lines changed: 41 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -122,14 +122,11 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) {
122122
return
123123
}
124124

125-
// If a live database has been uploaded but doesn't have a live node handling its requests, then create one
125+
// If a live database has been uploaded but doesn't have a live node handling its requests, then error out as this
126+
// should never happen
126127
if isLive && liveNode == "" {
127-
// Send a request to the AMQP backend to setup a node with the database
128-
err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, com.SetToPrivate)
129-
if err != nil {
130-
jsonErr(w, err.Error(), http.StatusInternalServerError)
131-
return
132-
}
128+
jsonErr(w, "No AMQP node available for request", http.StatusInternalServerError)
129+
return
133130
}
134131

135132
// If it's a standard database, process it locally. Else send the query to our AMQP backend
@@ -366,57 +363,43 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
366363
}
367364
}
368365

369-
// Delete the database
370-
err = com.DeleteDatabase(dbOwner, dbName)
371-
if err != nil {
372-
jsonErr(w, err.Error(), http.StatusInternalServerError)
373-
return
374-
}
375-
376366
// For a live database, delete it from both Minio and our AMQP backend
367+
var bucket, id string
377368
if isLive {
378-
// Delete the database from Minio
379-
bucket := fmt.Sprintf("live-%s", dbOwner)
380-
id := dbName
381-
err = com.MinioDeleteDatabase("API server", dbOwner, dbName, bucket, id)
369+
// Get the Minio bucket and object names for this database
370+
bucket, id, err = com.LiveGetMinioNames(loggedInUser, dbOwner, dbName)
382371
if err != nil {
383372
jsonErr(w, err.Error(), http.StatusInternalServerError)
384373
return
385374
}
386375

387-
// Delete the database from our AMQP backend
388-
var rawResponse []byte
389-
rawResponse, err = com.MQRequest(com.AmqpChan, liveNode, "delete", loggedInUser, dbOwner, dbName, "")
376+
// Delete the database from Minio
377+
err = com.MinioDeleteDatabase("API server", dbOwner, dbName, bucket, id)
390378
if err != nil {
391379
jsonErr(w, err.Error(), http.StatusInternalServerError)
392-
log.Println(err)
393380
return
394381
}
395382

396-
// Decode the response
397-
var resp com.LiveDBErrorResponse
398-
err = json.Unmarshal(rawResponse, &resp)
383+
// Delete the database from our AMQP backend
384+
err = com.LiveDelete(liveNode, loggedInUser, dbOwner, dbName)
399385
if err != nil {
400386
jsonErr(w, err.Error(), http.StatusInternalServerError)
401-
log.Println(err)
402-
return
403-
}
404-
if resp.Error != "" {
405-
err = errors.New(resp.Error)
406-
jsonErr(w, err.Error(), http.StatusInternalServerError)
407-
return
408-
}
409-
if resp.Node == "" {
410-
log.Printf("In API (Live) deleteHandler(). A node responded, but didn't identify itself.")
411387
return
412388
}
413389
}
414390

391+
// Delete the database in PostgreSQL
392+
err = com.DeleteDatabase(dbOwner, dbName)
393+
if err != nil {
394+
jsonErr(w, err.Error(), http.StatusInternalServerError)
395+
return
396+
}
397+
415398
// Return a "success" message
416399
z := com.StatusResponseContainer{Status: "OK"}
417400
jsonData, err := json.MarshalIndent(z, "", " ")
418401
if err != nil {
419-
log.Printf("Error when JSON marshalling returned data in deleteHandler(): %v\n", err)
402+
log.Printf("Error when JSON marshalling returned data in deleteHandler(): %v", err)
420403
jsonErr(w, err.Error(), http.StatusInternalServerError)
421404
return
422405
}
@@ -684,15 +667,11 @@ func executeHandler(w http.ResponseWriter, r *http.Request) {
684667
return
685668
}
686669

687-
// If a live database has been uploaded but doesn't have a live node handling its requests, then create one
670+
// If a live database has been uploaded but doesn't have a live node handling its requests, then error out as this
671+
// should never happen
688672
if isLive && liveNode == "" {
689-
// Send a request to the AMQP backend to set up the database there, ready for querying
690-
err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, com.SetToPrivate)
691-
if err != nil {
692-
log.Println(err) // FIXME: Debug output while developing
693-
jsonErr(w, err.Error(), http.StatusInternalServerError)
694-
return
695-
}
673+
jsonErr(w, "No AMQP node available for request", http.StatusInternalServerError)
674+
return
696675
}
697676

698677
// Send the SQL execution request to our AMQP backend
@@ -737,14 +716,11 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) {
737716
return
738717
}
739718

740-
// If a live database has been uploaded but doesn't have a live node handling its requests, then create one
719+
// If a live database has been uploaded but doesn't have a live node handling its requests, then error out as this
720+
// should never happen
741721
if isLive && liveNode == "" {
742-
// Send a request to the AMQP backend to set up a node with the database
743-
err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, com.SetToPrivate)
744-
if err != nil {
745-
jsonErr(w, err.Error(), http.StatusInternalServerError)
746-
return
747-
}
722+
jsonErr(w, "No AMQP node available for request", http.StatusInternalServerError)
723+
return
748724
}
749725

750726
// If it's a standard database, process it locally. Else send the query to our AMQP backend
@@ -937,15 +913,11 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
937913
return
938914
}
939915

940-
// If a live database has been uploaded but doesn't have a live node handling its requests, then create one
916+
// If a live database has been uploaded but doesn't have a live node handling its requests, then error out as this
917+
// should never happen
941918
if isLive && liveNode == "" {
942-
// Send a request to the AMQP backend to set up the database there, ready for querying
943-
err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, com.SetToPrivate)
944-
if err != nil {
945-
log.Println(err) // FIXME: Debug output while developing
946-
jsonErr(w, err.Error(), http.StatusInternalServerError)
947-
return
948-
}
919+
jsonErr(w, "No AMQP node available for request", http.StatusInternalServerError)
920+
return
949921
}
950922

951923
// Run the query
@@ -1068,14 +1040,11 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) {
10681040
return
10691041
}
10701042

1071-
// If a live database has been uploaded but doesn't have a live node handling its requests, then create one
1043+
// If a live database has been uploaded but doesn't have a live node handling its requests, then error out as this
1044+
// should never happen
10721045
if isLive && liveNode == "" {
1073-
// Send a request to the AMQP backend to setup a node with the database
1074-
err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, com.SetToPrivate)
1075-
if err != nil {
1076-
jsonErr(w, err.Error(), http.StatusInternalServerError)
1077-
return
1078-
}
1046+
jsonErr(w, "No AMQP node available for request", http.StatusInternalServerError)
1047+
return
10791048
}
10801049

10811050
// If it's a standard database, process it locally. Else send the query to our AMQP backend
@@ -1335,7 +1304,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
13351304
}
13361305

13371306
// Store the database in Minio
1338-
err = com.LiveStoreDatabaseMinio(tempDB, dbOwner, dbName, numBytes)
1307+
objectID, err := com.LiveStoreDatabaseMinio(tempDB, dbOwner, dbName, numBytes)
13391308
if err != nil {
13401309
jsonErr(w, err.Error(), http.StatusInternalServerError)
13411310
return
@@ -1346,7 +1315,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
13461315
com.SanitiseLogString(dbOwner), com.SanitiseLogString(dbName), numBytes)
13471316

13481317
// Send a request to the AMQP backend to set up the database there, ready for querying
1349-
err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, com.SetToPrivate)
1318+
err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, objectID, com.SetToPrivate)
13501319
if err != nil {
13511320
log.Println(err)
13521321
jsonErr(w, err.Error(), http.StatusInternalServerError)
@@ -1405,14 +1374,11 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) {
14051374
return
14061375
}
14071376

1408-
// If a live database has been uploaded but doesn't have a live node handling its requests, then create one
1377+
// If a live database has been uploaded but doesn't have a live node handling its requests, then error out as this
1378+
// should never happen
14091379
if isLive && liveNode == "" {
1410-
// Send a request to the AMQP backend to setup a node with the database
1411-
err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, com.SetToPrivate)
1412-
if err != nil {
1413-
jsonErr(w, err.Error(), http.StatusInternalServerError)
1414-
return
1415-
}
1380+
jsonErr(w, "No AMQP node available for request", http.StatusInternalServerError)
1381+
return
14161382
}
14171383

14181384
// If it's a standard database, process it locally. Else send the query to our AMQP backend

‎common/cypress.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,14 +78,14 @@ func CypressSeed(w http.ResponseWriter, r *http.Request) {
7878
defer liveDB1.Close()
7979

8080
// Store the live database in Minio
81-
err = LiveStoreDatabaseMinio(liveDB1, "default", "Join Testing with index.sqlite", 16384)
81+
objectID, err := LiveStoreDatabaseMinio(liveDB1, "default", "Join Testing with index.sqlite", 16384)
8282
if err != nil {
8383
http.Error(w, err.Error(), http.StatusInternalServerError)
8484
return
8585
}
8686

8787
// Send the live database file to our AMQP backend for setup
88-
err = LiveCreateDB(AmqpChan, "default", "Join Testing with index.sqlite", SetToPrivate)
88+
err = LiveCreateDB(AmqpChan, "default", "Join Testing with index.sqlite", objectID, SetToPrivate)
8989
if err != nil {
9090
log.Println(err)
9191
http.Error(w, err.Error(), http.StatusInternalServerError)

‎common/live.go

Lines changed: 92 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,33 @@ func ConnectMQ() (channel *amqp.Channel, err error) {
7878
return
7979
}
8080

81+
// LiveBackup asks the AMQP backend to store the given database back into Minio
82+
func LiveBackup(liveNode, loggedInUser, dbOwner, dbName string) (err error) {
83+
var rawResponse []byte
84+
rawResponse, err = MQRequest(AmqpChan, liveNode, "backup", loggedInUser, dbOwner, dbName, "")
85+
if err != nil {
86+
return
87+
}
88+
89+
// Decode the response
90+
var resp LiveDBErrorResponse
91+
err = json.Unmarshal(rawResponse, &resp)
92+
if err != nil {
93+
return
94+
}
95+
96+
// If the backup failed, then provide the error message to the user
97+
if resp.Error != "" {
98+
err = errors.New(resp.Error)
99+
return
100+
}
101+
if resp.Node == "" {
102+
log.Println("A node responded to a 'backup' request, but didn't identify itself.")
103+
return
104+
}
105+
return
106+
}
107+
81108
// LiveColumns requests the AMQP backend to return a list of all columns of the given table
82109
func LiveColumns(liveNode, loggedInUser, dbOwner, dbName, table string) (columns []sqlite.Column, pk []string, err error) {
83110
var rawResponse []byte
@@ -97,7 +124,7 @@ func LiveColumns(liveNode, loggedInUser, dbOwner, dbName, table string) (columns
97124
return
98125
}
99126
if resp.Node == "" {
100-
log.Printf("A node responded to a 'columns' request, but didn't identify itself")
127+
log.Println("A node responded to a 'columns' request, but didn't identify itself.")
101128
return
102129
}
103130
columns = resp.Columns
@@ -106,10 +133,10 @@ func LiveColumns(liveNode, loggedInUser, dbOwner, dbName, table string) (columns
106133
}
107134

108135
// LiveCreateDB requests the AMQP backend create a new live SQLite database
109-
func LiveCreateDB(channel *amqp.Channel, dbOwner, dbName string, accessType SetAccessType) (err error) {
136+
func LiveCreateDB(channel *amqp.Channel, dbOwner, dbName, objectID string, accessType SetAccessType) (err error) {
110137
// Send the database setup request to our AMQP backend
111138
var rawResponse []byte
112-
rawResponse, err = MQRequest(channel, "create_queue", "createdb", "", dbOwner, dbName, "")
139+
rawResponse, err = MQRequest(channel, "create_queue", "createdb", "", dbOwner, dbName, objectID)
113140
if err != nil {
114141
return
115142
}
@@ -126,7 +153,7 @@ func LiveCreateDB(channel *amqp.Channel, dbOwner, dbName string, accessType SetA
126153
return
127154
}
128155
if resp.Node == "" {
129-
err = errors.New("A node responded, but didn't identify itself. :(")
156+
log.Println("A node responded to a 'create' request, but didn't identify itself.")
130157
return
131158
}
132159
if resp.Result != "success" {
@@ -136,7 +163,7 @@ func LiveCreateDB(channel *amqp.Channel, dbOwner, dbName string, accessType SetA
136163
}
137164

138165
// Update PG, so it has a record of this database existing and knows the node/queue name for querying it
139-
err = LiveAddDatabasePG(dbOwner, dbName, resp.Node, accessType)
166+
err = LiveAddDatabasePG(dbOwner, dbName, objectID, resp.Node, accessType)
140167
if err != nil {
141168
return
142169
}
@@ -146,6 +173,34 @@ func LiveCreateDB(channel *amqp.Channel, dbOwner, dbName string, accessType SetA
146173
return
147174
}
148175

176+
// LiveDelete asks our AMQP backend to delete a database
177+
func LiveDelete(liveNode, loggedInUser, dbOwner, dbName string) (err error) {
178+
// Delete the database from our AMQP backend
179+
var rawResponse []byte
180+
rawResponse, err = MQRequest(AmqpChan, liveNode, "delete", loggedInUser, dbOwner, dbName, "")
181+
if err != nil {
182+
log.Println(err)
183+
return
184+
}
185+
186+
// Decode the response
187+
var resp LiveDBErrorResponse
188+
err = json.Unmarshal(rawResponse, &resp)
189+
if err != nil {
190+
log.Println(err)
191+
return
192+
}
193+
if resp.Error != "" {
194+
err = errors.New(resp.Error)
195+
return
196+
}
197+
if resp.Node == "" {
198+
log.Println("A node responded to a 'delete' request, but didn't identify itself.")
199+
return
200+
}
201+
return
202+
}
203+
149204
// LiveExecute asks our AMQP backend to execute a SQL statement on a database
150205
func LiveExecute(liveNode, loggedInUser, dbOwner, dbName, sql string) (rowsChanged int, err error) {
151206
var rawResponse []byte
@@ -195,6 +250,35 @@ func LiveQueryDB(channel *amqp.Channel, nodeName, requestingUser, dbOwner, dbNam
195250
return
196251
}
197252

253+
// LiveRowData asks our AMQP backend to send us the SQLite table data for a given range of rows
254+
func LiveRowData(liveNode, loggedInUser, dbOwner, dbName string, reqData LiveDBRowsRequest) (rowData SQLiteRecordSet, err error) {
255+
var rawResponse []byte
256+
rawResponse, err = MQRequest(AmqpChan, liveNode, "rowdata", loggedInUser, dbOwner, dbName, reqData)
257+
if err != nil {
258+
log.Println(err)
259+
return
260+
}
261+
262+
// Decode the response
263+
var resp LiveDBRowsResponse
264+
err = json.Unmarshal(rawResponse, &resp)
265+
if err != nil {
266+
log.Println(err)
267+
return
268+
}
269+
if resp.Error != "" {
270+
err = errors.New(resp.Error)
271+
log.Println(err)
272+
return
273+
}
274+
if resp.Node == "" {
275+
log.Println("A node responded to a 'rowdata' request, but didn't identify itself.")
276+
return
277+
}
278+
rowData = resp.RowData
279+
return
280+
}
281+
198282
// LiveSize asks our AMQP backend for the file size of a database
199283
func LiveSize(liveNode, loggedInUser, dbOwner, dbName string) (size int64, err error) {
200284
// Send the size request to our AMQP backend
@@ -215,7 +299,7 @@ func LiveSize(liveNode, loggedInUser, dbOwner, dbName string) (size int64, err e
215299
return
216300
}
217301
if resp.Node == "" {
218-
log.Printf("A node responded to a 'size' request, but didn't identify itself")
302+
log.Println("A node responded to a 'size' request, but didn't identify itself.")
219303
return
220304
}
221305
size = resp.Size
@@ -242,7 +326,7 @@ func LiveTables(liveNode, loggedInUser, dbOwner, dbName string) (tables []string
242326
return
243327
}
244328
if resp.Node == "" {
245-
log.Printf("A node responded to a 'tables' request, but didn't identify itself")
329+
log.Println("A node responded to a 'tables' request, but didn't identify itself.")
246330
return
247331
}
248332
tables = resp.Tables
@@ -289,7 +373,7 @@ func LiveViews(liveNode, loggedInUser, dbOwner, dbName string) (views []string,
289373
return
290374
}
291375
if resp.Node == "" {
292-
log.Printf("A node responded to a 'views' request, but didn't identify itself")
376+
log.Println("A node responded to a 'views' request, but didn't identify itself.")
293377
return
294378
}
295379
views = resp.Views

0 commit comments

Comments
 (0)