Skip to content
This repository was archived by the owner on Mar 4, 2025. It is now read-only.

Commit a798e62

Browse files
justincliftjustinclift
committed
api, common, live: Add a new Execute() function to the API server
It's for executing SQLite statements (eg INSERT, UPDATE, DELETE) on live databases.
1 parent da5c98d commit a798e62

File tree

6 files changed

+269
-6
lines changed

6 files changed

+269
-6
lines changed

‎api/handlers.go

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,106 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) {
615615
}
616616
}
617617

618+
// executeHandler executes a SQL query on a SQLite database. It's used for running SQL queries which don't
619+
// return a result set, like `INSERT`, `UPDATE`, `DELETE`, and so forth.
620+
// This can be run from the command line using curl, like this:
621+
// $ curl -kD headers.out -F apikey="YOUR_API_KEY_HERE" -F dbowner="justinclift" -F dbname="Join Testing.sqlite" \
622+
// -F sql="VVBEQVRFIHRhYmxlMSBTRVQgTmFtZSA9ICdUZXN0aW5nIDEnIFdIRVJFIGlkID0gMQ==" \
623+
// https://api.dbhub.io/v1/execute
624+
// * "apikey" is one of your API keys. These can be generated from your Settings page once logged in
625+
// * "dbowner" is the owner of the database
626+
// * "dbname" is the name of the database
627+
// * "sql" is the SQL query to execute, base64 encoded
628+
// NOTE that the above example (base64) encoded sql is: "UPDATE table1 SET Name = 'Testing 1' WHERE id = 1"
629+
func executeHandler(w http.ResponseWriter, r *http.Request) {
630+
loggedInUser, err := checkAuth(w, r)
631+
if err != nil {
632+
jsonErr(w, err.Error(), http.StatusUnauthorized)
633+
return
634+
}
635+
636+
// Extract the database owner name, database name, and (optional) commit ID for the database from the request
637+
dbOwner, dbName, _, err := com.GetFormODC(r)
638+
if err != nil {
639+
jsonErr(w, err.Error(), http.StatusInternalServerError)
640+
return
641+
}
642+
dbFolder := "/"
643+
644+
// Grab the incoming SQLite query
645+
rawInput := r.FormValue("sql")
646+
query, err := com.CheckUnicode(rawInput)
647+
if err != nil {
648+
jsonErr(w, err.Error(), http.StatusBadRequest)
649+
return
650+
}
651+
652+
// Check if the requested database exists
653+
exists, err := com.CheckDBPermissions(loggedInUser, dbOwner, dbFolder, dbName, false)
654+
if err != nil {
655+
jsonErr(w, err.Error(), http.StatusInternalServerError)
656+
return
657+
}
658+
if !exists {
659+
jsonErr(w, fmt.Sprintf("Database '%s%s%s' doesn't exist", dbOwner, dbFolder, dbName),
660+
http.StatusNotFound)
661+
return
662+
}
663+
664+
// Check if the database is a live database, and get the node/queue to send the request to
665+
isLive, liveNode, err := com.CheckDBLive(dbOwner, dbFolder, dbName)
666+
if err != nil {
667+
jsonErr(w, err.Error(), http.StatusInternalServerError)
668+
return
669+
}
670+
671+
// Reject attempts to run Exec() on non-live databases
672+
if !isLive {
673+
jsonErr(w, "Exec() only runs on Live databases, which this isn't.", http.StatusInternalServerError)
674+
return
675+
}
676+
677+
// If a live database has been uploaded but doesn't have a live node handling its requests, then create one
678+
if isLive && liveNode == "" {
679+
// Send a request to the AMQP backend to set up the database there, ready for querying
680+
err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName)
681+
if err != nil {
682+
log.Println(err) // FIXME: Debug output while developing
683+
jsonErr(w, err.Error(), http.StatusInternalServerError)
684+
return
685+
}
686+
}
687+
688+
// Send the query execution request to our AMQP backend
689+
var rawResponse []byte
690+
rawResponse, err = com.MQSendRequest(com.AmqpChan, liveNode, "exec", loggedInUser, dbOwner, dbName, query)
691+
if err != nil {
692+
return
693+
}
694+
695+
// Decode the response
696+
var resp com.LiveDBErrorResponse
697+
err = json.Unmarshal(rawResponse, &resp)
698+
if err != nil {
699+
log.Println(err)
700+
return
701+
}
702+
if resp.Error != "" {
703+
err = errors.New(resp.Error)
704+
return
705+
}
706+
707+
// Return a "success" message
708+
z := com.StatusResponseContainer{Status: "OK"}
709+
jsonData, err := json.MarshalIndent(z, "", " ")
710+
if err != nil {
711+
log.Printf("Error when JSON marshalling returned data in execHandler(): %v\n", err)
712+
jsonErr(w, err.Error(), http.StatusInternalServerError)
713+
return
714+
}
715+
fmt.Fprintf(w, string(jsonData))
716+
}
717+
618718
// indexesHandler returns the details of all indexes in a SQLite database
619719
// This can be run from the command line using curl, like this:
620720
// $ curl -F apikey="YOUR_API_KEY_HERE" -F dbowner="justinclift" -F dbname="Join Testing.sqlite" https://api.dbhub.io/v1/indexes

‎api/main.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ func main() {
126126
http.Handle("/v1/delete", gz.GzipHandler(handleWrapper(deleteHandler)))
127127
http.Handle("/v1/diff", gz.GzipHandler(handleWrapper(diffHandler)))
128128
http.Handle("/v1/download", gz.GzipHandler(handleWrapper(downloadHandler)))
129+
http.Handle("/v1/execute", gz.GzipHandler(handleWrapper(executeHandler)))
129130
http.Handle("/v1/indexes", gz.GzipHandler(handleWrapper(indexesHandler)))
130131
http.Handle("/v1/metadata", gz.GzipHandler(handleWrapper(metadataHandler)))
131132
http.Handle("/v1/query", gz.GzipHandler(handleWrapper(queryHandler)))

‎api/templates/root.html

Lines changed: 83 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@
5858
<div class="col-md-12"><a href="https://dbhub.io"><h1>DBHub API (in early development)</h1></a></div>
5959
</div>
6060
<div class="row">
61-
<div class="col-md-12 version">API version 0.2, last modified 2023-03-20</div>
61+
<div class="col-md-12 version">API version 0.2, last modified 2023-03-22</div>
6262
</div>
6363
<div class="panel panel-primary" id="note">
6464
<div class="panel-heading heading">Note</div>
@@ -98,9 +98,10 @@
9898
<li class="list-group-item"><a href="#delete" class="apiheading">Delete</a> - Deletes a database from the requesting users account</li>
9999
<li class="list-group-item"><a href="#diff" class="apiheading">Diff</a> - Generates a diff between two databases or two versions of a database</li>
100100
<li class="list-group-item"><a href="#download" class="apiheading">Download</a> - Returns the requested SQLite database file</li>
101+
<li class="list-group-item" style="color: #a01e1a"><a href="#execute" class="apiheading" style="color: #a01e1a">Execute</a> - Executes a SQLite statement on a LIVE database <span style="font-style: italic">(new in version 0.2)</span> - <span style="font-weight: bold">EXPERIMENTAL ONLY</span></li>
101102
<li class="list-group-item"><a href="#indexes" class="apiheading">Indexes</a> - Returns the details of all indexes in a SQLite database</li>
102103
<li class="list-group-item"><a href="#metadata" class="apiheading">Metadata</a> - Returns the commit, branch, release, tag and web page information for a database</li>
103-
<li class="list-group-item"><a href="#query" class="apiheading">Query</a> - Run a SQLite query on a database</li>
104+
<li class="list-group-item"><a href="#query" class="apiheading">Query</a> - Runs a SQLite SELECT query on a database</li>
104105
<li class="list-group-item"><a href="#releases" class="apiheading">Releases</a> - Returns the details of all releases for a database</li>
105106
<li class="list-group-item"><a href="#tables" class="apiheading">Tables</a> - Returns the list of tables in a SQLite database</li>
106107
<li class="list-group-item"><a href="#tags" class="apiheading">Tags</a> - Returns the details of all tags for a database</li>
@@ -810,6 +811,85 @@
810811
</div>
811812
</div>
812813

814+
<!-- Execute -->
815+
<div class="panel panel-default" id="execute">
816+
<div class="panel-heading heading" style="color: #a01e1a;"><span style="font-size: x-large;">Execute</span> &nbsp; (new in version 0.2)</div>
817+
<div class="panel-body">
818+
<div class="row">
819+
<div class="col-md-11 heading" style="color: #a01e1a; padding-bottom: 10px;">This function is EXPERIMENTAL, and may change</div>
820+
<div class="col-md-1">&nbsp;</div>
821+
</div>
822+
<div class="row">
823+
<div class="col-md-2 heading">URL</div>
824+
<div class="col-md-10 heading">Description</div>
825+
</div>
826+
<div class="row indent">
827+
<div class="col-md-2"><a href="/v1/execute">/v1/execute</a></div>
828+
<div class="col-md-10">Executes a SQLite statement on a LIVE database. eg INSERT, UPDATE, DELETE</div>
829+
</div>
830+
<div class="row">
831+
<div class="col-md-2 heading pad">Parameters (POST)</div>
832+
<div class="col-md-10"></div>
833+
</div>
834+
<div class="row indent">
835+
<div class="col-md-2 paramname">apikey</div>
836+
<div class="col-md-10">Your API key. These can be generated in your <a href="https://[[ .ServerName ]]/pref">Settings</a> page, when logged in</div>
837+
</div>
838+
<div class="row indent">
839+
<div class="col-md-2 paramname">dbowner</div>
840+
<div class="col-md-10">The owner of the database</div>
841+
</div>
842+
<div class="row indent">
843+
<div class="col-md-2 paramname">dbname</div>
844+
<div class="col-md-10">The name of the database</div>
845+
</div>
846+
<div class="row indent">
847+
<div class="col-md-2 paramname">sql</div>
848+
<div class="col-md-10">The SQL query, <span style="font-weight: bold;">Base64</span> encoded</div>
849+
</div>
850+
<div class="row">
851+
<div class="col-md-2 heading pad">Return values (JSON)</div>
852+
<div class="col-md-10"></div>
853+
</div>
854+
<div class="row returndesc">
855+
<div class="col-md-12">
856+
On successful execution, the returned JSON will be a simple status name/value pair
857+
</div>
858+
</div>
859+
<div class="row indent returnhdr">
860+
<div class="col-md-4">Name</div>
861+
<div class="col-md-1">Type</div>
862+
<div class="col-md-7">Description</div>
863+
</div>
864+
<div class="row indent">
865+
<div class="col-md-4 paramname">status</div>
866+
<div class="col-md-1 type">string</div>
867+
<div class="col-md-7">Contains the string "OK" on successful execution</div>
868+
</div>
869+
<div class="row">
870+
<div class="col-md-12 heading">Example</div>
871+
</div>
872+
<div class="row indent">
873+
<div class="col-md-12">To execute this statement on <a href="https://dbhub.io/justinclift/Join%20Testing.sqlite">dbhub.io/justinclift/Join Testing.sqlite</a>:
874+
<pre>UPDATE table1
875+
SET Name = 'Testing 1'
876+
WHERE id = 1</pre>
877+
Using <a href="https://curl.haxx.se">curl</a>, it would be:
878+
<pre>$ curl -F apikey="YOUR_API_KEY_HERE" -F dbowner="justinclift" -F dbname="Join Testing.sqlite" \
879+
-F sql="VVBEQVRFIHRhYmxlMSBTRVQgTmFtZSA9ICdUZXN0aW5nIDEnIFdIRVJFIGlkID0gMQ==" \
880+
https://api.dbhub.io/v1/execute</pre>
881+
Output:
882+
<pre>{
883+
"status": "OK"
884+
}</pre>
885+
<span style="color: darkred; font-size: larger; font-weight: bolder;">Note</span> - This is a newly added EXPERIMENTAL function, that only works with
886+
LIVE databases. To try it out, <a href="https://dbhub.io/upload/">Upload</a> a new database and
887+
select the LIVE option under "Advanced".
888+
</div>
889+
</div>
890+
</div>
891+
</div>
892+
813893
<!-- Indexes -->
814894
<div class="panel panel-default" id="indexes">
815895
<div class="panel-heading heading" style="font-size: x-large; color: #2e6da4;">Indexes</div>
@@ -1077,7 +1157,7 @@
10771157
</div>
10781158
<div class="row indent">
10791159
<div class="col-md-2"><a href="/v1/query">/v1/query</a></div>
1080-
<div class="col-md-10">Run a SQLite query on a database</div>
1160+
<div class="col-md-10">Run a SQLite SELECT query on a database</div>
10811161
</div>
10821162
<div class="row">
10831163
<div class="col-md-2 heading pad">Parameters (POST)</div>

‎common/live.go

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ func MQCreateResponse(msg amqp.Delivery, channel *amqp.Channel, nodeName, result
274274
// It is probably only useful for returning errors that occur before we've decoded the incoming AMQP
275275
// request to know what type it is
276276
func MQErrorResponse(msg amqp.Delivery, channel *amqp.Channel, nodeName string, errMsg string) (err error) {
277-
// Construct the query response request
277+
// Construct the response
278278
resp := LiveDBErrorResponse{
279279
Node: nodeName,
280280
Error: errMsg,
@@ -307,7 +307,7 @@ func MQErrorResponse(msg amqp.Delivery, channel *amqp.Channel, nodeName string,
307307

308308
// MQDeleteResponse sends an error message in response to an AMQP database deletion request
309309
func MQDeleteResponse(msg amqp.Delivery, channel *amqp.Channel, nodeName string, errMsg string) (err error) {
310-
// Construct the query response request
310+
// Construct the response
311311
resp := LiveDBErrorResponse{ // Yep, we're reusing this super basic error response instead of creating a new one
312312
Node: nodeName,
313313
Error: errMsg,
@@ -338,6 +338,39 @@ func MQDeleteResponse(msg amqp.Delivery, channel *amqp.Channel, nodeName string,
338338
return
339339
}
340340

341+
// MQExecResponse sends a message in response to an AMQP database query exec request
342+
func MQExecResponse(msg amqp.Delivery, channel *amqp.Channel, nodeName string, errMsg string) (err error) {
343+
// Construct the response
344+
resp := LiveDBErrorResponse{ // Yep, we're reusing this super basic error response instead of creating a new one
345+
Node: nodeName,
346+
Error: errMsg,
347+
}
348+
var z []byte
349+
z, err = json.Marshal(resp)
350+
if err != nil {
351+
log.Println(err)
352+
return
353+
}
354+
355+
// Send the message
356+
ctx, cancel := context.WithTimeout(context.Background(), contextTimeout)
357+
defer cancel()
358+
err = channel.PublishWithContext(ctx, "", msg.ReplyTo, false, false,
359+
amqp.Publishing{
360+
ContentType: "text/json",
361+
CorrelationId: msg.CorrelationId,
362+
Body: z,
363+
})
364+
if err != nil {
365+
log.Println(err)
366+
}
367+
msg.Ack(false)
368+
if AmqpDebug {
369+
log.Printf("[EXEC] Live node '%s' responded with ACK to message with correlationID: '%s', msg.ReplyTo: '%s'", nodeName, msg.CorrelationId, msg.ReplyTo)
370+
}
371+
return
372+
}
373+
341374
// MQIndexesResponse sends an indexes list response
342375
func MQIndexesResponse(msg amqp.Delivery, channel *amqp.Channel, nodeName string, indexes []APIJSONIndex, errMsg string) (err error) {
343376
// Construct the response
@@ -374,7 +407,7 @@ func MQIndexesResponse(msg amqp.Delivery, channel *amqp.Channel, nodeName string
374407

375408
// MQQueryResponse sends a successful query response back
376409
func MQQueryResponse(msg amqp.Delivery, channel *amqp.Channel, nodeName string, results SQLiteRecordSet, errMsg string) (err error) {
377-
// Construct the query response request
410+
// Construct the response
378411
resp := LiveDBQueryResponse{
379412
Node: nodeName,
380413
Results: results,

‎common/sqlite.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -814,6 +814,40 @@ func ReadSQLiteDBCSV(sdb *sqlite.Conn, dbTable string) ([][]string, error) {
814814
return resultSet, nil
815815
}
816816

817+
// SQLiteExecQueryLive is used by our AMQP backend infrastructure to execute a user provided SQLite query
818+
func SQLiteExecQueryLive(baseDir, dbOwner, dbName, loggedInUser, query string) (err error) {
819+
// Open the Live database on the local node
820+
var sdb *sqlite.Conn
821+
sdb, err = OpenSQLiteDatabaseLive(baseDir, dbOwner, dbName)
822+
if err != nil {
823+
return
824+
}
825+
defer sdb.Close()
826+
827+
// TODO: Probably add in the before and after logging info at some point (as per query function),
828+
// so we can analyse query execution times, memory use, etc
829+
830+
// Prepare the statement
831+
// FIXME: Do we need to do this prepare step? Am suspecting "no", as we don't (yet) have a way to
832+
// batch call the matching stmt.Exec() below. Not remembering what else it might benefit though.
833+
var stmt *sqlite.Stmt
834+
stmt, err = sdb.Prepare(query)
835+
if err != nil {
836+
return
837+
}
838+
defer stmt.Finalize()
839+
840+
// Execute the statement
841+
err = stmt.Exec()
842+
if err != nil {
843+
log.Printf("Error when executing query by '%s' for LIVE database (%s/%s): '%s'\n",
844+
SanitiseLogString(loggedInUser), SanitiseLogString(dbOwner), SanitiseLogString(dbName),
845+
SanitiseLogString(err.Error()))
846+
return
847+
}
848+
return
849+
}
850+
817851
// SQLiteGetColumnsLive is used by our AMQP backend nodes to retrieve the list of columns from a SQLite database
818852
func SQLiteGetColumnsLive(baseDir, dbOwner, dbName, table string) (columns []sqlite.Column, err error) {
819853
// Open the database on the local node

‎live/main.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,21 @@ func main() {
182182
}
183183
continue
184184

185+
case "exec":
186+
// Execute a query on the database file
187+
err = com.SQLiteExecQueryLive(baseDir, req.DBOwner, req.DBName, req.RequestingUser, req.Query)
188+
if err != nil {
189+
err = com.MQExecResponse(msg, ch, com.NodeName, err.Error())
190+
continue
191+
}
192+
193+
// Return a success message (empty string in this case) to the caller
194+
err = com.MQExecResponse(msg, ch, com.NodeName, "")
195+
if err != nil {
196+
log.Printf("Error: occurred on '%s' in MQExecResponse() while constructing the AMQP execute query response: '%s'", com.NodeName, err)
197+
}
198+
continue
199+
185200
case "indexes":
186201
var indexes []com.APIJSONIndex
187202
indexes, err = com.SQLiteGetIndexesLive(baseDir, req.DBOwner, req.DBName)

0 commit comments

Comments
 (0)