api, common. Log all API calls into our database, for stats generation

With this commit, we record basic info for each API call so we can
start doing useful statistic generation for API usage.

The backend database needs a new table created to hold the info:

  CREATE TABLE public.api_call_log (
      api_call_id bigint NOT NULL,
      api_call_date timestamp with time zone DEFAULT now(),
      caller_id bigint,
      db_owner_id bigint,
      db_id bigint,
      api_operation text NOT NULL,
      api_caller_sw text
  );

  COMMENT ON COLUMN public.api_call_log.db_owner_id IS 'This field must be nullable, as not all api calls act on a database';
  COMMENT ON COLUMN public.api_call_log.db_id IS 'This field must be nullable, as not all api calls act on a database';

  CREATE SEQUENCE public.api_log_log_id_seq
      START WITH 1
      INCREMENT BY 1
      NO MINVALUE
      NO MAXVALUE
      CACHE 1;

  ALTER SEQUENCE public.api_log_log_id_seq OWNED BY public.api_call_log.api_call_id;

  ALTER TABLE ONLY public.api_call_log ALTER COLUMN api_call_id SET DEFAULT nextval('public.api_log_log_id_seq'::regclass);

  ALTER TABLE ONLY public.api_call_log ADD CONSTRAINT api_log_users_user_id_fk FOREIGN KEY (caller_id) REFERENCES public.users(user_id);

Author: justinclift

api/handlers.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"sort"
 
 	sqlite "github.com/gwenn/gosqlite"
@@ -27,12 +28,15 @@ import (
 //	* "dbname" is the name of the database
 func branchesHandler(w http.ResponseWriter, r *http.Request) {
 	// Do auth check, grab request info
-	_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
+	loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
 	if err != nil {
 		jsonErr(w, err.Error(), httpStatus)
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "branches", r.Header.Get("User-Agent"))
+
 	// If the database is a live database, we return an error message
 	isLive, _, err := com.CheckDBLive(dbOwner, dbName)
 	if err != nil {
@@ -102,6 +106,9 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "columns", r.Header.Get("User-Agent"))
+
 	// Extract the table name
 	table, err := com.GetFormTable(r, false)
 	if err != nil {
@@ -221,12 +228,15 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) {
 //	* "dbname" is the name of the database
 func commitsHandler(w http.ResponseWriter, r *http.Request) {
 	// Do auth check, grab request info
-	_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
+	loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
 	if err != nil {
 		jsonErr(w, err.Error(), httpStatus)
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "commits", r.Header.Get("User-Agent"))
+
 	// If the database is a live database, we return an error message
 	isLive, _, err := com.CheckDBLive(dbOwner, dbName)
 	if err != nil {
@@ -279,6 +289,13 @@ func databasesHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Record the api call in our backend database
+	operation := "databases"
+	if live {
+		operation = "LIVE databases"
+	}
+	com.ApiCallLog(loggedInUser, "", "", operation, r.Header.Get("User-Agent"))
+
 	// Retrieve the list of databases in the user account
 	var databases []com.DBInfo
 	if !live {
@@ -336,6 +353,9 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
 	}
 	dbOwner := loggedInUser
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "delete", r.Header.Get("User-Agent"))
+
 	// Check if the database exists
 	exists, err := com.CheckDBPermissions(loggedInUser, dbOwner, dbName, false)
 	if err != nil {
@@ -517,6 +537,10 @@ func diffHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Record the api call in our backend database
+	// Note - Lets not bother adding additional api logging fields just for the diff function at this stage
+	com.ApiCallLog(loggedInUser, dbOwnerA, dbNameA, "diff", r.Header.Get("User-Agent"))
+
 	// Check permissions of the first database
 	var allowed bool
 	allowed, err = com.CheckDBPermissions(loggedInUser, dbOwnerA, dbNameA, false)
@@ -593,6 +617,9 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "download", r.Header.Get("User-Agent"))
+
 	// Return the requested database to the user
 	_, err = com.DownloadDatabase(w, r, dbOwner, dbName, commitID, loggedInUser, "api")
 	if err != nil {
@@ -630,6 +657,9 @@ func executeHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "execute", r.Header.Get("User-Agent"))
+
 	// Grab the incoming SQLite query
 	rawInput := r.FormValue("sql")
 	var sql string
@@ -709,6 +739,9 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "indexes", r.Header.Get("User-Agent"))
+
 	// Check if the database is a live database, and get the node/queue to send the request to
 	isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName)
 	if err != nil {
@@ -826,12 +859,15 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) {
 //	* "dbname" is the name of the database
 func metadataHandler(w http.ResponseWriter, r *http.Request) {
 	// Do auth check, grab request info
-	_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
+	loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
 	if err != nil {
 		jsonErr(w, err.Error(), httpStatus)
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "metadata", r.Header.Get("User-Agent"))
+
 	// If the database is a live database, we return an error message
 	isLive, _, err := com.CheckDBLive(dbOwner, dbName)
 	if err != nil {
@@ -886,6 +922,9 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "query", r.Header.Get("User-Agent"))
+
 	// Grab the incoming SQLite query
 	rawInput := r.FormValue("sql")
 	query, err := com.CheckUnicode(rawInput)
@@ -958,12 +997,15 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
 //	* "dbname" is the name of the database
 func releasesHandler(w http.ResponseWriter, r *http.Request) {
 	// Do auth check, grab request info
-	_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
+	loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
 	if err != nil {
 		jsonErr(w, err.Error(), httpStatus)
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "releases", r.Header.Get("User-Agent"))
+
 	// If the database is a live database, we return an error message
 	isLive, _, err := com.CheckDBLive(dbOwner, dbName)
 	if err != nil {
@@ -1033,6 +1075,9 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "tables", r.Header.Get("User-Agent"))
+
 	// Check if the database is a live database, and get the node/queue to send the request to
 	isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName)
 	if err != nil {
@@ -1108,12 +1153,15 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) {
 //	* "dbname" is the name of the database
 func tagsHandler(w http.ResponseWriter, r *http.Request) {
 	// Do auth check, grab request info
-	_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
+	loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
 	if err != nil {
 		jsonErr(w, err.Error(), httpStatus)
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "tags", r.Header.Get("User-Agent"))
+
 	// If the database is a live database, we return an error message
 	isLive, _, err := com.CheckDBLive(dbOwner, dbName)
 	if err != nil {
@@ -1314,18 +1362,39 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 			com.SanitiseLogString(dbOwner), com.SanitiseLogString(dbName), numBytes)
 
 		// Send a request to the AMQP backend to set up the database there, ready for querying
-		err = com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, objectID, com.SetToPrivate)
+		liveNode, err := com.LiveCreateDB(com.AmqpChan, dbOwner, dbName, objectID)
 		if err != nil {
 			log.Println(err)
 			jsonErr(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
 
+		// Update PG, so it has a record of this database existing and knows the node/queue name for querying it
+		err = com.LiveAddDatabasePG(dbOwner, dbName, objectID, liveNode, com.SetToPrivate)
+		if err != nil {
+			jsonErr(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// Enable the watch flag for the uploader for this database
+		err = com.ToggleDBWatch(dbOwner, dbOwner, dbName)
+		if err != nil {
+			jsonErr(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
 		// Upload was successful, so we construct a fake commit ID then return a success message to the user
 		x = make(map[string]string)
 		x["commit_id"] = ""
-		x["url"] = fmt.Sprintf("/%s", dbOwner)
+		x["url"] = server + filepath.Join("/", dbOwner, dbName)
+	}
+
+	// Record the api call in our backend database
+	operation := "upload"
+	if live {
+		operation = "LIVE upload"
 	}
+	com.ApiCallLog(loggedInUser, loggedInUser, dbName, operation, r.Header.Get("User-Agent"))
 
 	// Construct the response message
 	var ok bool
@@ -1366,6 +1435,9 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, loggedInUser, dbName, "views", r.Header.Get("User-Agent"))
+
 	// Check if the database is a live database, and get the node/queue to send the request to
 	isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName)
 	if err != nil {
@@ -1441,12 +1513,15 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) {
 //	* "dbname" is the name of the database being queried
 func webpageHandler(w http.ResponseWriter, r *http.Request) {
 	// Authenticate user and collect requested database details
-	_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
+	loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
 	if err != nil {
 		jsonErr(w, err.Error(), httpStatus)
 		return
 	}
 
+	// Record the api call in our backend database
+	com.ApiCallLog(loggedInUser, dbOwner, dbName, "webpage", r.Header.Get("User-Agent"))
+
 	// Return the database webUI URL to the user
 	var z com.WebpageResponseContainer
 	z.WebPage = "https://" + com.Conf.Web.ServerName + "/" + dbOwner + "/" + dbName

api/main.go

@@ -3,10 +3,6 @@ package main
 // TODO: API functions that still need updating for Live databases
 //         * diff - already updated to just return an error for live databases.  needs testing though
 
-// FIXME: Update the documented Upload() function return values on the API doc page.  Currently it talks about
-//        returning the commit ID for the upload.  We'll probably return that field with a blank value for live
-//        databases though.  TBD.
-
 // FIXME: After the API and webui pieces are done, figure out how the DB4S end
 //        point and dio should be updated to use live databases too
 
@@ -113,29 +109,29 @@ func main() {
 	}
 
 	// Our pages
-	http.Handle("/", gz.GzipHandler(handleWrapper(rootHandler)))
-	http.Handle("/changelog", gz.GzipHandler(handleWrapper(changeLogHandler)))
-	http.Handle("/changelog.html", gz.GzipHandler(handleWrapper(changeLogHandler)))
-	http.Handle("/v1/branches", gz.GzipHandler(handleWrapper(branchesHandler)))
-	http.Handle("/v1/columns", gz.GzipHandler(handleWrapper(columnsHandler)))
-	http.Handle("/v1/commits", gz.GzipHandler(handleWrapper(commitsHandler)))
-	http.Handle("/v1/databases", gz.GzipHandler(handleWrapper(databasesHandler)))
-	http.Handle("/v1/delete", gz.GzipHandler(handleWrapper(deleteHandler)))
-	http.Handle("/v1/diff", gz.GzipHandler(handleWrapper(diffHandler)))
-	http.Handle("/v1/download", gz.GzipHandler(handleWrapper(downloadHandler)))
-	http.Handle("/v1/execute", gz.GzipHandler(handleWrapper(executeHandler)))
-	http.Handle("/v1/indexes", gz.GzipHandler(handleWrapper(indexesHandler)))
-	http.Handle("/v1/metadata", gz.GzipHandler(handleWrapper(metadataHandler)))
-	http.Handle("/v1/query", gz.GzipHandler(handleWrapper(queryHandler)))
-	http.Handle("/v1/releases", gz.GzipHandler(handleWrapper(releasesHandler)))
-	http.Handle("/v1/tables", gz.GzipHandler(handleWrapper(tablesHandler)))
-	http.Handle("/v1/tags", gz.GzipHandler(handleWrapper(tagsHandler)))
-	http.Handle("/v1/upload", gz.GzipHandler(handleWrapper(uploadHandler)))
-	http.Handle("/v1/views", gz.GzipHandler(handleWrapper(viewsHandler)))
-	http.Handle("/v1/webpage", gz.GzipHandler(handleWrapper(webpageHandler)))
+	http.Handle("/", gz.GzipHandler(corsWrapper(rootHandler)))
+	http.Handle("/changelog", gz.GzipHandler(corsWrapper(changeLogHandler)))
+	http.Handle("/changelog.html", gz.GzipHandler(corsWrapper(changeLogHandler)))
+	http.Handle("/v1/branches", gz.GzipHandler(corsWrapper(branchesHandler)))
+	http.Handle("/v1/columns", gz.GzipHandler(corsWrapper(columnsHandler)))
+	http.Handle("/v1/commits", gz.GzipHandler(corsWrapper(commitsHandler)))
+	http.Handle("/v1/databases", gz.GzipHandler(corsWrapper(databasesHandler)))
+	http.Handle("/v1/delete", gz.GzipHandler(corsWrapper(deleteHandler)))
+	http.Handle("/v1/diff", gz.GzipHandler(corsWrapper(diffHandler)))
+	http.Handle("/v1/download", gz.GzipHandler(corsWrapper(downloadHandler)))
+	http.Handle("/v1/execute", gz.GzipHandler(corsWrapper(executeHandler)))
+	http.Handle("/v1/indexes", gz.GzipHandler(corsWrapper(indexesHandler)))
+	http.Handle("/v1/metadata", gz.GzipHandler(corsWrapper(metadataHandler)))
+	http.Handle("/v1/query", gz.GzipHandler(corsWrapper(queryHandler)))
+	http.Handle("/v1/releases", gz.GzipHandler(corsWrapper(releasesHandler)))
+	http.Handle("/v1/tables", gz.GzipHandler(corsWrapper(tablesHandler)))
+	http.Handle("/v1/tags", gz.GzipHandler(corsWrapper(tagsHandler)))
+	http.Handle("/v1/upload", gz.GzipHandler(corsWrapper(uploadHandler)))
+	http.Handle("/v1/views", gz.GzipHandler(corsWrapper(viewsHandler)))
+	http.Handle("/v1/webpage", gz.GzipHandler(corsWrapper(webpageHandler)))
 
 	// favicon.ico
-	http.Handle("/favicon.ico", gz.GzipHandler(handleWrapper(func(w http.ResponseWriter, r *http.Request) {
+	http.Handle("/favicon.ico", gz.GzipHandler(corsWrapper(func(w http.ResponseWriter, r *http.Request) {
 		logReq(r, "-")
 		http.ServeFile(w, r, filepath.Join(com.Conf.Web.BaseDir, "webui", "favicon.ico"))
 	})))
@@ -329,9 +325,8 @@ func extractUserFromClientCert(w http.ResponseWriter, r *http.Request) (userAcc
 	return
 }
 
-// handleWrapper does nothing useful except interface between types
-// TODO: Get rid of this, as it shouldn't be needed
-func handleWrapper(fn http.HandlerFunc) http.HandlerFunc {
+// corsWrapper sets a general allow for all our api calls
+func corsWrapper(fn http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// Enable CORS (https://enable-cors.org)
 		w.Header().Set("Access-Control-Allow-Origin", "*")
@@ -361,6 +356,5 @@ func jsonErr(w http.ResponseWriter, msg string, statusCode int) {
 // logReq writes an entry for the incoming request to the request log
 func logReq(r *http.Request, loggedInUser string) {
 	fmt.Fprintf(reqLog, "%v - %s [%s] \"%s %s %s\" \"-\" \"-\" \"%s\" \"%s\"\n", r.RemoteAddr,
-		loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto,
-		r.Referer(), r.Header.Get("User-Agent"))
+		loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto, r.Referer(), r.Header.Get("User-Agent"))
 }

common/cypress.go

@@ -85,13 +85,29 @@ func CypressSeed(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Send the live database file to our AMQP backend for setup
-	err = LiveCreateDB(AmqpChan, "default", "Join Testing with index.sqlite", objectID, SetToPrivate)
+	dbOwner := "default"
+	dbName := "Join Testing with index.sqlite"
+	liveNode, err := LiveCreateDB(AmqpChan, dbOwner, dbName, objectID)
 	if err != nil {
 		log.Println(err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
+	// Update PG, so it has a record of this database existing and knows the node/queue name for querying it
+	err = LiveAddDatabasePG(dbOwner, dbName, objectID, liveNode, SetToPrivate)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Enable the watch flag for the uploader for this database
+	err = ToggleDBWatch(dbOwner, dbOwner, dbName)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
 	// *** Add a test LIVE SQLite database (end) ***
 
 	// Add some test users

common/live.go

@@ -133,7 +133,7 @@ func LiveColumns(liveNode, loggedInUser, dbOwner, dbName, table string) (columns
 }
 
 // LiveCreateDB requests the AMQP backend create a new live SQLite database
-func LiveCreateDB(channel *amqp.Channel, dbOwner, dbName, objectID string, accessType SetAccessType) (err error) {
+func LiveCreateDB(channel *amqp.Channel, dbOwner, dbName, objectID string) (liveNode string, err error) {
 	// Send the database setup request to our AMQP backend
 	var rawResponse []byte
 	rawResponse, err = MQRequest(channel, "create_queue", "createdb", "", dbOwner, dbName, objectID)
@@ -161,15 +161,7 @@ func LiveCreateDB(channel *amqp.Channel, dbOwner, dbName, objectID string, acces
 			dbOwner, dbName))
 		return
 	}
-
-	// Update PG, so it has a record of this database existing and knows the node/queue name for querying it
-	err = LiveAddDatabasePG(dbOwner, dbName, objectID, resp.Node, accessType)
-	if err != nil {
-		return
-	}
-
-	// Enable the watch flag for the uploader for this database
-	err = ToggleDBWatch(dbOwner, dbOwner, dbName)
+	liveNode = resp.Node
 	return
 }