webui: Save history of executed statements in the database

This saves the SQL statements executed via the execute page in a
database table. This means a user can see their last statements
including their results. This a) makes sure the state of the terminal
component is not lost when the page is refreshed or a link clicked and
b) allows users to continue their work on a different device or day.

The migrations for this are:
create table public.sql_terminal_history (history_id bigint not null, user_id bigint not null, db_id bigint not null, sql_stmt text, result jsonb, state text not null);
comment on table public.sql_terminal_history is 'This table holds the history of executed SQL statements for a user and database';
create sequence public.sql_terminal_history_history_id_seq start with 1 increment by 1 no minvalue no maxvalue cache 1;
alter sequence public.sql_terminal_history_history_id_seq owned by public.sql_terminal_history.history_id;
alter table only public.sql_terminal_history alter column history_id set default nextval('public.sql_terminal_history_history_id_seq'::regclass);
create index sql_terminal_history_user_id_db_id_index on public.sql_terminal_history using btree(user_id,db_id);
alter table only public.sql_terminal_history add constraint sql_terminal_history_users_user_id_fk foreign key (user_id) references public.users(user_id);
alter table only public.sql_terminal_history add constraint sql_terminal_history_sqlite_databases_db_id_fk foreign key (db_id) references public.sqlite_databases(db_id) on delete cascade;

Author: MKleusberg

common/postgresql.go

@@ -2666,6 +2666,112 @@ func LiveGetMinioNames(loggedInUser, dbOwner, dbName string) (bucketName, object
 	return
 }
 
+// LiveSqlHistoryAdd adds a new record to the history of recently executed SQL statements
+func LiveSqlHistoryAdd(loggedInUser, dbOwner, dbName, stmt string, state SqlHistoryItemStates, result interface{}) (err error) {
+	// Delete old records. We want to keep 100 records, so delete all but 99 and add one new in the next step
+	// TODO Make this number configurable or something
+	err = LiveSqlHistoryDeleteOld(loggedInUser, dbOwner, dbName, 99)
+	if err != nil {
+		return err
+	}
+
+	dbQuery := `
+		WITH u AS (
+			SELECT user_id
+			FROM users
+			WHERE lower(user_name) = lower($1)
+		), d AS (
+			SELECT db.db_id
+			FROM sqlite_databases AS db, u
+			WHERE db.user_id = u.user_id
+				AND db.db_name = $2
+		), l AS (
+			SELECT user_id
+			FROM users
+			WHERE lower(user_name) = lower($3)
+		)
+		INSERT INTO sql_terminal_history (user_id, db_id, sql_stmt, state, result)
+		VALUES ((SELECT user_id FROM l), (SELECT db_id FROM d), $4, $5,  $6)`
+	commandTag, err := pdb.Exec(dbQuery, dbOwner, dbName, loggedInUser, stmt, state, result)
+	if err != nil {
+		return err
+	}
+	if numRows := commandTag.RowsAffected(); numRows != 1 {
+		log.Printf("Wrong number of rows (%d) affected while saving SQL statement for user '%s'", numRows,
+			SanitiseLogString(loggedInUser))
+	}
+	return
+}
+
+// LiveSqlHistoryDeleteOld deletes all saved SQL statements in the SQL history table, except for the most recent ones
+func LiveSqlHistoryDeleteOld(loggedInUser, dbOwner, dbName string, keepRecords int) (err error) {
+	dbQuery := `
+		WITH u AS (
+			SELECT user_id
+			FROM users
+			WHERE lower(user_name) = lower($1)
+		), d AS (
+			SELECT db.db_id
+			FROM sqlite_databases AS db, u
+			WHERE db.user_id = u.user_id
+				AND db.db_name = $2
+		), l AS (
+			SELECT user_id
+			FROM users
+			WHERE lower(user_name) = lower($3)
+		)
+		DELETE FROM sql_terminal_history
+		WHERE history_id NOT IN (
+			SELECT h.history_id FROM sql_terminal_history h, u, d, l WHERE h.user_id=u.user_id AND h.db_id=d.db_id
+			ORDER BY h.history_id DESC LIMIT $4
+		)`
+	_, err = pdb.Exec(dbQuery, dbOwner, dbName, loggedInUser, keepRecords)
+	if err != nil {
+		return err
+	}
+	return
+}
+
+// LiveSqlHistoryGet returns the list of recently executed SQL statement for a user and database
+func LiveSqlHistoryGet(loggedInUser, dbOwner, dbName string) (history []SqlHistoryItem, err error) {
+	dbQuery := `
+		WITH u AS (
+			SELECT user_id
+			FROM users
+			WHERE lower(user_name) = lower($1)
+		), d AS (
+			SELECT db.db_id
+			FROM sqlite_databases AS db, u
+			WHERE db.user_id = u.user_id
+				AND db.db_name = $2
+		), l AS (
+			SELECT user_id
+			FROM users
+			WHERE lower(user_name) = lower($3)
+		)
+		SELECT h.sql_stmt, h.result, h.state
+		FROM sql_terminal_history h, l, d, u
+		WHERE h.user_id=l.user_id AND h.db_id=d.db_id
+		ORDER BY history_id ASC`
+	rows, err := pdb.Query(dbQuery, dbOwner, dbName, loggedInUser)
+	if err != nil {
+		log.Printf("Database query failed: %v", err)
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var item SqlHistoryItem
+		err = rows.Scan(&item.Statement, &item.Result, &item.State)
+		if err != nil {
+			return nil, err
+		}
+
+		history = append(history, item)
+	}
+	return
+}
+
 // LiveUserDBs returns the list of live databases owned by the user
 func LiveUserDBs(dbOwner string, public AccessType) (list []DBInfo, err error) {
 	dbQuery := `
@@ -3044,6 +3150,7 @@ func ResetDB() error {
 		"discussions",
 		"email_queue",
 		"events",
+		"sql_terminal_history",
 		"sqlite_databases",
 		"users",
 		"vis_params",
@@ -3063,6 +3170,7 @@ func ResetDB() error {
 		"discussions_disc_id_seq",
 		"email_queue_email_id_seq",
 		"events_event_id_seq",
+		"sql_terminal_history_history_id_seq",
 		"sqlite_databases_db_id_seq",
 		"users_user_id_seq",
 		"vis_query_runs_query_run_id_seq",

common/types.go

@@ -379,6 +379,20 @@ type ShareDatabasePermissionsUser struct {
 	Permission ShareDatabasePermissions `json:"permission"`
 }
 
+type SqlHistoryItemStates string
+
+const (
+	Executed SqlHistoryItemStates = "executed"
+	Queried  SqlHistoryItemStates = "queried"
+	Error    SqlHistoryItemStates = "error"
+)
+
+type SqlHistoryItem struct {
+	Statement string               `json:"input"`
+	Result    interface{}          `json:"output"`
+	State     SqlHistoryItemStates `json:"state"`
+}
+
 type SQLiteDBinfo struct {
 	Info     DBInfo
 	MaxRows  int

database/dbhub.sql

@@ -3,7 +3,7 @@
 --
 
 -- Dumped from database version 15.2
--- Dumped by pg_dump version 15.2 (Ubuntu 15.2-1.pgdg20.04+1)
+-- Dumped by pg_dump version 15.2
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -425,6 +425,46 @@ CREATE SEQUENCE public.events_event_id_seq
 ALTER SEQUENCE public.events_event_id_seq OWNED BY public.events.event_id;
 
 
+--
+-- Name: sql_terminal_history; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.sql_terminal_history (
+    history_id bigint NOT NULL,
+    user_id bigint NOT NULL,
+    db_id bigint NOT NULL,
+    sql_stmt text,
+    result jsonb,
+    state text NOT NULL
+);
+
+
+--
+-- Name: TABLE sql_terminal_history; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON TABLE public.sql_terminal_history IS 'This table holds the history of executed SQL statements for a user and database';
+
+
+--
+-- Name: sql_terminal_history_history_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.sql_terminal_history_history_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: sql_terminal_history_history_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.sql_terminal_history_history_id_seq OWNED BY public.sql_terminal_history.history_id;
+
+
 --
 -- Name: sqlite_databases; Type: TABLE; Schema: public; Owner: -
 --
@@ -669,6 +709,13 @@ ALTER TABLE ONLY public.email_queue ALTER COLUMN email_id SET DEFAULT nextval('p
 ALTER TABLE ONLY public.events ALTER COLUMN event_id SET DEFAULT nextval('public.events_event_id_seq'::regclass);
 
 
+--
+-- Name: sql_terminal_history history_id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.sql_terminal_history ALTER COLUMN history_id SET DEFAULT nextval('public.sql_terminal_history_history_id_seq'::regclass);
+
+
 --
 -- Name: sqlite_databases db_id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -956,6 +1003,13 @@ CREATE INDEX fki_discussion_comments_db_id_fkey ON public.discussion_comments US
 CREATE INDEX fki_discussions_source_db_id_fkey ON public.discussions USING btree (mr_source_db_id);
 
 
+--
+-- Name: sql_terminal_history_user_id_db_id_index; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX sql_terminal_history_user_id_db_id_index ON public.sql_terminal_history USING btree (user_id, db_id);
+
+
 --
 -- Name: users_lower_user_name_idx; Type: INDEX; Schema: public; Owner: -
 --
@@ -1135,6 +1189,22 @@ ALTER TABLE ONLY public.events
     ADD CONSTRAINT events_db_id_fkey FOREIGN KEY (db_id) REFERENCES public.sqlite_databases(db_id) ON UPDATE CASCADE ON DELETE CASCADE;
 
 
+--
+-- Name: sql_terminal_history sql_terminal_history_sqlite_databases_db_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.sql_terminal_history
+    ADD CONSTRAINT sql_terminal_history_sqlite_databases_db_id_fk FOREIGN KEY (db_id) REFERENCES public.sqlite_databases(db_id) ON DELETE CASCADE;
+
+
+--
+-- Name: sql_terminal_history sql_terminal_history_users_user_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.sql_terminal_history
+    ADD CONSTRAINT sql_terminal_history_users_user_id_fk FOREIGN KEY (user_id) REFERENCES public.users(user_id);
+
+
 --
 -- Name: sqlite_databases sqlite_databases_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
 --

webui/execute.go

@@ -15,6 +15,7 @@ func executePage(w http.ResponseWriter, r *http.Request) {
 	var pageData struct {
 		DB           com.SQLiteDBinfo
 		PageMeta     PageMetaInfo
+		SqlHistory   []com.SqlHistoryItem
 	}
 
 	// Get all meta information
@@ -67,6 +68,13 @@ func executePage(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Get SQL history
+	pageData.SqlHistory, err = com.LiveSqlHistoryGet(pageData.PageMeta.LoggedInUser, dbName.Owner, dbName.Database)
+	if err != nil {
+		errorPage(w, r, http.StatusInternalServerError, err.Error())
+		return
+	}
+
 	// Fill out various metadata fields
 	pageData.PageMeta.Title = fmt.Sprintf("Execute SQL - %s / %s", dbName.Owner, dbName.Database)
 	pageData.PageMeta.PageSection = "db_exec"
@@ -155,6 +163,18 @@ func execLiveSQL(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// In case of an error in the statement, save it in the terminal history as well.
+	// This should not be registered too early because we do not want to save it when
+	// there are validation or permission errors.
+	var logError = func(e error) {
+		// Store statement in sql terminal history
+		err = com.LiveSqlHistoryAdd(loggedInUser, dbOwner, dbName, sql, com.Error, map[string]interface{}{"error": e.Error()})
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			fmt.Fprint(w, err)
+		}
+	}
+
 	// Send the SQL execution request to our AMQP backend
 	var z interface{}
 	rowsChanged, err := com.LiveExecute(liveNode, loggedInUser, dbOwner, dbName, sql)
@@ -163,6 +183,7 @@ func execLiveSQL(w http.ResponseWriter, r *http.Request) {
 			log.Println(err)
 			w.WriteHeader(http.StatusInternalServerError)
 			fmt.Fprint(w, err)
+			logError(err)
 			return
 		}
 
@@ -171,11 +192,26 @@ func execLiveSQL(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			w.WriteHeader(http.StatusInternalServerError)
 			fmt.Fprint(w, err.Error())
+			logError(err)
 			return
 		}
+
+		// Store statement in sql terminal history
+		err = com.LiveSqlHistoryAdd(loggedInUser, dbOwner, dbName, sql, com.Queried, z)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			fmt.Fprint(w, err)
+		}
 	} else {
 		// The SQL statement execution succeeded, so pass along the # of rows changed
 		z = com.ExecuteResponseContainer{RowsChanged: rowsChanged, Status: "OK"}
+
+		// Store statement in sql terminal history
+		err = com.LiveSqlHistoryAdd(loggedInUser, dbOwner, dbName, sql, com.Executed, z)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			fmt.Fprint(w, err)
+		}
 	}
 
 	// Return the success message
@@ -184,8 +220,10 @@ func execLiveSQL(w http.ResponseWriter, r *http.Request) {
 		log.Println(err)
 		w.WriteHeader(http.StatusInternalServerError)
 		fmt.Fprint(w, err)
+		logError(err)
 		return
 	}
 	fmt.Fprintf(w, "%s", jsonData)
+
 	return
 }

webui/jsx/sql-terminal.js

@@ -40,7 +40,7 @@ function SqlTerminalCommandOutput({state, data}) {
 	if (state === "loading") {
 		output = <span className="text-muted">loading...</span>;
 	} else if (state === "error") {
-		output = <span className="text-danger"><strong>error: </strong>{data}</span>;
+		output = <span className="text-danger"><strong>error: </strong>{data.error}</span>;
 	} else if (state === "executed") {
 		output = <span className="text-info"><strong>done: </strong>{data.rows_changed + " row" + (data.rows_changed === 1 ? "" : "s") + " changed"}</span>;
 	} else if (state === "queried") {
@@ -102,7 +102,7 @@ function SqlTerminalCommand({command}) {
 			.catch(error => {
 				error.text().then(text => {
 					setState("error");
-					setOutput(text);
+					setOutput({error: text});
 				});
 			});
 		}
@@ -118,7 +118,7 @@ function SqlTerminalCommand({command}) {
 
 export default function SqlTerminal() {
 	const [code, setCode] = React.useState("");
-	const [recentCommands, setRecentCommands] = React.useState([]);
+	const [recentCommands, setRecentCommands] = React.useState(historyData === null ? [] : historyData);
 	const [executeOnEnter, setExecuteOnEnter] = React.useState(false);
 	const [isDragging, setDragging] = React.useState(false);		// We use this to distinguish simple clicks on the component from drag & drop movements. The latter are ignored to not interfere with selecting text for copy & paste.
 
@@ -217,7 +217,7 @@ export default function SqlTerminal() {
 							minHeight: "42px",
 						}}
 					/>
-					<div className="input-group-btn dropup">
+					<div className={recentCommands.length > 0 ? "input-group-btn dropup" : "input-group-btn"}>
 						<button type="button" className="btn btn-primary" disabled={code.trim() === "" ? "disabled" : null} onClick={() => execute()} data-cy="executebtn"><i className="fa fa-play" /> Execute</button>
 						<button type="button" className="btn btn-primary dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" data-cy="dropdownbtn"><span className="caret"></span></button>
 						<ul className="dropdown-menu dropdown-menu-right">

webui/templates/execute.html

@@ -9,6 +9,9 @@
     <div id="sql-terminal"></div>
 </div>
 [[ template "script_db_header" . ]]
+<script>
+    const historyData = [[ .SqlHistory ]];
+</script>
 [[ template "footer" . ]]
 </body>
 </html>