common: Include data diff when changing a table schema

When detecting a table creation, drop, or altering include all the table
rows in the diff. Note that this still does not include modified rows of
tables with an unchanged schema.

Author: MKleusberg

common/diff.go

@@ -22,14 +22,22 @@ type SchemaDiff struct {
 	Sql        string   `json:"sql"`
 }
 
+type DataDiff struct {
+	ActionType DiffType    `json:"action_type"`
+	Sql        string      `json:"sql"`
+	Pk         []DataValue `json:"pk"`
+}
+
 type DiffObjectChangeset struct {
 	ObjectName string         `json:"object_name"`
 	ObjectType string         `json:"object_type"`
 	Schema  SchemaDiff        `json:"schema"`
+	Data    []DataDiff        `json:"data"`
 }
 
 type Diffs struct {
 	Diff []DiffObjectChangeset  `json:"diff"`
+	// TODO Add PRAGMAs here
 }
 
 // Diff generates the differences between the two commits commitA and commitB of the two databases specified in the other parameters
@@ -128,6 +136,8 @@ func dbDiff(dbA string, dbB string) (Diffs, error) {
 		return Diffs{}, err
 	}
 
+	// TODO Check for differences in the PRAGMAs of both databases
+
 	// Return
 	return diff, nil
 }
@@ -144,19 +154,28 @@ func diffSingleObject(sdb *sqlite.Conn, objectName string, objectType string) (b
 	var sqlInMain, sqlInAux string
 	err := sdb.OneValue("SELECT sql FROM main.sqlite_master WHERE name = ? AND type = ?", &sqlInMain, objectName, objectType)
 	if err != nil && err != io.EOF {	// io.EOF is okay. It is returned when the object does not exist in the main database
-		return false, diff, err
+		return false, DiffObjectChangeset{}, err
 	}
 	err = sdb.OneValue("SELECT sql FROM aux.sqlite_master WHERE name = ? AND type = ?", &sqlInAux, objectName, objectType)
 	if err != nil && err != io.EOF {	// io.EOF is okay. It is returned when the object does not exist in the aux database
-		return false, diff, err
+		return false, DiffObjectChangeset{}, err
 	}
 
 	// Check for dropped object
 	if sqlInMain != "" && sqlInAux == "" {
 		diff.Schema.ActionType = ACTION_DELETE
 		diff.Schema.Sql = "DROP " + strings.ToUpper(objectType) + " "  + EscapeId(objectName) + ";"
 
-		// No data changes for added objects so we can return here
+		// If this is a table, also add all the deleted data to the diff
+		if objectType == "table" {
+			// We never include the SQL statements because there is no need to delete all the rows when we DROP the table anyway
+			diff.Data, err = dataDiffForAllTableRows(sdb, "main", objectName, ACTION_DELETE, false)
+			if err != nil {
+				return false, DiffObjectChangeset{}, err
+			}
+		}
+
+		// No further changes for dropped objects. So we can return here
 		return true, diff, nil 
 	}
 
@@ -165,8 +184,15 @@ func diffSingleObject(sdb *sqlite.Conn, objectName string, objectType string) (b
 		diff.Schema.ActionType = ACTION_ADD
 		diff.Schema.Sql = sqlInAux + ";"
 
-		// TODO If this is a table, also add all the data to the diff
+		// If this is a table, also add all the added data to the diff
+		if objectType == "table" {
+			diff.Data, err = dataDiffForAllTableRows(sdb, "aux", objectName, ACTION_ADD, true)
+			if err != nil {
+				return false, DiffObjectChangeset{}, err
+			}
+		}
 
+		// No further changes for created objects. So we can return here
 		return true, diff, nil
 	}
 
@@ -177,13 +203,111 @@ func diffSingleObject(sdb *sqlite.Conn, objectName string, objectType string) (b
 
 		// TODO If this is a table, be more clever and try to get away with ALTER TABLE instead of DROP and CREATE
 
-		// TODO If this is a table, also add all the data to the diff
+		// If this is a table, also add all the data to the diff
+		if objectType == "table" {
+			delete_data, err := dataDiffForAllTableRows(sdb, "main", objectName, ACTION_DELETE, false)
+			if err != nil {
+				return false, DiffObjectChangeset{}, err
+			}
+			add_data, err := dataDiffForAllTableRows(sdb, "aux", objectName, ACTION_ADD, true)
+			if err != nil {
+				return false, DiffObjectChangeset{}, err
+			}
+			diff.Data = append(delete_data, add_data...)
+		}
 
+		// No further changes for modified objects. So we can return here
 		return true, diff, nil
 	}
 
-	// TODO If this is a table, check for modified data
+	// If this is a table, check for modified data
+	if objectType == "table" {
+		// TODO
+	}
 
 	// Nothing has changed
 	return false, diff, nil
 }
+
+func dataDiffForAllTableRows(sdb *sqlite.Conn, schemaName string, tableName string, action DiffType, includeSql bool) (diff []DataDiff, err error) {
+	// Retrieve a list of all primary key columns in this table
+	pk, err := GetPrimaryKeyColumns(sdb, schemaName, tableName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Escape all the column names
+	var pk_escaped []string
+	for _, v := range pk {
+		pk_escaped = append(pk_escaped, EscapeId(v))
+	}
+
+	// Prepare query for the primary keys of all rows in this table. Only include the rest of the data
+	// in the rows if required
+	query := "SELECT " + strings.Join(pk_escaped, ",")
+	if includeSql && action == ACTION_ADD {
+		query += ", *"
+	}
+	query += " FROM " + EscapeId(schemaName) + "." + EscapeId(tableName)
+
+	// Retrieve data and add it to the data diff object
+	_, _, data, err := SQLiteRunQuery(sdb, Internal, query, false, false)
+	if err != nil {
+		log.Printf("Error getting rows in dataDiffForAllTableRows(): %s\n", err)
+		return nil, err
+	}
+	for _, row := range data.Records {
+		var d DataDiff
+		d.ActionType = action
+
+		// Prepare SQL statement when needed
+		if includeSql {
+			if action == ACTION_DELETE {
+				d.Sql = "DELETE FROM " + EscapeId(tableName) + " WHERE "
+			} else if action == ACTION_ADD {
+				d.Sql = "INSERT INTO " + EscapeId(tableName) + " VALUES("
+			}
+		}
+
+		// Get primary key data
+		for i := 0; i < data.ColCount; i++ {
+			// If this column is still part of the primary key, add it to the data diff
+			if i < len(pk) {
+				d.Pk = append(d.Pk, row[i])
+			}
+
+			// If we want to include a SQL statement for deleting data and this is still
+			// part of the primary key, add this to the prepared DELETE statement
+			if includeSql && action == ACTION_DELETE && i < len(pk) {
+				d.Sql += pk_escaped[i];
+				if row[i].Type == Null {
+					d.Sql += " IS NULL"
+				} else {
+					d.Sql += "=" + EscapeValue(row[i])
+				}
+				d.Sql += " AND "
+			}
+
+			// If we want to include a SQL statement for adding data and this is the regular
+			// data part, add this to the prepared INSERT statement
+			if includeSql && action == ACTION_ADD && i >= len(pk) {
+				d.Sql += EscapeValue(row[i]) + ","
+			}
+		}
+
+		// Remove the last " AND " of the SQL query for DELETE statements and the last "," for INSERT statements
+		// and add a semicolon instead
+		if includeSql {
+			if action == ACTION_DELETE {
+				d.Sql = strings.TrimSuffix(d.Sql, " AND ") + ";"
+			} else if action == ACTION_ADD {
+				d.Sql = strings.TrimSuffix(d.Sql, ",") + ");"
+			}
+		}
+
+		// Add row to data diff set
+		diff = append(diff, d)
+	}
+
+	return diff, nil
+}

common/sqlite.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net/http"
 	"reflect"
+	"sort"
 	"strconv"
 	"time"
 
@@ -766,6 +767,9 @@ func SQLiteRunQuery(sdb *sqlite.Conn, querySource QuerySource, dbQuery string, i
 						case API:
 							row = append(row, DataValue{Name: dataRows.ColNames[i], Type: Binary,
 								Value: base64.StdEncoding.EncodeToString(b)})
+						case Internal:
+							row = append(row, DataValue{Name: dataRows.ColNames[i], Type: Binary,
+								Value: b})
 						default:
 							row = append(row, DataValue{Name: dataRows.ColNames[i], Type: Binary,
 								Value: "<i>BINARY DATA</i>"})
@@ -782,7 +786,7 @@ func SQLiteRunQuery(sdb *sqlite.Conn, querySource QuerySource, dbQuery string, i
 			if isNull && !ignoreNull {
 				// Different sources of the query have different requirements for the output
 				switch querySource {
-				case API:
+				case API, Internal:
 					row = append(row, DataValue{Name: dataRows.ColNames[i], Type: Null})
 				default:
 					row = append(row, DataValue{Name: dataRows.ColNames[i], Type: Null, Value: "<i>NULL</i>"})
@@ -942,6 +946,87 @@ func Views(sdb *sqlite.Conn) (vw []string, err error) {
 	return
 }
 
+// Escape an identifier for safe use in SQL queries
 func EscapeId(id string) string {
 	return sqlite.Mprintf("\"%w\"", id)
 }
+
+// Format and escape a string value for use in SQL queries
+func EscapeValue(val DataValue) string {
+	if val.Type == Null {
+		return "NULL"
+	} else if val.Type == Integer || val.Type == Float {
+		return val.Value.(string)
+	} else {
+		return sqlite.Mprintf("%Q", val.Value.(string))
+	}
+}
+
+// Figure out the primary key columns of a table.
+// The schema and table parameters specify the schema and table names to use.
+func GetPrimaryKeyColumns(sdb *sqlite.Conn, schema string, table string) (pks []string, err error) {
+	// Prepare query
+	var stmt *sqlite.Stmt
+	stmt, err = sdb.Prepare("PRAGMA " + EscapeId(schema) + ".table_info(" + EscapeId(table) + ")")
+	if err != nil {
+		log.Printf("Error when preparing statement in GetPrimaryKey(): %s\n", err)
+		return nil, err
+	}
+	defer stmt.Finalize()
+
+	// Execute query and retrieve all primary key columns
+	primaryKeyColumns := make(map[int]string)
+	var hasColumnRowid, hasColumn_Rowid_, hasColumnOid bool
+	err = stmt.Select(func(s *sqlite.Stmt) error {
+		columnName, _ := s.ScanText(1)
+		pkOrder, _, _ := s.ScanInt(5)
+		if pkOrder > 0 {
+			primaryKeyColumns[pkOrder] = columnName
+		}
+
+		// While here check if there are any columns called rowid or similar in this table
+		if columnName == "rowid" {
+			hasColumnRowid = true
+		} else if columnName == "_rowid_" {
+			hasColumn_Rowid_ = true
+		} else if columnName == "oid" {
+			hasColumnOid = true
+		}
+		return nil
+	})
+	if err != nil {
+		log.Printf("Error when retrieving rows in GetPrimaryKey(): %s\n", err)
+		return nil, err
+	}
+
+	// Did we get any primary key columns? If not, this table has only an implicit primary key which
+	// is accessible by the name rowid, _rowid_, or oid
+	if len(primaryKeyColumns) > 0 {
+		// Explicit primary key
+
+		// Sort the columns by their order in the PK
+		keys := make([]int, 0, len(primaryKeyColumns))
+		for k := range primaryKeyColumns {
+			keys = append(keys, k)
+		}
+		sort.Ints(keys)
+
+		// Return columns in order
+		for _, k := range keys {
+			pks = append(pks, primaryKeyColumns[k])
+		}
+	} else {
+		// Implicit primary key
+		if !hasColumnRowid {
+			pks = append(pks, "rowid")
+		} else if !hasColumn_Rowid_ {
+			pks = append(pks, "_rowid_")
+		} else if !hasColumnOid {
+			pks = append(pks, "oid")
+		} else {
+			log.Printf("Unreachable rowid column in GetPrimaryKey()\n")
+			return nil, errors.New("Unreachable rowid column")
+		}
+	}
+	return pks, nil
+}

common/types.go

@@ -64,6 +64,7 @@ const (
 	DB4S
 	Visualisation
 	API
+	Internal
 )
 
 // ************************