common: Include changed table data in diff results

This adds diffing of table data and includes the changed rows in the
diff results. Note that while this tells you which rows have changed it
does not yet include the SQL statements required for merging the data.

This also makes the schema diff optional. If there are no changes to the
schema, the resulting schema diff will be nil.

Author: MKleusberg

common/diff.go

@@ -13,14 +13,14 @@ type DiffType string
 
 const (
 	ACTION_ADD    DiffType = "add"
-	ACTION_DELETE          = "delete"
-	ACTION_MODIFY          = "modify"
+	ACTION_DELETE DiffType = "delete"
+	ACTION_MODIFY DiffType = "modify"
 )
 
 type MergeStrategy int
 
 const (
-	NoMerge         MergeStrategy = iota
+	NoMerge MergeStrategy = iota
 	PreservePkMerge
 	NewPkMerge
 )
@@ -37,10 +37,10 @@ type DataDiff struct {
 }
 
 type DiffObjectChangeset struct {
-	ObjectName string     `json:"object_name"`
-	ObjectType string     `json:"object_type"`
-	Schema     SchemaDiff `json:"schema"`
-	Data       []DataDiff `json:"data"`
+	ObjectName string      `json:"object_name"`
+	ObjectType string      `json:"object_type"`
+	Schema     *SchemaDiff `json:"schema"`
+	Data       []DataDiff  `json:"data"`
 }
 
 type Diffs struct {
@@ -171,7 +171,7 @@ func diffSingleObject(sdb *sqlite.Conn, objectName string, objectType string, me
 
 	// Check for dropped object
 	if sqlInMain != "" && sqlInAux == "" {
-		diff.Schema.ActionType = ACTION_DELETE
+		diff.Schema = &SchemaDiff{ActionType: ACTION_DELETE}
 		if merge != NoMerge {
 			diff.Schema.Sql = "DROP " + strings.ToUpper(objectType) + " " + EscapeId(objectName) + ";"
 		}
@@ -191,7 +191,7 @@ func diffSingleObject(sdb *sqlite.Conn, objectName string, objectType string, me
 
 	// Check for added object
 	if sqlInMain == "" && sqlInAux != "" {
-		diff.Schema.ActionType = ACTION_ADD
+		diff.Schema = &SchemaDiff{ActionType: ACTION_ADD}
 		if merge != NoMerge {
 			diff.Schema.Sql = sqlInAux + ";"
 		}
@@ -210,7 +210,7 @@ func diffSingleObject(sdb *sqlite.Conn, objectName string, objectType string, me
 
 	// Check for modified object
 	if sqlInMain != "" && sqlInAux != "" && sqlInMain != sqlInAux {
-		diff.Schema.ActionType = ACTION_MODIFY
+		diff.Schema = &SchemaDiff{ActionType: ACTION_MODIFY}
 		if merge != NoMerge {
 			diff.Schema.Sql = "DROP " + strings.ToUpper(objectType) + " " + EscapeId(objectName) + ";" + sqlInAux + ";"
 		}
@@ -236,11 +236,22 @@ func diffSingleObject(sdb *sqlite.Conn, objectName string, objectType string, me
 
 	// If this is a table, check for modified data
 	if objectType == "table" {
-		// TODO
+		diff.Data, err = dataDiffForModifiedTableRows(sdb, objectName, merge)
+		if err != nil {
+			return false, DiffObjectChangeset{}, err
+		}
+
+		// When there are data changes, fill in the rest of the diff information and return the diff
+		if diff.Data != nil {
+			diff.ObjectName = objectName
+			diff.ObjectType = objectType
+
+			return true, diff, nil
+		}
 	}
 
 	// Nothing has changed
-	return false, diff, nil
+	return false, DiffObjectChangeset{}, nil
 }
 
 func dataDiffForAllTableRows(sdb *sqlite.Conn, schemaName string, tableName string, action DiffType, includeSql bool) (diff []DataDiff, err error) {
@@ -337,3 +348,125 @@ func dataDiffForAllTableRows(sdb *sqlite.Conn, schemaName string, tableName stri
 
 	return diff, nil
 }
+
+// This helper function gets the differences between the two tables named tableName in the main and in the aux schema.
+// It then builds an array of DataDiff objects which represents these differences. This function assumes that the table
+// schemas match.
+func dataDiffForModifiedTableRows(sdb *sqlite.Conn, tableName string, merge MergeStrategy) (diff []DataDiff, err error) {
+	// Retrieve a list of all primary key columns and other columns in this table
+	pk, _, other_columns, err := GetPrimaryKeyAndOtherColumns(sdb, "aux", tableName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Escape all column names
+	var pk_escaped, other_escaped []string
+	for _, v := range pk {
+		pk_escaped = append(pk_escaped, EscapeId(v))
+	}
+	for _, v := range other_columns {
+		other_escaped = append(other_escaped, EscapeId(v))
+	}
+
+	// Build query for getting differences. This is based on the query produced by the sqldiff utility for SQLite.
+	// The resulting query returns n+1+m*2 number of rows where n is the number of columns in the primary key and
+	// m is the number of columns in the table which are not part of the primary key. The extra column between the
+	// primary key columns and the other columns contains the diff type as specified by the DiffType constants.
+	// We generate two columns for each table column. The first item of each pair is 0 if the value of this column
+	// has not been modified and 1 if it was modified. The second item of each pair contains the new value.
+
+	var query string
+
+	// Updated rows
+	// There can only be updated rows in tables with more columns than the primary key columns
+	if len(other_columns) > 0 {
+		query = "SELECT "
+		for _, c := range pk_escaped { // Primary key columns first
+			query += "B." + c + ","
+		}
+		query += "'" + string(ACTION_MODIFY) + "'" // Updated row
+		for _, c := range other_escaped {          // Other columns last
+			query += ",A." + c + " IS NOT B." + c + ",B." + c
+		}
+
+		query += " FROM main." + EscapeId(tableName) + " A, aux." + EscapeId(tableName) + " B WHERE "
+
+		for _, c := range pk_escaped { // Where all primary key columns equal
+			query += "A." + c + "=B." + c + " AND "
+		}
+
+		query += "(" // And at least one of the other columns differs
+		for _, c := range other_escaped {
+			query += "A." + c + " IS NOT B." + c + " OR "
+		}
+		query = strings.TrimSuffix(query, " OR ") + ")"
+
+		query += " UNION ALL "
+	}
+
+	// Deleted rows
+	query += "SELECT "
+	for _, c := range pk_escaped { // Primary key columns first. This needs to be from the first table for deleted rows
+		query += "A." + c + ","
+	}
+	query += "'" + string(ACTION_DELETE) + "'"             // Deleted row
+	query += strings.Repeat(",NULL", len(other_escaped)*2) // Just NULL for all the other columns. They don't matter for deleted rows
+
+	query += " FROM main." + EscapeId(tableName) + " A WHERE "
+
+	query += "NOT EXISTS(SELECT 1 FROM aux." + EscapeId(tableName) + " B WHERE " // Where a row with the same primary key doesn't exist in the second table
+	for _, c := range pk_escaped {
+		query += "A." + c + " IS B." + c + " AND "
+	}
+	query = strings.TrimSuffix(query, " AND ") + ") UNION ALL "
+
+	// Inserted rows
+	query += "SELECT "
+	for _, c := range pk_escaped { // Primary key columns first. This needs to be from the second table for inserted rows
+		query += "B." + c + ","
+	}
+	query += "'" + string(ACTION_ADD) + "'" // Inserted row
+	for _, c := range other_escaped {       // Other columns last. Always set the modified flag for inserted rows
+		query += ",1,B." + c
+	}
+
+	query += " FROM aux." + EscapeId(tableName) + " B WHERE "
+
+	query += "NOT EXISTS(SELECT 1 FROM main." + EscapeId(tableName) + " A WHERE " // Where a row with the same primary key doesn't exist in the first table
+	for _, c := range pk_escaped {
+		query += "A." + c + " IS B." + c + " AND "
+	}
+	query = strings.TrimSuffix(query, " AND ") + ")"
+
+	// Finish query
+	query += " ORDER BY 1;" // Order by first primary key column
+
+	// Run the query and retrieve the data. Each row in the result set is a difference between the two tables.
+	// The column after the primary key columns and before the other columns specifies the type of change, i.e.
+	// update, insert or delete. While the primary key bit of the DataDiff object we create for each row can
+	// be taken directly from the first couple of columns and the action type of the DataDiff object can be
+	// deduced from the type column in a straightforward way, the generated SQL statements for merging highly
+	// depend on the diff type.
+
+	// Retrieve data and generate a new DataDiff object for each row
+	_, _, data, err := SQLiteRunQuery(sdb, Internal, query, false, false)
+	if err != nil {
+		log.Printf("Error getting rows in dataDiffForModifiedTableRows(): %s\n", err)
+		return nil, err
+	}
+	for _, row := range data.Records {
+		var d DataDiff
+
+		// Get the diff type
+		d.ActionType = DiffType(row[len(pk)].Value.(string))
+
+		// Fill in the primary key columns
+		for i := 0; i < len(pk); i++ {
+			d.Pk = append(d.Pk, row[i])
+		}
+
+		diff = append(diff, d)
+	}
+
+	return diff, nil
+}