Generated Columns Support

- Added generated column type in Mysql Info schema.
- Serialized generated column expression.

Author: pratickchokhani

common/constants/constants.go

@@ -131,6 +131,8 @@ const (
 	CHECK_EXPRESSION   = "CHECK"
 	DEFAULT_EXPRESSION = "DEFAULT"
 	DEFAULT_GENERATED  = "DEFAULT_GENERATED"
+	STORED_GENERATED   = "STORED"
+	VIRTUAL_GENERATED  = "VIRTUAL"
 	TEMP_DB            = "smt-staging-db"
 	DB_URI             = "projects/%s/instances/%s/databases/%s"
 

expressions_api/expression_verify.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	mtrand "math/rand"
 	"sync"
 	"time"
@@ -129,6 +130,9 @@ func (ev *ExpressionVerificationAccessorImpl) verifyExpressionInternal(expressio
 		sqlStatement = fmt.Sprintf("SELECT 1 from %s where %s;", expressionDetail.ReferenceElement.Name, expressionDetail.Expression)
 	case constants.DEFAULT_EXPRESSION:
 		sqlStatement = fmt.Sprintf("SELECT CAST(%s as %s)", expressionDetail.Expression, expressionDetail.ReferenceElement.Name)
+	case constants.STORED_GENERATED, constants.VIRTUAL_GENERATED:
+		sqlStatement = fmt.Sprintf("SELECT %s as %s FROM %s", expressionDetail.Expression, expressionDetail.ReferenceElement.Name, expressionDetail.SpTableName)
+		logger.Log.Info(fmt.Sprintf("Expression Statemnet: %s", sqlStatement))
 	default:
 		return task.TaskResult[internal.ExpressionVerificationOutput]{Result: internal.ExpressionVerificationOutput{Result: false, Err: fmt.Errorf("invalid expression type requested")}, Err: nil}
 	}
@@ -171,6 +175,7 @@ func (ev *ExpressionVerificationAccessorImpl) removeExpressions(inputConv *inter
 		for colName, colDef := range table.ColDefs {
 			colDef.AutoGen = ddl.AutoGenCol{}
 			colDef.DefaultValue = ddl.DefaultValue{}
+			colDef.GeneratedColumn = ddl.GeneratedColumn{}
 			table.ColDefs[colName] = colDef
 		}
 	}
@@ -197,27 +202,64 @@ func (ddlv *DDLVerifierImpl) GetSourceExpressionDetails(conv *internal.Conv, tab
 		srcTable := conv.SrcSchema[tableId]
 		for _, srcColId := range srcTable.ColIds {
 			srcCol := srcTable.ColDefs[srcColId]
+			var expression ddl.Expression
+			var expressionType string
+			isExpressionAvailable := false
 			if srcCol.DefaultValue.IsPresent {
+				expression = srcCol.DefaultValue.Value
+				isExpressionAvailable = true
+				expressionType = constants.DEFAULT_EXPRESSION
+			} else if srcCol.GeneratedColumn.IsPresent {
+				expression = srcCol.GeneratedColumn.Value
+				isExpressionAvailable = true
+				if srcCol.GeneratedColumn.Type == ddl.GeneratedColStored {
+					expressionType = constants.STORED_GENERATED
+				} else {
+					expressionType = constants.VIRTUAL_GENERATED
+				}
+			}
+			if isExpressionAvailable {
 				tyName := conv.SpSchema[tableId].ColDefs[srcColId].T.Name
 				if conv.SpDialect == constants.DIALECT_POSTGRESQL {
 					tyName = ddl.GetPGType(conv.SpSchema[tableId].ColDefs[srcColId].T)
 				}
-				defaultValueExp := internal.ExpressionDetail{
+				expressionDetail := internal.ExpressionDetail{
 					ReferenceElement: internal.ReferenceElement{
 						Name: tyName,
 					},
-					ExpressionId: srcCol.DefaultValue.Value.ExpressionId,
-					Expression:   srcCol.DefaultValue.Value.Statement,
-					Type:         constants.DEFAULT_EXPRESSION,
+					ExpressionId: expression.ExpressionId,
+					Expression:   expression.Statement,
+					Type:         expressionType,
+					SpTableName:  conv.ToSpanner[srcTable.Name].Name,
 					Metadata:     map[string]string{"TableId": tableId, "ColId": srcColId},
 				}
-				expressionDetails = append(expressionDetails, defaultValueExp)
+				logger.Log.Info(fmt.Sprintf("AExpression Details: %v", expressionDetail))
+				expressionDetails = append(expressionDetails, expressionDetail)
 			}
 		}
 	}
 	return expressionDetails
 }
 
+func (ddlv *DDLVerifierImpl) getExpressionDetail(
+	conv *internal.Conv, tableId, spColId, expressionType, expressionId, expression string, spCol ddl.ColumnDef) internal.ExpressionDetail {
+	tyName := conv.SpSchema[tableId].ColDefs[spColId].T.Name
+	if conv.SpDialect == constants.DIALECT_POSTGRESQL {
+		tyName = ddl.GetPGType(conv.SpSchema[tableId].ColDefs[spColId].T)
+	}
+	expressionDetail := internal.ExpressionDetail{
+		ReferenceElement: internal.ReferenceElement{
+			Name: tyName,
+		},
+		ExpressionId: expressionId,
+		Expression:   expression,
+		Type:         expressionType,
+		Metadata:     map[string]string{"TableId": tableId, "ColId": spColId},
+	}
+	return expressionDetail
+
+}
+
 func (ddlv *DDLVerifierImpl) GetSpannerExpressionDetails(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail {
 	expressionDetails := []internal.ExpressionDetail{}
 	// Collect default values for verification
@@ -226,20 +268,15 @@ func (ddlv *DDLVerifierImpl) GetSpannerExpressionDetails(conv *internal.Conv, ta
 		for _, spColId := range spTable.ColIds {
 			spCol := spTable.ColDefs[spColId]
 			if spCol.DefaultValue.IsPresent {
-				tyName := conv.SpSchema[tableId].ColDefs[spColId].T.Name
-				if conv.SpDialect == constants.DIALECT_POSTGRESQL {
-					tyName = ddl.GetPGType(conv.SpSchema[tableId].ColDefs[spColId].T)
-				}
-				defaultValueExp := internal.ExpressionDetail{
-					ReferenceElement: internal.ReferenceElement{
-						Name: tyName,
-					},
-					ExpressionId: spCol.DefaultValue.Value.ExpressionId,
-					Expression:   spCol.DefaultValue.Value.Statement,
-					Type:         constants.DEFAULT_EXPRESSION,
-					Metadata:     map[string]string{"TableId": tableId, "ColId": spColId},
-				}
+				defaultValueExp := ddlv.getExpressionDetail(
+					conv, tableId, spColId, constants.DEFAULT_EXPRESSION, spCol.DefaultValue.Value.ExpressionId,
+					spCol.DefaultValue.Value.Statement, spCol)
 				expressionDetails = append(expressionDetails, defaultValueExp)
+			} else if spCol.GeneratedColumn.IsPresent {
+				generatedColExp := ddlv.getExpressionDetail(
+					conv, tableId, spColId, string(spCol.GeneratedColumn.Type), spCol.GeneratedColumn.Value.ExpressionId,
+					spCol.GeneratedColumn.Value.Statement, spCol)
+				expressionDetails = append(expressionDetails, generatedColExp)
 			}
 		}
 	}

internal/convert.go

@@ -158,6 +158,7 @@ const (
 	CassandraMAP
 	PossibleOverflow
 	IdentitySkipRange
+	GeneratedColumnValueError
 )
 
 const (
@@ -331,6 +332,7 @@ type ExpressionDetail struct {
 	ExpressionId     string
 	Expression       string
 	Type             string
+	SpTableName      string
 	Metadata         map[string]string
 }
 

internal/reports/report_helpers.go

@@ -306,7 +306,7 @@ func buildTableReportBody(conv *internal.Conv, tableId string, issues map[string
 				// on case of srcType.
 				spColType = strings.ToLower(spColType)
 				switch i {
-				case internal.DefaultValue:
+				case internal.DefaultValue, internal.GeneratedColumnValueError:
 					toAppend := Issue{
 						Category:    IssueDB[i].Category,
 						Description: fmt.Sprintf("%s for table '%s' e.g. column '%s'", IssueDB[i].Brief, conv.SpSchema[tableId].Name, spColName),
@@ -621,6 +621,7 @@ var IssueDB = map[internal.SchemaIssue]struct {
 	CategoryDescription string
 }{
 	internal.DefaultValue:                         {Brief: "Some columns have default values which Spanner migration tool does not migrate. Please add the default constraints manually after the migration is complete", Severity: note, batch: true, Category: "MISSING_DEFAULT_VALUE_CONSTRAINTS"},
+	internal.GeneratedColumnValueError:            {Brief: "Some columns have generated expression which Spanner migration tool does not migrate. Please add the expressions manually after the migration is complete", Severity: Errors, batch: true, Category: "MISSING_GENERATED_COL_VALUE_CONSTRAINTS"},
 	internal.TypeMismatch:                         {Brief: "Type mismatch in check constraint mention in table", Severity: warning, Category: "TYPE_MISMATCH"},
 	internal.TypeMismatchError:                    {Brief: "Type mismatch in check constraint mention in table", Severity: Errors, Category: "TYPE_MISMATCH_ERROR"},
 	internal.InvalidCondition:                     {Brief: "Invalid condition in check constraint mention in table", Severity: warning, Category: "INVALID_CONDITION"},

schema/schema.go

@@ -50,13 +50,14 @@ type Table struct {
 // Column represents a database column.
 // TODO: add support for foreign keys.
 type Column struct {
-	Name         string
-	Type         Type
-	NotNull      bool
-	Ignored      Ignored
-	Id           string
-	AutoGen      ddl.AutoGenCol
-	DefaultValue ddl.DefaultValue
+	Name            string
+	Type            Type
+	NotNull         bool
+	Ignored         Ignored
+	Id              string
+	AutoGen         ddl.AutoGenCol
+	DefaultValue    ddl.DefaultValue
+	GeneratedColumn ddl.GeneratedColumn
 }
 
 // ForeignKey represents a foreign key.
@@ -129,6 +130,7 @@ type Ignored struct {
 	Exclusion     bool
 	ForeignKey    bool
 	AutoIncrement bool
+	GeneratedCol  bool
 }
 
 // Print converts ty to a string suitable for printing.

sources/common/toddl.go

@@ -31,7 +31,9 @@ package common
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"reflect"
 	"strconv"
 	"strings"
@@ -377,6 +379,9 @@ func (ss *SchemaToSpannerImpl) SchemaToSpannerDDLHelper(conv *internal.Conv, tod
 		if srcCol.Ignored.Default {
 			issues = append(issues, internal.DefaultValue)
 		}
+		if srcCol.Ignored.GeneratedCol {
+			issues = append(issues, internal.GeneratedColumnValueError)
+		}
 		if srcCol.Ignored.AutoIncrement { // TODO(adibh) - check why this is not there in postgres
 			issues = append(issues, internal.AutoIncrement)
 		}
@@ -673,6 +678,9 @@ func CvtIndexHelper(conv *internal.Conv, tableId string, srcIndex schema.Index,
 
 // Applies all valid expressions which can be migrated to spanner conv object
 func spannerSchemaApplyExpressions(conv *internal.Conv, expressions internal.VerifyExpressionsOutput) {
+	// Log expression verication output list
+	expression_str, _ := json.Marshal(expressions.ExpressionVerificationOutputList)
+	logger.Log.Info(fmt.Sprintf("Expressions: %s", string(expression_str)))
 	for _, expression := range expressions.ExpressionVerificationOutputList {
 		switch expression.ExpressionDetail.Type {
 		case "DEFAULT":
@@ -696,6 +704,29 @@ func spannerSchemaApplyExpressions(conv *internal.Conv, expressions internal.Ver
 					conv.SchemaIssues[tableId].ColumnLevelIssues[columnId] = colIssues
 				}
 			}
+		case constants.VIRTUAL_GENERATED, constants.STORED_GENERATED:
+			{
+				tableId := expression.ExpressionDetail.Metadata["TableId"]
+				columnId := expression.ExpressionDetail.Metadata["ColId"]
+
+				if expression.Result {
+					col := conv.SpSchema[tableId].ColDefs[columnId]
+					col.GeneratedColumn = ddl.GeneratedColumn{
+						IsPresent: true,
+						Value: ddl.Expression{
+							ExpressionId: expression.ExpressionDetail.ExpressionId,
+							Statement:    expression.ExpressionDetail.Expression,
+						},
+						Type: ddl.GeneratedColType(expression.ExpressionDetail.Type),
+					}
+					conv.SpSchema[tableId].ColDefs[columnId] = col
+				} else {
+					colIssues := conv.SchemaIssues[tableId].ColumnLevelIssues[columnId]
+					colIssues = append(colIssues, internal.GeneratedColumnValueError)
+					conv.SchemaIssues[tableId].ColumnLevelIssues[columnId] = colIssues
+				}
+
+			}
 		}
 	}
 }

sources/mysql/infoschema.go

@@ -176,7 +176,7 @@ func (isi InfoSchemaImpl) GetTables() ([]common.SchemaAndName, error) {
 
 // GetColumns returns a list of Column objects and names// ProcessColumns
 func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAndName, constraints map[string][]string, primaryKeys []string) (map[string]schema.Column, []string, error) {
-	q := `SELECT c.column_name, c.data_type, c.column_type, c.is_nullable, c.column_default, c.character_maximum_length, c.numeric_precision, c.numeric_scale, c.extra
+	q := `SELECT c.column_name, c.data_type, c.column_type, c.is_nullable, c.column_default, c.character_maximum_length, c.numeric_precision, c.numeric_scale, c.generation_expression, c.extra
               FROM information_schema.COLUMNS c
               where table_schema = ? and table_name = ? ORDER BY c.ordinal_position;`
 	cols, err := isi.Db.Query(q, table.Schema, table.Name)
@@ -187,15 +187,20 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd
 	colDefs := make(map[string]schema.Column)
 	var colIds []string
 	var colName, dataType, isNullable, columnType string
-	var colDefault, colExtra sql.NullString
+	var colDefault, colExtra, colGeneratedExpression sql.NullString
 	var charMaxLen, numericPrecision, numericScale sql.NullInt64
 	var colAutoGen ddl.AutoGenCol
 	for cols.Next() {
-		err := cols.Scan(&colName, &dataType, &columnType, &isNullable, &colDefault, &charMaxLen, &numericPrecision, &numericScale, &colExtra)
+		err := cols.Scan(&colName, &dataType, &columnType, &isNullable, &colDefault, &charMaxLen, &numericPrecision, &numericScale, &colGeneratedExpression, &colExtra)
 		if err != nil {
 			conv.Unexpected(fmt.Sprintf("Can't scan: %v", err))
 			continue
 		}
+
+		// It's required as empty string is considered as valid within Database SQL.
+		if colGeneratedExpression.String == "" {
+			colGeneratedExpression.Valid = false
+		}
 		ignored := schema.Ignored{}
 		ignored.Default = colDefault.Valid
 		colId := internal.GenerateColumnId()
@@ -223,14 +228,32 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd
 			}
 		}
 
+		generatedColumn := ddl.GeneratedColumn{
+			IsPresent: colGeneratedExpression.Valid,
+			Value:     ddl.Expression{},
+		}
+		if colGeneratedExpression.Valid {
+			ignored.GeneratedCol = colGeneratedExpression.Valid
+			// Defaults to STORED type
+			generatedColumn.Type = ddl.GeneratedColStored
+			if colExtra.String == constants.VIRTUAL_GENERATED {
+				generatedColumn.Type = ddl.GeneratedColVirtual
+			}
+			generatedColumn.Value = ddl.Expression{
+				ExpressionId: internal.GenerateExpressionId(),
+				Statement:    common.SanitizeDefaultValue(colGeneratedExpression.String, "", false),
+			}
+		}
+
 		c := schema.Column{
-			Id:           colId,
-			Name:         colName,
-			Type:         toType(dataType, columnType, charMaxLen, numericPrecision, numericScale),
-			NotNull:      common.ToNotNull(conv, isNullable),
-			Ignored:      ignored,
-			AutoGen:      colAutoGen,
-			DefaultValue: defaultVal,
+			Id:              colId,
+			Name:            colName,
+			Type:            toType(dataType, columnType, charMaxLen, numericPrecision, numericScale),
+			NotNull:         common.ToNotNull(conv, isNullable),
+			Ignored:         ignored,
+			AutoGen:         colAutoGen,
+			DefaultValue:    defaultVal,
+			GeneratedColumn: generatedColumn,
 		}
 		colDefs[colId] = c
 		colIds = append(colIds, colId)