Skip to content

Commit c157187

Browse files
pratickchokhanipratickchokhani
committed
Generated Columns Support
- Added generated column type in Mysql Info schema. - Serialized generated column expression.
1 parent d00573f commit c157187

File tree

25 files changed

+495
-117
lines changed

25 files changed

+495
-117
lines changed

‎common/constants/constants.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@ const (
131131
CHECK_EXPRESSION = "CHECK"
132132
DEFAULT_EXPRESSION = "DEFAULT"
133133
DEFAULT_GENERATED = "DEFAULT_GENERATED"
134+
STORED_GENERATED = "STORED"
135+
VIRTUAL_GENERATED = "VIRTUAL"
134136
TEMP_DB = "smt-staging-db"
135137
DB_URI = "projects/%s/instances/%s/databases/%s"
136138

‎expressions_api/expression_verify.go‎

Lines changed: 54 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,8 @@ func (ev *ExpressionVerificationAccessorImpl) verifyExpressionInternal(expressio
129129
sqlStatement = fmt.Sprintf("SELECT 1 from %s where %s;", expressionDetail.ReferenceElement.Name, expressionDetail.Expression)
130130
case constants.DEFAULT_EXPRESSION:
131131
sqlStatement = fmt.Sprintf("SELECT CAST(%s as %s)", expressionDetail.Expression, expressionDetail.ReferenceElement.Name)
132+
case constants.STORED_GENERATED, constants.VIRTUAL_GENERATED:
133+
sqlStatement = fmt.Sprintf("SELECT %s as %s FROM %s", expressionDetail.Expression, expressionDetail.ReferenceElement.Name, expressionDetail.SpTableName)
132134
default:
133135
return task.TaskResult[internal.ExpressionVerificationOutput]{Result: internal.ExpressionVerificationOutput{Result: false, Err: fmt.Errorf("invalid expression type requested")}, Err: nil}
134136
}
@@ -171,6 +173,7 @@ func (ev *ExpressionVerificationAccessorImpl) removeExpressions(inputConv *inter
171173
for colName, colDef := range table.ColDefs {
172174
colDef.AutoGen = ddl.AutoGenCol{}
173175
colDef.DefaultValue = ddl.DefaultValue{}
176+
colDef.GeneratedColumn = ddl.GeneratedColumn{}
174177
table.ColDefs[colName] = colDef
175178
}
176179
}
@@ -197,27 +200,64 @@ func (ddlv *DDLVerifierImpl) GetSourceExpressionDetails(conv *internal.Conv, tab
197200
srcTable := conv.SrcSchema[tableId]
198201
for _, srcColId := range srcTable.ColIds {
199202
srcCol := srcTable.ColDefs[srcColId]
203+
var expression ddl.Expression
204+
var expressionType string
205+
isExpressionAvailable := false
200206
if srcCol.DefaultValue.IsPresent {
207+
expression = srcCol.DefaultValue.Value
208+
isExpressionAvailable = true
209+
expressionType = constants.DEFAULT_EXPRESSION
210+
} else if srcCol.GeneratedColumn.IsPresent {
211+
expression = srcCol.GeneratedColumn.Value
212+
isExpressionAvailable = true
213+
if srcCol.GeneratedColumn.Type == ddl.GeneratedColStored {
214+
expressionType = constants.STORED_GENERATED
215+
} else {
216+
expressionType = constants.VIRTUAL_GENERATED
217+
}
218+
}
219+
if isExpressionAvailable {
201220
tyName := conv.SpSchema[tableId].ColDefs[srcColId].T.Name
202221
if conv.SpDialect == constants.DIALECT_POSTGRESQL {
203222
tyName = ddl.GetPGType(conv.SpSchema[tableId].ColDefs[srcColId].T)
204223
}
205-
defaultValueExp := internal.ExpressionDetail{
224+
expressionDetail := internal.ExpressionDetail{
206225
ReferenceElement: internal.ReferenceElement{
207226
Name: tyName,
208227
},
209-
ExpressionId: srcCol.DefaultValue.Value.ExpressionId,
210-
Expression: srcCol.DefaultValue.Value.Statement,
211-
Type: constants.DEFAULT_EXPRESSION,
228+
ExpressionId: expression.ExpressionId,
229+
Expression: expression.Statement,
230+
Type: expressionType,
231+
SpTableName: conv.ToSpanner[srcTable.Name].Name,
212232
Metadata: map[string]string{"TableId": tableId, "ColId": srcColId},
213233
}
214-
expressionDetails = append(expressionDetails, defaultValueExp)
234+
expressionDetails = append(expressionDetails, expressionDetail)
215235
}
216236
}
217237
}
218238
return expressionDetails
219239
}
220240

241+
func (ddlv *DDLVerifierImpl) getExpressionDetail(
242+
conv *internal.Conv, tableId, spColId, expressionType, expressionId, expression string, spCol ddl.ColumnDef) internal.ExpressionDetail {
243+
tyName := conv.SpSchema[tableId].ColDefs[spColId].T.Name
244+
if conv.SpDialect == constants.DIALECT_POSTGRESQL {
245+
tyName = ddl.GetPGType(conv.SpSchema[tableId].ColDefs[spColId].T)
246+
}
247+
expressionDetail := internal.ExpressionDetail{
248+
ReferenceElement: internal.ReferenceElement{
249+
Name: tyName,
250+
},
251+
ExpressionId: expressionId,
252+
Expression: expression,
253+
Type: expressionType,
254+
Metadata: map[string]string{"TableId": tableId, "ColId": spColId},
255+
SpTableName: conv.SpSchema[tableId].Name,
256+
}
257+
return expressionDetail
258+
259+
}
260+
221261
func (ddlv *DDLVerifierImpl) GetSpannerExpressionDetails(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail {
222262
expressionDetails := []internal.ExpressionDetail{}
223263
// Collect default values for verification
@@ -226,21 +266,17 @@ func (ddlv *DDLVerifierImpl) GetSpannerExpressionDetails(conv *internal.Conv, ta
226266
for _, spColId := range spTable.ColIds {
227267
spCol := spTable.ColDefs[spColId]
228268
if spCol.DefaultValue.IsPresent {
229-
tyName := conv.SpSchema[tableId].ColDefs[spColId].T.Name
230-
if conv.SpDialect == constants.DIALECT_POSTGRESQL {
231-
tyName = ddl.GetPGType(conv.SpSchema[tableId].ColDefs[spColId].T)
232-
}
233-
defaultValueExp := internal.ExpressionDetail{
234-
ReferenceElement: internal.ReferenceElement{
235-
Name: tyName,
236-
},
237-
ExpressionId: spCol.DefaultValue.Value.ExpressionId,
238-
Expression: spCol.DefaultValue.Value.Statement,
239-
Type: constants.DEFAULT_EXPRESSION,
240-
Metadata: map[string]string{"TableId": tableId, "ColId": spColId},
241-
}
269+
defaultValueExp := ddlv.getExpressionDetail(
270+
conv, tableId, spColId, constants.DEFAULT_EXPRESSION, spCol.DefaultValue.Value.ExpressionId,
271+
spCol.DefaultValue.Value.Statement, spCol)
242272
expressionDetails = append(expressionDetails, defaultValueExp)
243273
}
274+
if spCol.GeneratedColumn.IsPresent {
275+
generatedColExp := ddlv.getExpressionDetail(
276+
conv, tableId, spColId, string(spCol.GeneratedColumn.Type), spCol.GeneratedColumn.Value.ExpressionId,
277+
spCol.GeneratedColumn.Value.Statement, spCol)
278+
expressionDetails = append(expressionDetails, generatedColExp)
279+
}
244280
}
245281
}
246282
return expressionDetails

‎internal/convert.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ const (
158158
CassandraMAP
159159
PossibleOverflow
160160
IdentitySkipRange
161+
GeneratedColumnValueError
161162
)
162163

163164
const (
@@ -331,6 +332,7 @@ type ExpressionDetail struct {
331332
ExpressionId string
332333
Expression string
333334
Type string
335+
SpTableName string
334336
Metadata map[string]string
335337
}
336338

‎internal/reports/report_helpers.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ func buildTableReportBody(conv *internal.Conv, tableId string, issues map[string
306306
// on case of srcType.
307307
spColType = strings.ToLower(spColType)
308308
switch i {
309-
case internal.DefaultValue:
309+
case internal.DefaultValue, internal.GeneratedColumnValueError:
310310
toAppend := Issue{
311311
Category: IssueDB[i].Category,
312312
Description: fmt.Sprintf("%s for table '%s' e.g. column '%s'", IssueDB[i].Brief, conv.SpSchema[tableId].Name, spColName),
@@ -621,6 +621,7 @@ var IssueDB = map[internal.SchemaIssue]struct {
621621
CategoryDescription string
622622
}{
623623
internal.DefaultValue: {Brief: "Some columns have default values which Spanner migration tool does not migrate. Please add the default constraints manually after the migration is complete", Severity: note, batch: true, Category: "MISSING_DEFAULT_VALUE_CONSTRAINTS"},
624+
internal.GeneratedColumnValueError: {Brief: "Some columns have generated expression which Spanner migration tool cannot not migrate. Please add the expressions manually", Severity: warning, batch: false, Category: "MISSING_GENERATED_COL_VALUE_CONSTRAINTS"},
624625
internal.TypeMismatch: {Brief: "Type mismatch in check constraint mention in table", Severity: warning, Category: "TYPE_MISMATCH"},
625626
internal.TypeMismatchError: {Brief: "Type mismatch in check constraint mention in table", Severity: Errors, Category: "TYPE_MISMATCH_ERROR"},
626627
internal.InvalidCondition: {Brief: "Invalid condition in check constraint mention in table", Severity: warning, Category: "INVALID_CONDITION"},

‎schema/schema.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,14 @@ type Table struct {
5050
// Column represents a database column.
5151
// TODO: add support for foreign keys.
5252
type Column struct {
53-
Name string
54-
Type Type
55-
NotNull bool
56-
Ignored Ignored
57-
Id string
58-
AutoGen ddl.AutoGenCol
59-
DefaultValue ddl.DefaultValue
53+
Name string
54+
Type Type
55+
NotNull bool
56+
Ignored Ignored
57+
Id string
58+
AutoGen ddl.AutoGenCol
59+
DefaultValue ddl.DefaultValue
60+
GeneratedColumn ddl.GeneratedColumn
6061
}
6162

6263
// ForeignKey represents a foreign key.

‎sources/common/infoschema.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -256,8 +256,8 @@ func (is *InfoSchemaImpl) GetIncludedSrcTablesFromConv(conv *internal.Conv) (sch
256256
return schemaToTablesMap, nil
257257
}
258258

259-
// SanitizeDefaultValue removes extra characters added to Default Value in information schema in MySQL.
260-
func SanitizeDefaultValue(defaultValue string, ty string, generated bool) string {
259+
// SanitizeExpressionsValue removes extra characters added to Default Value in information schema in MySQL.
260+
func SanitizeExpressionsValue(expressionValue string, ty string, generated bool) string {
261261
types := []string{"char", "varchar", "text", "varbinary", "tinyblob", "tinytext", "text",
262262
"blob", "mediumtext", "mediumblob", "longtext", "longblob", "STRING"}
263263
// Check if ty exists in the types array
@@ -268,11 +268,11 @@ func SanitizeDefaultValue(defaultValue string, ty string, generated bool) string
268268
break
269269
}
270270
}
271-
defaultValue = strings.ReplaceAll(defaultValue, "_utf8mb4", "")
272-
defaultValue = strings.ReplaceAll(defaultValue, "\\\\", "\\")
273-
defaultValue = strings.ReplaceAll(defaultValue, "\\'", "'")
274-
if !generated && stringType && !strings.HasPrefix(defaultValue, "'") && !strings.HasSuffix(defaultValue, "'") {
275-
defaultValue = "'" + defaultValue + "'"
271+
expressionValue = strings.ReplaceAll(expressionValue, "_utf8mb4", "")
272+
expressionValue = strings.ReplaceAll(expressionValue, "\\\\", "\\")
273+
expressionValue = strings.ReplaceAll(expressionValue, "\\'", "'")
274+
if !generated && stringType && !strings.HasPrefix(expressionValue, "'") && !strings.HasSuffix(expressionValue, "'") {
275+
expressionValue = "'" + expressionValue + "'"
276276
}
277-
return defaultValue
277+
return expressionValue
278278
}

‎sources/common/infoschema_test.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ import (
2020
"github.com/stretchr/testify/assert"
2121
)
2222

23-
24-
func TestSanitizeDefaultValue(t *testing.T) {
23+
func TestSanitizeExpressionsValue(t *testing.T) {
2524
tests := []struct {
2625
inputString string
2726
ty string
@@ -41,7 +40,7 @@ func TestSanitizeDefaultValue(t *testing.T) {
4140
{"_utf8mb4\\'C:\\\\\\\\Users\\\\\\\\johndoe\\\\\\\\Documents\\\\\\\\myfile.txt\\'", "char", false, "'C:\\\\Users\\\\johndoe\\\\Documents\\\\myfile.txt'"},
4241
}
4342
for _, test := range tests {
44-
result := SanitizeDefaultValue(test.inputString, test.ty, test.generated)
43+
result := SanitizeExpressionsValue(test.inputString, test.ty, test.generated)
4544
assert.Equal(t, test.expectedString, result)
4645
}
4746
}

‎sources/common/toddl.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,6 +696,28 @@ func spannerSchemaApplyExpressions(conv *internal.Conv, expressions internal.Ver
696696
conv.SchemaIssues[tableId].ColumnLevelIssues[columnId] = colIssues
697697
}
698698
}
699+
case constants.VIRTUAL_GENERATED, constants.STORED_GENERATED:
700+
{
701+
tableId := expression.ExpressionDetail.Metadata["TableId"]
702+
columnId := expression.ExpressionDetail.Metadata["ColId"]
703+
704+
if expression.Result {
705+
col := conv.SpSchema[tableId].ColDefs[columnId]
706+
col.GeneratedColumn = ddl.GeneratedColumn{
707+
IsPresent: true,
708+
Value: ddl.Expression{
709+
ExpressionId: expression.ExpressionDetail.ExpressionId,
710+
Statement: fmt.Sprintf("(%s)", expression.ExpressionDetail.Expression),
711+
},
712+
Type: ddl.GeneratedColType(expression.ExpressionDetail.Type),
713+
}
714+
conv.SpSchema[tableId].ColDefs[columnId] = col
715+
} else {
716+
colIssues := conv.SchemaIssues[tableId].ColumnLevelIssues[columnId]
717+
colIssues = append(colIssues, internal.GeneratedColumnValueError)
718+
conv.SchemaIssues[tableId].ColumnLevelIssues[columnId] = colIssues
719+
}
720+
}
699721
}
700722
}
701723
}

‎sources/mysql/infoschema.go‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ func (isi InfoSchemaImpl) GetTables() ([]common.SchemaAndName, error) {
176176

177177
// GetColumns returns a list of Column objects and names// ProcessColumns
178178
func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAndName, constraints map[string][]string, primaryKeys []string) (map[string]schema.Column, []string, error) {
179-
q := `SELECT c.column_name, c.data_type, c.column_type, c.is_nullable, c.column_default, c.character_maximum_length, c.numeric_precision, c.numeric_scale, c.extra
179+
q := `SELECT c.column_name, c.data_type, c.column_type, c.is_nullable, c.column_default, c.character_maximum_length, c.numeric_precision, c.numeric_scale, c.generation_expression, c.extra
180180
FROM information_schema.COLUMNS c
181181
where table_schema = ? and table_name = ? ORDER BY c.ordinal_position;`
182182
cols, err := isi.Db.Query(q, table.Schema, table.Name)
@@ -187,15 +187,20 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd
187187
colDefs := make(map[string]schema.Column)
188188
var colIds []string
189189
var colName, dataType, isNullable, columnType string
190-
var colDefault, colExtra sql.NullString
190+
var colDefault, colExtra, colGeneratedExpression sql.NullString
191191
var charMaxLen, numericPrecision, numericScale sql.NullInt64
192192
var colAutoGen ddl.AutoGenCol
193193
for cols.Next() {
194-
err := cols.Scan(&colName, &dataType, &columnType, &isNullable, &colDefault, &charMaxLen, &numericPrecision, &numericScale, &colExtra)
194+
err := cols.Scan(&colName, &dataType, &columnType, &isNullable, &colDefault, &charMaxLen, &numericPrecision, &numericScale, &colGeneratedExpression, &colExtra)
195195
if err != nil {
196196
conv.Unexpected(fmt.Sprintf("Can't scan: %v", err))
197197
continue
198198
}
199+
200+
// It's required as empty string is considered as valid within Database SQL.
201+
if colGeneratedExpression.String == "" {
202+
colGeneratedExpression.Valid = false
203+
}
199204
ignored := schema.Ignored{}
200205
ignored.Default = colDefault.Valid
201206
colId := internal.GenerateColumnId()
@@ -219,18 +224,35 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd
219224
}
220225
defaultVal.Value = ddl.Expression{
221226
ExpressionId: internal.GenerateExpressionId(),
222-
Statement: common.SanitizeDefaultValue(colDefault.String, ty, colExtra.String == constants.DEFAULT_GENERATED),
227+
Statement: common.SanitizeExpressionsValue(colDefault.String, ty, colExtra.String == constants.DEFAULT_GENERATED),
228+
}
229+
}
230+
231+
generatedColumn := ddl.GeneratedColumn{
232+
IsPresent: colGeneratedExpression.Valid,
233+
Value: ddl.Expression{},
234+
}
235+
if colGeneratedExpression.Valid {
236+
// Defaults to STORED type
237+
generatedColumn.Type = ddl.GeneratedColStored
238+
if colExtra.String == constants.VIRTUAL_GENERATED {
239+
generatedColumn.Type = ddl.GeneratedColVirtual
240+
}
241+
generatedColumn.Value = ddl.Expression{
242+
ExpressionId: internal.GenerateExpressionId(),
243+
Statement: common.SanitizeExpressionsValue(colGeneratedExpression.String, "", false),
223244
}
224245
}
225246

226247
c := schema.Column{
227-
Id: colId,
228-
Name: colName,
229-
Type: toType(dataType, columnType, charMaxLen, numericPrecision, numericScale),
230-
NotNull: common.ToNotNull(conv, isNullable),
231-
Ignored: ignored,
232-
AutoGen: colAutoGen,
233-
DefaultValue: defaultVal,
248+
Id: colId,
249+
Name: colName,
250+
Type: toType(dataType, columnType, charMaxLen, numericPrecision, numericScale),
251+
NotNull: common.ToNotNull(conv, isNullable),
252+
Ignored: ignored,
253+
AutoGen: colAutoGen,
254+
DefaultValue: defaultVal,
255+
GeneratedColumn: generatedColumn,
234256
}
235257
colDefs[colId] = c
236258
colIds = append(colIds, colId)

0 commit comments

Comments
 (0)