Skip to content

Commit 3a82ec4

Browse files
pratickchokhanipratickchokhani
committed
Generated Columns Support
- Added generated column type in Mysql Info schema. - Serialized generated column expression.
1 parent dedefca commit 3a82ec4

File tree

12 files changed

+127
-37
lines changed

12 files changed

+127
-37
lines changed

‎common/constants/constants.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,8 @@ const (
129129
CHECK_EXPRESSION = "CHECK"
130130
DEFAULT_EXPRESSION = "DEFAULT"
131131
DEFAULT_GENERATED = "DEFAULT_GENERATED"
132+
STORED_GENERATED = "STORED GENERATED"
133+
VIRTUAL_GENERATED = "VIRTUAL GENERATED"
132134
TEMP_DB = "smt-staging-db"
133135
DB_URI = "projects/%s/instances/%s/databases/%s"
134136

‎expressions_api/expression_verify.go‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
78
mtrand "math/rand"
89
"sync"
910
"time"
@@ -197,21 +198,38 @@ func (ddlv *DDLVerifierImpl) GetSourceExpressionDetails(conv *internal.Conv, tab
197198
srcTable := conv.SrcSchema[tableId]
198199
for _, srcColId := range srcTable.ColIds {
199200
srcCol := srcTable.ColDefs[srcColId]
201+
var expression ddl.Expression
202+
var expressionType string
203+
isExpressionAvailable := false
200204
if srcCol.DefaultValue.IsPresent {
205+
expression = srcCol.DefaultValue.Value
206+
isExpressionAvailable = true
207+
expressionType = constants.DEFAULT_EXPRESSION
208+
} else if srcCol.GeneratedColumn.IsPresent {
209+
expression = srcCol.GeneratedColumn.Value
210+
isExpressionAvailable = true
211+
if srcCol.GeneratedColumn.Type == ddl.GeneratedColStored {
212+
expressionType = constants.STORED_GENERATED
213+
} else {
214+
expressionType = constants.VIRTUAL_GENERATED
215+
}
216+
}
217+
if isExpressionAvailable {
201218
tyName := conv.SpSchema[tableId].ColDefs[srcColId].T.Name
202219
if conv.SpDialect == constants.DIALECT_POSTGRESQL {
203220
tyName = ddl.GetPGType(conv.SpSchema[tableId].ColDefs[srcColId].T)
204221
}
205-
defaultValueExp := internal.ExpressionDetail{
222+
expressionDetail := internal.ExpressionDetail{
206223
ReferenceElement: internal.ReferenceElement{
207224
Name: tyName,
208225
},
209-
ExpressionId: srcCol.DefaultValue.Value.ExpressionId,
210-
Expression: srcCol.DefaultValue.Value.Statement,
211-
Type: constants.DEFAULT_EXPRESSION,
226+
ExpressionId: expression.ExpressionId,
227+
Expression: expression.Statement,
228+
Type: expressionType,
212229
Metadata: map[string]string{"TableId": tableId, "ColId": srcColId},
213230
}
214-
expressionDetails = append(expressionDetails, defaultValueExp)
231+
logger.Log.Info(fmt.Sprintf("AExpression Details: %v", expressionDetail))
232+
expressionDetails = append(expressionDetails, expressionDetail)
215233
}
216234
}
217235
}

‎schema/schema.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,14 @@ type Table struct {
5050
// Column represents a database column.
5151
// TODO: add support for foreign keys.
5252
type Column struct {
53-
Name string
54-
Type Type
55-
NotNull bool
56-
Ignored Ignored
57-
Id string
58-
AutoGen ddl.AutoGenCol
59-
DefaultValue ddl.DefaultValue
53+
Name string
54+
Type Type
55+
NotNull bool
56+
Ignored Ignored
57+
Id string
58+
AutoGen ddl.AutoGenCol
59+
DefaultValue ddl.DefaultValue
60+
GeneratedColumn ddl.GeneratedColumn
6061
}
6162

6263
// ForeignKey represents a foreign key.

‎sources/mysql/infoschema.go‎

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"strings"
2424

2525
sp "cloud.google.com/go/spanner"
26+
"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
2627
_ "github.com/go-sql-driver/mysql" // The driver should be used via the database/sql package.
2728
_ "github.com/lib/pq"
2829

@@ -176,7 +177,7 @@ func (isi InfoSchemaImpl) GetTables() ([]common.SchemaAndName, error) {
176177

177178
// GetColumns returns a list of Column objects and names// ProcessColumns
178179
func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAndName, constraints map[string][]string, primaryKeys []string) (map[string]schema.Column, []string, error) {
179-
q := `SELECT c.column_name, c.data_type, c.column_type, c.is_nullable, c.column_default, c.character_maximum_length, c.numeric_precision, c.numeric_scale, c.extra
180+
q := `SELECT c.column_name, c.data_type, c.column_type, c.is_nullable, c.column_default, c.character_maximum_length, c.numeric_precision, c.numeric_scale, c.generation_expression, c.extra
180181
FROM information_schema.COLUMNS c
181182
where table_schema = ? and table_name = ? ORDER BY c.ordinal_position;`
182183
cols, err := isi.Db.Query(q, table.Schema, table.Name)
@@ -187,15 +188,23 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd
187188
colDefs := make(map[string]schema.Column)
188189
var colIds []string
189190
var colName, dataType, isNullable, columnType string
190-
var colDefault, colExtra sql.NullString
191+
var colDefault, colExtra, colGeneratedExpression sql.NullString
191192
var charMaxLen, numericPrecision, numericScale sql.NullInt64
192193
var colAutoGen ddl.AutoGenCol
193194
for cols.Next() {
194-
err := cols.Scan(&colName, &dataType, &columnType, &isNullable, &colDefault, &charMaxLen, &numericPrecision, &numericScale, &colExtra)
195+
logger.Log.Info("")
196+
err := cols.Scan(&colName, &dataType, &columnType, &isNullable, &colDefault, &charMaxLen, &numericPrecision, &numericScale, &colGeneratedExpression, &colExtra)
197+
// Log all the details
198+
logger.Log.Info(fmt.Sprintf("%v, %v, %v, %v, %v, %v, %v, %v, %v", colName, dataType, columnType, isNullable, colDefault, charMaxLen, numericPrecision, numericScale, colExtra))
195199
if err != nil {
196200
conv.Unexpected(fmt.Sprintf("Can't scan: %v", err))
197201
continue
198202
}
203+
204+
// It's required as empty string is considered as valid within Database SQL.
205+
if colGeneratedExpression.String == "" {
206+
colGeneratedExpression.Valid = false
207+
}
199208
ignored := schema.Ignored{}
200209
ignored.Default = colDefault.Valid
201210
colId := internal.GenerateColumnId()
@@ -230,14 +239,33 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd
230239
}
231240
}
232241

242+
generatedColumn := ddl.GeneratedColumn{
243+
IsPresent: colGeneratedExpression.Valid,
244+
Value: ddl.Expression{},
245+
}
246+
logger.Log.Info(fmt.Sprintf("colGeneratedExpression: %v. colDefault: %v", colGeneratedExpression, colDefault))
247+
if colGeneratedExpression.Valid {
248+
// Defaults to STORED type
249+
generatedColumn.Type = ddl.GeneratedColStored
250+
if colExtra.String == constants.VIRTUAL_GENERATED {
251+
generatedColumn.Type = ddl.GeneratedColVirtual
252+
}
253+
generatedColumn.Value = ddl.Expression{
254+
ExpressionId: internal.GenerateExpressionId(),
255+
Statement: common.SanitizeDefaultValue(colGeneratedExpression.String, "", false),
256+
}
257+
}
258+
259+
logger.Log.Info(fmt.Sprintf("Generated Col: %v", generatedColumn))
233260
c := schema.Column{
234-
Id: colId,
235-
Name: colName,
236-
Type: toType(dataType, columnType, charMaxLen, numericPrecision, numericScale),
237-
NotNull: common.ToNotNull(conv, isNullable),
238-
Ignored: ignored,
239-
AutoGen: colAutoGen,
240-
DefaultValue: defaultVal,
261+
Id: colId,
262+
Name: colName,
263+
Type: toType(dataType, columnType, charMaxLen, numericPrecision, numericScale),
264+
NotNull: common.ToNotNull(conv, isNullable),
265+
Ignored: ignored,
266+
AutoGen: colAutoGen,
267+
DefaultValue: defaultVal,
268+
GeneratedColumn: generatedColumn,
241269
}
242270
colDefs[colId] = c
243271
colIds = append(colIds, colId)

‎spanner/ddl/ast.go‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ import (
3131
"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
3232
)
3333

34+
type GeneratedColType string
35+
3436
const (
3537
// Types supported by Spanner with google_standard_sql (default) dialect.
3638
// Bool represent BOOL type.
@@ -78,7 +80,9 @@ const (
7880
// Jsonb represents the PG.JSONB type
7981
PGJSONB string = "JSONB"
8082
// PGMaxLength represents sentinel for Type's Len field in PG.
81-
PGMaxLength = 2621440
83+
PGMaxLength = 2621440
84+
GeneratedColStored GeneratedColType = "STORED"
85+
GeneratedColVirtual GeneratedColType = "VIRTUAL"
8286
)
8387

8488
var STANDARD_TYPE_TO_PGSQL_TYPEMAP = map[string]string{
@@ -253,12 +257,12 @@ func (cd ColumnDef) PrintColumnDef(c Config) (string, string) {
253257
s += cd.DefaultValue.PrintDefaultValue(cd.T)
254258
s += cd.AutoGen.PrintAutoGenCol()
255259
}
256-
var opts []string
260+
var opts []string
257261
if cd.Opts != nil {
258262
if opt, ok := cd.Opts["cassandra_type"]; ok && opt != "" {
259263
opts = append(opts, fmt.Sprintf("cassandra_type = '%s'", opt))
260264
}
261-
}
265+
}
262266
if len(opts) > 0 {
263267
s += " OPTIONS (" + strings.Join(opts, ", ") + ")"
264268
}
@@ -445,6 +449,13 @@ type CreateIndex struct {
445449
// interleaving clauses yet, so we omit them for now.
446450
}
447451

452+
// GeneratedColumn represents a Generated Column.
453+
type GeneratedColumn struct {
454+
IsPresent bool
455+
Value Expression
456+
Type GeneratedColType
457+
}
458+
448459
type AutoGenCol struct {
449460
Name string
450461
// Type of autogenerated column, example, pre-defined(uuid) or user-defined(sequence)

‎ui/src/app/components/object-detail/object-detail.component.html‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,18 @@ <h3 class="title">
150150
</td>
151151
</ng-container>
152152

153+
<ng-container matColumnDef="srcGeneratedColExp" *ngIf="mySqlSource">
154+
<th mat-header-cell class="table_header" *matHeaderCellDef>Generated-Col</th>
155+
<td mat-cell *matCellDef="let element">
156+
<div
157+
class="trimmed-text"
158+
matTooltip="{{ element.get('srcGeneratedColExp').value }}"
159+
matTooltipPosition="above">
160+
{{ element.get('srcGeneratedColExp').value }}
161+
</div>
162+
</td>
163+
</ng-container>
164+
153165
<ng-container matColumnDef="srcIsPk">
154166
<th class="table_header" mat-header-cell *matHeaderCellDef>Pk</th>
155167
<td mat-cell *matCellDef="let element">
@@ -326,13 +338,13 @@ <h3 class="title">
326338
<td mat-cell *matCellDef="let element">
327339
<div *ngIf="!isEditMode" class="trimmed-text"
328340
matTooltip="{{ element.get('spDefaultValue').value }}" matTooltipPosition="above">
329-
{{ element.get('spDefaultValue').value }}
341+
{{ element.get('spDefaultValue').value }}
330342
</div>
331-
343+
332344
<div *ngIf="isEditMode">
333-
<input matInput class="name_input" type="text"
334-
[formControl]="element.get('spDefaultValue')"
335-
matTooltip="Specify the default value"/>
345+
<input matInput class="name_input" type="text"
346+
[formControl]="element.get('spDefaultValue')"
347+
matTooltip="Specify the default value"/>
336348
</div>
337349
</td>
338350
</ng-container>
@@ -1173,4 +1185,4 @@ <h3 class="title">
11731185
</button>
11741186
</div>
11751187
</div>
1176-
</div>
1188+
</div>

‎ui/src/app/components/object-detail/object-detail.component.scss‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@
207207
.mat-column-spColName,
208208
.mat-column-srcDataType,
209209
.mat-column-spDataType,
210+
.mat-column-srcGeneratedColExp,
210211
.mat-column-spAutoGen,
211212
.mat-column-spDefaultValue,
212213
.mat-column-spIndexColName,

‎ui/src/app/components/object-detail/object-detail.component.spec.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ describe('ObjectDetailComponent', () => {
101101
GenerationType: ''
102102
},
103103
srcDefaultValue: '',
104+
srcGeneratedColExp: '',
104105
spDefaultValue: {
105106
Value: {
106107
ExpressionId: '',
@@ -133,6 +134,7 @@ describe('ObjectDetailComponent', () => {
133134
Name: '',
134135
GenerationType: ''
135136
},
137+
srcGeneratedColExp: '',
136138
srcDefaultValue: '',
137139
spDefaultValue: {
138140
Value: {
@@ -469,8 +471,9 @@ describe('ObjectDetailComponent', () => {
469471
spIsPk: true,
470472
spIsNotNull: true,
471473
spColMaxLength: '',
472-
spCassandraOption: 'list<bigint>',
474+
spCassandraOption: 'list<bigint>',
473475
spAutoGen: { Name: '', GenerationType: '' },
476+
srcGeneratedColExp: '',
474477
spDefaultValue: '',
475478
}),
476479
])

‎ui/src/app/components/object-detail/object-detail.component.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ export class ObjectDetailComponent implements OnInit, OnDestroy {
225225
this.interleaveType = this.getInterleaveTypeFromConv()
226226
this.onDeleteAction = this.getInterleaveOnDeleteActionFromConv() ?? ''
227227

228-
228+
229229
this.isEditMode = false
230230
this.isFkEditMode = false
231231
this.isIndexEditMode = false
@@ -251,7 +251,8 @@ export class ObjectDetailComponent implements OnInit, OnDestroy {
251251
this.displayedPkColumns.splice(8, 0, "spAutoGen");
252252
this.srcDisplayedColumns.splice(2, 0, "srcAutoGen");
253253
this.displayedPkColumns.splice(2, 0, "srcAutoGen");
254-
this.srcDisplayedColumns.push("srcDefaultValue");;
254+
this.srcDisplayedColumns.push("srcGeneratedColExp");
255+
this.srcDisplayedColumns.push("srcDefaultValue");
255256
this.spDisplayedColumns.splice(4, 0,"spDefaultValue");
256257
this.spColspan+=2;
257258
this.srcColspan+=2;
@@ -300,6 +301,7 @@ export class ObjectDetailComponent implements OnInit, OnDestroy {
300301
srcIsPk: new FormControl(row.srcIsPk),
301302
srcIsNotNull: new FormControl(row.srcIsNotNull),
302303
srcDefaultValue: new FormControl(row.srcDefaultValue),
304+
srcGeneratedColExp: new FormControl(row.srcGeneratedColExp),
303305
srcColMaxLength: new FormControl(row.srcColMaxLength),
304306
srcAutoGen: new FormControl(row.srcAutoGen),
305307
spOrder: new FormControl(row.srcOrder),
@@ -369,6 +371,7 @@ export class ObjectDetailComponent implements OnInit, OnDestroy {
369371
srcIsPk: new FormControl(col.srcIsPk),
370372
srcIsNotNull: new FormControl(col.srcIsNotNull),
371373
srcDefaultValue: new FormControl(col.srcDefaultValue),
374+
srcGeneratedColExp: new FormControl(col.srcGeneratedColExp),
372375
srcColMaxLength: new FormControl(col.srcColMaxLength),
373376
srcAutoGen: new FormControl(col.srcAutoGen),
374377
spOrder: new FormControl(col.spOrder),
@@ -408,6 +411,7 @@ export class ObjectDetailComponent implements OnInit, OnDestroy {
408411
srcIsPk: new FormControl(col.srcIsPk),
409412
srcIsNotNull: new FormControl(col.srcIsNotNull),
410413
srcDefaultValue: new FormControl(col.srcDefaultValue),
414+
srcGeneratedColExp: new FormControl(col.srcGeneratedColExp),
411415
srcColMaxLength: new FormControl(col.srcColMaxLength),
412416
srcAutoGen: new FormControl(col.srcAutoGen),
413417
spOrder: new FormControl(col.srcOrder),
@@ -1512,7 +1516,7 @@ export class ObjectDetailComponent implements OnInit, OnDestroy {
15121516
console.error('Interleave type cannot be empty');
15131517
return;
15141518
}
1515-
1519+
15161520
let tableId = this.currentObject!.id;
15171521

15181522
this.data
@@ -1959,4 +1963,4 @@ export class ObjectDetailComponent implements OnInit, OnDestroy {
19591963
ngOnDestroy(): void {
19601964
this.subscriptions.unsubscribe()
19611965
}
1962-
}
1966+
}

‎ui/src/app/model/conv.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@ export interface IDefaultValue {
2929
Value: IExpression
3030
}
3131

32+
export interface IGeneratedColumn {
33+
IsPresent: boolean
34+
Value: IExpression
35+
Type: string
36+
}
37+
3238
export interface IExpression {
3339
Statement: string
3440
ExpressionId: string
@@ -74,6 +80,7 @@ export interface IColumn {
7480
Id: string
7581
AutoGen: AutoGen
7682
DefaultValue: IDefaultValue
83+
GeneratedColumn: IGeneratedColumn
7784
}
7885

7986
export interface IIgnored {

0 commit comments

Comments
 (0)