Skip to content

Commit 80102dd

Browse files
author
Shlomi Noach
authored
Merge pull request #595 from github/support-generated-columns
Support for GENERATED (aka virtual) columns
2 parents 582e44a + b931204 commit 80102dd

File tree

17 files changed

+134
-22
lines changed

17 files changed

+134
-22
lines changed

‎doc/requirements-and-limitations.md

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,6 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
2626

2727
- Triggers are not supported. They may be supported in the future.
2828

29-
- MySQL 5.7 generated columns are not supported. They may be supported in the future.
30-
31-
- MySQL 5.7 `POINT` column type is not supported.
32-
3329
- MySQL 5.7 `JSON` columns are supported but not as part of `PRIMARY KEY`
3430

3531
- The two _before_ & _after_ tables must share a `PRIMARY KEY` or other `UNIQUE KEY`. This key will be used by `gh-ost` to iterate through the table rows when copying. [Read more](shared-key.md)
@@ -53,4 +49,4 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
5349

5450
- Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles.
5551

56-
- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation).
52+
- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation).

‎go/base/context.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,8 +186,10 @@ type MigrationContext struct {
186186

187187
OriginalTableColumnsOnApplier *sql.ColumnList
188188
OriginalTableColumns *sql.ColumnList
189+
OriginalTableVirtualColumns *sql.ColumnList
189190
OriginalTableUniqueKeys [](*sql.UniqueKey)
190191
GhostTableColumns *sql.ColumnList
192+
GhostTableVirtualColumns *sql.ColumnList
191193
GhostTableUniqueKeys [](*sql.UniqueKey)
192194
UniqueKey *sql.UniqueKey
193195
SharedColumns *sql.ColumnList

‎go/logic/applier.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ func (this *Applier) validateAndReadTimeZone() error {
117117
// readTableColumns reads table columns on applier
118118
func (this *Applier) readTableColumns() (err error) {
119119
log.Infof("Examining table structure on applier")
120-
this.migrationContext.OriginalTableColumnsOnApplier, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName)
120+
this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName)
121121
if err != nil {
122122
return err
123123
}

‎go/logic/inspect.go

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,24 +89,24 @@ func (this *Inspector) ValidateOriginalTable() (err error) {
8989
return nil
9090
}
9191

92-
func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) {
92+
func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, virtualColumns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) {
9393
uniqueKeys, err = this.getCandidateUniqueKeys(tableName)
9494
if err != nil {
95-
return columns, uniqueKeys, err
95+
return columns, virtualColumns, uniqueKeys, err
9696
}
9797
if len(uniqueKeys) == 0 {
98-
return columns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out")
98+
return columns, virtualColumns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out")
9999
}
100-
columns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName)
100+
columns, virtualColumns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName)
101101
if err != nil {
102-
return columns, uniqueKeys, err
102+
return columns, virtualColumns, uniqueKeys, err
103103
}
104104

105-
return columns, uniqueKeys, nil
105+
return columns, virtualColumns, uniqueKeys, nil
106106
}
107107

108108
func (this *Inspector) InspectOriginalTable() (err error) {
109-
this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName)
109+
this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName)
110110
if err != nil {
111111
return err
112112
}
@@ -122,7 +122,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
122122
return fmt.Errorf("It seems like table structure is not identical between master and replica. This scenario is not supported.")
123123
}
124124

125-
this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName())
125+
this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName())
126126
if err != nil {
127127
return err
128128
}
@@ -166,7 +166,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
166166
}
167167
}
168168

169-
this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.ColumnRenameMap)
169+
this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap)
170170
log.Infof("Shared columns are %s", this.migrationContext.SharedColumns)
171171
// By fact that a non-empty unique key exists we also know the shared columns are non-empty
172172

@@ -692,7 +692,7 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [
692692
}
693693

694694
// getSharedColumns returns the intersection of two lists of columns in same order as the first list
695-
func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) {
695+
func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, originalVirtualColumns, ghostVirtualColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) {
696696
sharedColumnNames := []string{}
697697
for _, originalColumn := range originalColumns.Names() {
698698
isSharedColumn := false
@@ -709,6 +709,16 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum
709709
isSharedColumn = false
710710
}
711711
}
712+
for _, virtualColumn := range originalVirtualColumns.Names() {
713+
if strings.EqualFold(originalColumn, virtualColumn) {
714+
isSharedColumn = false
715+
}
716+
}
717+
for _, virtualColumn := range ghostVirtualColumns.Names() {
718+
if strings.EqualFold(originalColumn, virtualColumn) {
719+
isSharedColumn = false
720+
}
721+
}
712722
if isSharedColumn {
713723
sharedColumnNames = append(sharedColumnNames, originalColumn)
714724
}

‎go/mysql/utils.go

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ package mysql
88
import (
99
gosql "database/sql"
1010
"fmt"
11+
"strings"
1112
"sync"
1213
"time"
1314

@@ -178,26 +179,32 @@ func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) {
178179
}
179180

180181
// GetTableColumns reads column list from given table
181-
func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, error) {
182+
func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, *sql.ColumnList, error) {
182183
query := fmt.Sprintf(`
183184
show columns from %s.%s
184185
`,
185186
sql.EscapeName(databaseName),
186187
sql.EscapeName(tableName),
187188
)
188189
columnNames := []string{}
190+
virtualColumnNames := []string{}
189191
err := sqlutils.QueryRowsMap(db, query, func(rowMap sqlutils.RowMap) error {
190-
columnNames = append(columnNames, rowMap.GetString("Field"))
192+
columnName := rowMap.GetString("Field")
193+
columnNames = append(columnNames, columnName)
194+
if strings.Contains(rowMap.GetString("Extra"), " GENERATED") {
195+
log.Debugf("%s is a generated column", columnName)
196+
virtualColumnNames = append(virtualColumnNames, columnName)
197+
}
191198
return nil
192199
})
193200
if err != nil {
194-
return nil, err
201+
return nil, nil, err
195202
}
196203
if len(columnNames) == 0 {
197-
return nil, log.Errorf("Found 0 columns on %s.%s. Bailing out",
204+
return nil, nil, log.Errorf("Found 0 columns on %s.%s. Bailing out",
198205
sql.EscapeName(databaseName),
199206
sql.EscapeName(tableName),
200207
)
201208
}
202-
return sql.NewColumnList(columnNames), nil
209+
return sql.NewColumnList(columnNames), sql.NewColumnList(virtualColumnNames), nil
203210
}

‎localtests/generated-columns-add57/create.sql

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
drop table if exists gh_ost_test;
2+
create table gh_ost_test (
3+
id int auto_increment,
4+
a int not null,
5+
b int not null,
6+
primary key(id)
7+
) auto_increment=1;
8+
9+
drop event if exists gh_ost_test;
10+
delimiter ;;
11+
create event gh_ost_test
12+
on schedule every 1 second
13+
starts current_timestamp
14+
ends current_timestamp + interval 60 second
15+
on completion not preserve
16+
enable
17+
do
18+
begin
19+
insert into gh_ost_test (id, a, b) values (null, 2,3);
20+
insert into gh_ost_test (id, a, b) values (null, 2,4);
21+
insert into gh_ost_test (id, a, b) values (null, 2,5);
22+
insert into gh_ost_test (id, a, b) values (null, 2,6);
23+
insert into gh_ost_test (id, a, b) values (null, 2,7);
24+
insert into gh_ost_test (id, a, b) values (null, 2,8);
25+
insert into gh_ost_test (id, a, b) values (null, 2,9);
26+
insert into gh_ost_test (id, a, b) values (null, 2,0);
27+
insert into gh_ost_test (id, a, b) values (null, 2,1);
28+
insert into gh_ost_test (id, a, b) values (null, 2,2);
29+
end ;;

‎localtests/generated-columns-add57/extra_args

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
--alter="add column sum_ab int as (a + b) virtual not null"

‎localtests/generated-columns-add57/ghost_columns

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
id, a, b

‎localtests/generated-columns-add57/ignore_versions

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
(5.5|5.6)

‎localtests/generated-columns-add57/order_by

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
id

0 commit comments

Comments
 (0)