Skip to content

Commit e9cb4a2

Browse files
wangzihuacoolwangzihuacool
committed
convert to bytes if character string without charsetConversion.
1 parent a17389e commit e9cb4a2

File tree

3 files changed

+37
-17
lines changed

3 files changed

+37
-17
lines changed

‎go/logic/inspect.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,9 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
192192
this.migrationContext.MappedSharedColumns.SetEnumToTextConversion(column.Name)
193193
this.migrationContext.MappedSharedColumns.SetEnumValues(column.Name, column.EnumValues)
194194
}
195+
if column.Name == mappedColumn.Name && column.Charset != mappedColumn.Charset {
196+
this.migrationContext.SharedColumns.SetCharsetConversion(column.Name, column.Charset, mappedColumn.Charset)
197+
}
195198
}
196199

197200
for _, column := range this.migrationContext.UniqueKey.Columns.Columns() {

‎go/sql/builder_test.go

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,7 @@ func TestBuildDMLDeleteQuery(t *testing.T) {
371371
((name = ?) and (position = ?))
372372
`
373373
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
374-
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{[]byte("testname"), 17}))
374+
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{"testname", 17}))
375375
}
376376
{
377377
uniqueKeyColumns := NewColumnList([]string{"position", "name"})
@@ -386,7 +386,7 @@ func TestBuildDMLDeleteQuery(t *testing.T) {
386386
((position = ?) and (name = ?))
387387
`
388388
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
389-
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, []byte("testname")}))
389+
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, "testname"}))
390390
}
391391
{
392392
uniqueKeyColumns := NewColumnList([]string{"position", "name"})
@@ -452,7 +452,7 @@ func TestBuildDMLInsertQuery(t *testing.T) {
452452
(?, ?, ?, ?)
453453
`
454454
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
455-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23}))
455+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
456456
}
457457
{
458458
sharedColumns := NewColumnList([]string{"position", "name", "age", "id"})
@@ -466,7 +466,7 @@ func TestBuildDMLInsertQuery(t *testing.T) {
466466
(?, ?, ?, ?)
467467
`
468468
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
469-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{17, []byte("testname"), 23, 3}))
469+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{17, "testname", 23, 3}))
470470
}
471471
{
472472
sharedColumns := NewColumnList([]string{"position", "name", "surprise", "id"})
@@ -499,7 +499,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
499499
(?, ?, ?, ?)
500500
`
501501
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
502-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), int8(-1), 23}))
502+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", int8(-1), 23}))
503503
}
504504
{
505505
// testing unsigned
@@ -515,7 +515,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
515515
(?, ?, ?, ?)
516516
`
517517
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
518-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), uint8(255), 23}))
518+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", uint8(255), 23}))
519519
}
520520
{
521521
// testing unsigned
@@ -531,7 +531,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
531531
(?, ?, ?, ?)
532532
`
533533
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
534-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), uint32(4294967295), 23}))
534+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", uint32(4294967295), 23}))
535535
}
536536
}
537537

@@ -554,7 +554,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
554554
((position = ?))
555555
`
556556
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
557-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23}))
557+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
558558
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17}))
559559
}
560560
{
@@ -570,8 +570,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
570570
((position = ?) and (name = ?))
571571
`
572572
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
573-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23}))
574-
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, []byte("testname")}))
573+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
574+
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, "testname"}))
575575
}
576576
{
577577
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
@@ -586,7 +586,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
586586
((age = ?))
587587
`
588588
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
589-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23}))
589+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
590590
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56}))
591591
}
592592
{
@@ -602,8 +602,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
602602
((age = ?) and (position = ?) and (id = ?) and (name = ?))
603603
`
604604
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
605-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23}))
606-
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56, 17, 3, []byte("testname")}))
605+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
606+
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56, 17, 3, "testname"}))
607607
}
608608
{
609609
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
@@ -631,7 +631,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
631631
((id = ?))
632632
`
633633
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
634-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23}))
634+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
635635
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{3}))
636636
}
637637
}
@@ -656,7 +656,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
656656
((position = ?))
657657
`
658658
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
659-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), int8(-17), int8(-2)}))
659+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", int8(-17), int8(-2)}))
660660
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{int8(-3)}))
661661
}
662662
{
@@ -673,7 +673,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
673673
((position = ?))
674674
`
675675
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
676-
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), int8(-17), uint8(254)}))
676+
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", int8(-17), uint8(254)}))
677677
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{uint8(253)}))
678678
}
679679
}

‎go/sql/types.go

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ type TimezoneConversion struct {
3232
ToTimezone string
3333
}
3434

35+
type CharacterSetConversion struct {
36+
ToCharset string
37+
FromCharset string
38+
}
39+
3540
type Column struct {
3641
Name string
3742
IsUnsigned bool
@@ -43,12 +48,20 @@ type Column struct {
4348
// add Octet length for binary type, fix bytes with suffix "00" get clipped in mysql binlog.
4449
// https://github.com/github/gh-ost/issues/909
4550
BinaryOctetLength uint
51+
charsetConversion *CharacterSetConversion
4652
}
4753

4854
func (this *Column) convertArg(arg interface{}, isUniqueKeyColumn bool) interface{} {
4955
if s, ok := arg.(string); ok {
5056
arg2Bytes := []byte(s)
51-
arg = arg2Bytes
57+
// convert to bytes if character string without charsetConversion.
58+
if this.Charset != "" && this.charsetConversion == nil {
59+
arg = arg2Bytes
60+
} else {
61+
if encoding, ok := charsetEncodingMap[this.Charset]; ok {
62+
arg, _ = encoding.NewDecoder().String(s)
63+
}
64+
}
5265

5366
if this.Type == BinaryColumnType && isUniqueKeyColumn {
5467
size := len(arg2Bytes)
@@ -235,6 +248,10 @@ func (this *ColumnList) Len() int {
235248
return len(this.columns)
236249
}
237250

251+
func (this *ColumnList) SetCharsetConversion(columnName string, fromCharset string, toCharset string) {
252+
this.GetColumn(columnName).charsetConversion = &CharacterSetConversion{FromCharset: fromCharset, ToCharset: toCharset}
253+
}
254+
238255
// UniqueKey is the combination of a key's name and columns
239256
type UniqueKey struct {
240257
Name string

0 commit comments

Comments
 (0)