Skip to content

Commit 7443a09

Browse files
fulghumfulghum
committed
Applying implicit prefix lengths for TEXT columns in secondary indexes
1 parent baafd97 commit 7443a09

File tree

8 files changed

+77
-20
lines changed

8 files changed

+77
-20
lines changed

‎memory/index.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,10 @@ func (idx *Index) PrefixLengths() []uint16 {
137137
return idx.PrefixLens
138138
}
139139

140+
func (idx *Index) SetPrefixLengths(prefixLengths []uint16) {
141+
idx.PrefixLens = prefixLengths
142+
}
143+
140144
func (idx *Index) IndexType() string {
141145
if len(idx.DriverName) > 0 {
142146
return idx.DriverName

‎sql/analyzer/analyzer.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ func (ab *Builder) Build() *Analyzer {
274274
Parallelism: ab.parallelism,
275275
Coster: memo.NewDefaultCoster(),
276276
ExecBuilder: rowexec.DefaultBuilder,
277-
DatabaseType: sql.EngineType_MySql,
277+
EngineType: sql.EngineType_MySql,
278278
}
279279
}
280280

@@ -299,9 +299,9 @@ type Analyzer struct {
299299
// EventScheduler is used to communiate with the event scheduler
300300
// for any EVENT related statements. It can be nil if EventScheduler is not defined.
301301
EventScheduler sql.EventScheduler
302-
// DatabaseType indicates whether the analyzer's behavior is compatible with a MySQL
302+
// EngineType indicates whether the analyzer's behavior is compatible with a MySQL
303303
// database or a PostgreSQL database.
304-
DatabaseType sql.EngineType
304+
EngineType sql.EngineType
305305
}
306306

307307
// NewDefault creates a default Analyzer instance with all default Rules and configuration.

‎sql/analyzer/index_analyzer_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,9 @@ func (i *dummyIdx) Comment() string { return "" }
152152
func (i *dummyIdx) IsGenerated() bool { return false }
153153
func (i *dummyIdx) CanSupportOrderBy(sql.Expression) bool { return false }
154154

155-
func (i *dummyIdx) IndexType() string { return "BTREE" }
156-
func (i *dummyIdx) PrefixLengths() []uint16 { return nil }
155+
func (i *dummyIdx) IndexType() string { return "BTREE" }
156+
func (i *dummyIdx) PrefixLengths() []uint16 { return nil }
157+
func (i dummyIdx) SetPrefixLengths(uint16s []uint16) {}
157158

158159
func (i *dummyIdx) NewLookup(*sql.Context, ...sql.Range) (sql.IndexLookup, error) {
159160
panic("not implemented")

‎sql/analyzer/validate_create_table.go‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ import (
2626

2727
const MaxBytePrefix = 3072
2828

29+
// defaultPrefixLength is used for PostgreSQL compatibility. We don't support secondary indexes including address
30+
// encoded types such as TEXT/BLOB, so we must apply an implicit prefix length.
31+
const defaultPrefixLength = 200
32+
2933
// validateCreateTable validates various constraints about CREATE TABLE statements.
3034
func validateCreateTable(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) {
3135
ct, ok := n.(*plan.CreateTable)
@@ -44,7 +48,10 @@ func validateCreateTable(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.
4448
if err != nil {
4549
return nil, transform.SameTree, err
4650
}
47-
validatePrefixLengths := a.DatabaseType == sql.EngineType_MySql
51+
52+
// For MySQL compatibility, users must specify a prefix length for any TEXT or BLOB columns used in an index,
53+
// otherwise we will apply an implicit prefix length.
54+
validatePrefixLengths := a.EngineType == sql.EngineType_MySql
4855
err = validateIndexes(ctx, sch, idxs, strictMySQLCompat, validatePrefixLengths)
4956
if err != nil {
5057
return nil, transform.SameTree, err
@@ -225,9 +232,11 @@ func resolveAlterColumn(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.S
225232

226233
sch = sch.Copy() // Make a copy of the original schema to deal with any references to the original table.
227234
initialSch := sch
228-
229235
addedColumn := false
230-
validatePrefixLengths := a.DatabaseType == sql.EngineType_MySql
236+
237+
// For MySQL compatibility, users must specify a prefix length for any TEXT or BLOB columns used in an index,
238+
// otherwise we will apply an implicit prefix length.
239+
validatePrefixLengths := a.EngineType == sql.EngineType_MySql
231240

232241
// Need a TransformUp here because multiple of these statement types can be nested under a Block node.
233242
// It doesn't look it, but this is actually an iterative loop over all the independent clauses in an ALTER statement
@@ -520,6 +529,24 @@ func validateModifyColumn(ctx *sql.Context, initialSch sql.Schema, schema sql.Sc
520529
}
521530
}
522531
}
532+
533+
// If we aren't validating user-specified prefix lengths, then apply an implicit prefix length for TEXT columns
534+
if !validatePrefixLengths {
535+
prefixLengths := make([]uint16, len(index.Expressions()))
536+
for i, expr := range index.Expressions() {
537+
col := plan.GetColumnFromIndexExpr(expr, tbl)
538+
if !strings.EqualFold(col.Name, oldColName) {
539+
continue
540+
}
541+
if types.IsJSON(newCol.Type) {
542+
return nil, sql.ErrJSONIndex.New(col.Name)
543+
}
544+
if types.IsText(newCol.Type) {
545+
prefixLengths[i] = defaultPrefixLength
546+
}
547+
}
548+
index.SetPrefixLengths(prefixLengths)
549+
}
523550
}
524551

525552
return newSch, nil
@@ -906,6 +933,25 @@ func validateIndex(ctx *sql.Context, colMap map[string]*sql.Column, idxDef *sql.
906933
}
907934
}
908935

936+
// If we aren't validating user-specified prefix lengths, then we need to apply implicit prefix lengths for
937+
// any TEXT columns.
938+
if !validatePrefixLengths {
939+
prefixLengths := make([]uint16, len(idxDef.Columns))
940+
for i, idxCol := range idxDef.Columns {
941+
schCol, exists := colMap[strings.ToLower(idxCol.Name)]
942+
if !exists {
943+
return sql.ErrKeyColumnDoesNotExist.New(idxCol.Name)
944+
}
945+
if types.IsJSON(schCol.Type) {
946+
return sql.ErrJSONIndex.New(schCol.Name)
947+
}
948+
if types.IsText(schCol.Type) {
949+
prefixLengths[i] = defaultPrefixLength
950+
}
951+
idxDef.Columns[i].Length = int64(prefixLengths[i])
952+
}
953+
}
954+
909955
if idxDef.IsSpatial() {
910956
if len(idxDef.Columns) != 1 {
911957
return sql.ErrTooManyKeyParts.New(1)

‎sql/index.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,10 @@ type Index interface {
126126
// Verifying that the expression's children match the index columns are done separately.
127127
CanSupportOrderBy(expr Expression) bool
128128

129-
// PrefixLengths returns the prefix lengths for each column in this index
129+
// PrefixLengths returns the prefix lengths for each column in this index.
130130
PrefixLengths() []uint16
131+
// SetPrefixLengths sets the prefix lengths for each column in this index.
132+
SetPrefixLengths([]uint16)
131133
}
132134

133135
// ExtendedIndex is an extension of Index, that allows access to appended primary keys. MySQL internally represents an

‎sql/index_builder_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,4 +225,6 @@ func (testIndex) PrefixLengths() []uint16 {
225225
return nil
226226
}
227227

228+
func (i testIndex) SetPrefixLengths(uint16s []uint16) {}
229+
228230
var _ sql.Index = testIndex{}

‎sql/index_registry_test.go‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -443,17 +443,18 @@ func (i dummyIdx) Expressions() []string {
443443
}
444444
return exprs
445445
}
446-
func (i dummyIdx) ID() string { return i.id }
447-
func (i dummyIdx) Database() string { return i.database }
448-
func (i dummyIdx) Table() string { return i.table }
449-
func (i dummyIdx) Driver() string { return "dummy" }
450-
func (i dummyIdx) IsUnique() bool { return false }
451-
func (i dummyIdx) IsSpatial() bool { return false }
452-
func (i dummyIdx) IsFullText() bool { return false }
453-
func (i dummyIdx) Comment() string { return "" }
454-
func (i dummyIdx) IsGenerated() bool { return false }
455-
func (i dummyIdx) IndexType() string { return "BTREE" }
456-
func (i dummyIdx) PrefixLengths() []uint16 { return nil }
446+
func (i dummyIdx) ID() string { return i.id }
447+
func (i dummyIdx) Database() string { return i.database }
448+
func (i dummyIdx) Table() string { return i.table }
449+
func (i dummyIdx) Driver() string { return "dummy" }
450+
func (i dummyIdx) IsUnique() bool { return false }
451+
func (i dummyIdx) IsSpatial() bool { return false }
452+
func (i dummyIdx) IsFullText() bool { return false }
453+
func (i dummyIdx) Comment() string { return "" }
454+
func (i dummyIdx) IsGenerated() bool { return false }
455+
func (i dummyIdx) IndexType() string { return "BTREE" }
456+
func (i dummyIdx) PrefixLengths() []uint16 { return nil }
457+
func (i dummyIdx) SetPrefixLengths(uint16s []uint16) {}
457458

458459
func (i dummyIdx) NewLookup(ctx *Context, ranges ...Range) (IndexLookup, error) {
459460
panic("not implemented")

‎sql/memo/rel_props_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,5 +257,6 @@ func (i dummyIndex) ColumnExpressionTypes() []sql.ColumnExpressionType {
257257
func (dummyIndex) PrefixLengths() []uint16 {
258258
return nil
259259
}
260+
func (i dummyIndex) SetPrefixLengths(uint16s []uint16) {}
260261

261262
var _ sql.Index = dummyIndex{}

0 commit comments

Comments
 (0)