diff --git a/memory/index.go b/memory/index.go index 9ca5f379b5..cd93dab996 100644 --- a/memory/index.go +++ b/memory/index.go @@ -137,6 +137,10 @@ func (idx *Index) PrefixLengths() []uint16 { return idx.PrefixLens } +func (idx *Index) SetPrefixLengths(prefixLengths []uint16) { + idx.PrefixLens = prefixLengths +} + func (idx *Index) IndexType() string { if len(idx.DriverName) > 0 { return idx.DriverName diff --git a/sql/analyzer/analyzer.go b/sql/analyzer/analyzer.go index 8a2834ecbb..6df7040071 100644 --- a/sql/analyzer/analyzer.go +++ b/sql/analyzer/analyzer.go @@ -274,6 +274,7 @@ func (ab *Builder) Build() *Analyzer { Parallelism: ab.parallelism, Coster: memo.NewDefaultCoster(), ExecBuilder: rowexec.DefaultBuilder, + EngineType: sql.EngineType_MySql, } } @@ -298,6 +299,9 @@ type Analyzer struct { // EventScheduler is used to communiate with the event scheduler // for any EVENT related statements. It can be nil if EventScheduler is not defined. EventScheduler sql.EventScheduler + // EngineType indicates whether the analyzer's behavior is compatible with a MySQL + // database or a PostgreSQL database. + EngineType sql.EngineType } // NewDefault creates a default Analyzer instance with all default Rules and configuration. diff --git a/sql/analyzer/index_analyzer_test.go b/sql/analyzer/index_analyzer_test.go index 0cd693cb7e..386cf40d1c 100644 --- a/sql/analyzer/index_analyzer_test.go +++ b/sql/analyzer/index_analyzer_test.go @@ -152,8 +152,9 @@ func (i *dummyIdx) Comment() string { return "" } func (i *dummyIdx) IsGenerated() bool { return false } func (i *dummyIdx) CanSupportOrderBy(sql.Expression) bool { return false } -func (i *dummyIdx) IndexType() string { return "BTREE" } -func (i *dummyIdx) PrefixLengths() []uint16 { return nil } +func (i *dummyIdx) IndexType() string { return "BTREE" } +func (i *dummyIdx) PrefixLengths() []uint16 { return nil } +func (i dummyIdx) SetPrefixLengths(uint16s []uint16) {} func (i *dummyIdx) NewLookup(*sql.Context, ...sql.Range) (sql.IndexLookup, error) { panic("not implemented") diff --git a/sql/analyzer/validate_create_table.go b/sql/analyzer/validate_create_table.go index e1031f4d70..51bc1b252f 100644 --- a/sql/analyzer/validate_create_table.go +++ b/sql/analyzer/validate_create_table.go @@ -26,6 +26,10 @@ import ( const MaxBytePrefix = 3072 +// defaultPrefixLength is used for PostgreSQL compatibility. We don't support secondary indexes including address +// encoded types such as TEXT/BLOB, so we must apply an implicit prefix length. +const defaultPrefixLength = 200 + // validateCreateTable validates various constraints about CREATE TABLE statements. func validateCreateTable(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { ct, ok := n.(*plan.CreateTable) @@ -44,7 +48,11 @@ func validateCreateTable(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan. if err != nil { return nil, transform.SameTree, err } - err = validateIndexes(ctx, sch, idxs, strictMySQLCompat) + + // For MySQL compatibility, users must specify a prefix length for any TEXT or BLOB columns used in an index, + // otherwise we will apply an implicit prefix length. + validatePrefixLengths := a.EngineType == sql.EngineType_MySql + err = validateIndexes(ctx, sch, idxs, strictMySQLCompat, validatePrefixLengths) if err != nil { return nil, transform.SameTree, err } @@ -224,9 +232,12 @@ func resolveAlterColumn(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.S sch = sch.Copy() // Make a copy of the original schema to deal with any references to the original table. initialSch := sch - addedColumn := false + // For MySQL compatibility, users must specify a prefix length for any TEXT or BLOB columns used in an index, + // otherwise we will apply an implicit prefix length. + validatePrefixLengths := a.EngineType == sql.EngineType_MySql + // Need a TransformUp here because multiple of these statement types can be nested under a Block node. // It doesn't look it, but this is actually an iterative loop over all the independent clauses in an ALTER statement n, same, err := transform.Node(n, func(n sql.Node) (sql.Node, transform.TreeIdentity, error) { @@ -237,7 +248,7 @@ func resolveAlterColumn(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.S return nil, transform.SameTree, err } - sch, err = validateModifyColumn(ctx, initialSch, sch, n.(*plan.ModifyColumn), keyedColumns) + sch, err = validateModifyColumn(ctx, initialSch, sch, n.(*plan.ModifyColumn), keyedColumns, validatePrefixLengths) if err != nil { return nil, transform.SameTree, err } @@ -282,7 +293,7 @@ func resolveAlterColumn(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.S if err != nil { return nil, transform.SameTree, err } - indexes, err = validateAlterIndex(ctx, initialSch, sch, n.(*plan.AlterIndex), indexes) + indexes, err = validateAlterIndex(ctx, sch, n.(*plan.AlterIndex), indexes, validatePrefixLengths) if err != nil { return nil, transform.SameTree, err } @@ -294,7 +305,7 @@ func resolveAlterColumn(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.S if err != nil { return nil, transform.SameTree, err } - sch, err = validatePrimaryKey(ctx, initialSch, sch, n.(*plan.AlterPK)) + sch, err = validatePrimaryKey(ctx, sch, n.(*plan.AlterPK), validatePrefixLengths) if err != nil { return nil, transform.SameTree, err } @@ -446,7 +457,9 @@ func isStrictMysqlCompatibilityEnabled(ctx *sql.Context) (bool, error) { return i == 1, nil } -func validateModifyColumn(ctx *sql.Context, initialSch sql.Schema, schema sql.Schema, mc *plan.ModifyColumn, keyedColumns map[string]bool) (sql.Schema, error) { +// validateModifyColumn validates the |mc| ModifyColumn node and returns the updated schema. If +// |validatePrefixLengths| is true, then prefix lengths on TEXT/BLOB columns will also be validated. +func validateModifyColumn(ctx *sql.Context, initialSch sql.Schema, schema sql.Schema, mc *plan.ModifyColumn, keyedColumns map[string]bool, validatePrefixLengths bool) (sql.Schema, error) { table := mc.Table tableName := table.(sql.Nameable).Name() @@ -496,7 +509,6 @@ func validateModifyColumn(ctx *sql.Context, initialSch sql.Schema, schema sql.Sc if index.IsFullText() { continue } - prefixLengths := index.PrefixLengths() for i, expr := range index.Expressions() { col := plan.GetColumnFromIndexExpr(expr, tbl) if !strings.EqualFold(col.Name, oldColName) { @@ -505,14 +517,35 @@ func validateModifyColumn(ctx *sql.Context, initialSch sql.Schema, schema sql.Sc if types.IsJSON(newCol.Type) { return nil, sql.ErrJSONIndex.New(col.Name) } - var prefixLen int64 - if i < len(prefixLengths) { - prefixLen = int64(prefixLengths[i]) + + if validatePrefixLengths { + var prefixLen int64 + if i < len(index.PrefixLengths()) { + prefixLen = int64(index.PrefixLengths()[i]) + } + err = validatePrefixLength(ctx, oldColName, prefixLen, newCol.Type, strictMySQLCompat, index.IsUnique()) + if err != nil { + return nil, err + } } - err = validatePrefixLength(ctx, oldColName, prefixLen, newCol.Type, strictMySQLCompat, index.IsUnique()) - if err != nil { - return nil, err + } + + // If we aren't validating user-specified prefix lengths, then apply an implicit prefix length for TEXT columns + if !validatePrefixLengths { + prefixLengths := make([]uint16, len(index.Expressions())) + for i, expr := range index.Expressions() { + col := plan.GetColumnFromIndexExpr(expr, tbl) + if !strings.EqualFold(col.Name, oldColName) { + continue + } + if types.IsJSON(newCol.Type) { + return nil, sql.ErrJSONIndex.New(col.Name) + } + if types.IsText(newCol.Type) { + prefixLengths[i] = defaultPrefixLength + } } + index.SetPrefixLengths(prefixLengths) } } @@ -619,8 +652,9 @@ func validateColumnSafeToDropWithCheckConstraint(columnName string, checks sql.C } // validateAlterIndex validates the specified column can have an index added, dropped, or renamed. Returns an updated -// list of index name given the add, drop, or rename operations. -func validateAlterIndex(ctx *sql.Context, initialSch, sch sql.Schema, ai *plan.AlterIndex, indexes []string) ([]string, error) { +// list of index name given the add, drop, or rename operations. If |validatePrefixLengths| is true, then index prefix +// lengths on TEXT and BLOB columns will also be validated. +func validateAlterIndex(ctx *sql.Context, sch sql.Schema, ai *plan.AlterIndex, indexes []string, validatePrefixLengths bool) ([]string, error) { switch ai.Action { case plan.IndexAction_Create: err := validateIdentifier(ai.IndexName) @@ -640,7 +674,7 @@ func validateAlterIndex(ctx *sql.Context, initialSch, sch sql.Schema, ai *plan.A Storage: ai.Using, Comment: ai.Comment, } - err = validateIndex(ctx, colMap, indexDef, strictMySQLCompat) + err = validateIndex(ctx, colMap, indexDef, strictMySQLCompat, validatePrefixLengths) if err != nil { return nil, err } @@ -839,14 +873,14 @@ func schToColMap(sch sql.Schema) map[string]*sql.Column { } // validateIndexes prevents creating tables with blob/text primary keys and indexes without a specified length -func validateIndexes(ctx *sql.Context, sch sql.Schema, idxDefs sql.IndexDefs, strictMySQLCompat bool) error { +func validateIndexes(ctx *sql.Context, sch sql.Schema, idxDefs sql.IndexDefs, strictMySQLCompat bool, validatePrefixLengths bool) error { colMap := schToColMap(sch) var hasPkIndexDef bool for _, idxDef := range idxDefs { if idxDef.IsPrimary() { hasPkIndexDef = true } - if err := validateIndex(ctx, colMap, idxDef, strictMySQLCompat); err != nil { + if err := validateIndex(ctx, colMap, idxDef, strictMySQLCompat, validatePrefixLengths); err != nil { return err } } @@ -865,13 +899,14 @@ func validateIndexes(ctx *sql.Context, sch sql.Schema, idxDefs sql.IndexDefs, st } // validateIndex ensures that the Index Definition is valid for the table schema. -// This function will throw errors and warnings as needed. -// All columns in the index must be: +// This function will throw errors and warnings as needed. If |validatePrefixLengths| is +// true, then TEXT and BLOB columns included in the index will also be checked to ensure +// they include a valid prefix length. All columns in the index must be: // - in the schema // - not duplicated // - not JSON Type // - have the proper prefix length -func validateIndex(ctx *sql.Context, colMap map[string]*sql.Column, idxDef *sql.IndexDef, strictMySQLCompat bool) error { +func validateIndex(ctx *sql.Context, colMap map[string]*sql.Column, idxDef *sql.IndexDef, strictMySQLCompat, validatePrefixLengths bool) error { seenCols := make(map[string]struct{}) for _, idxCol := range idxDef.Columns { schCol, exists := colMap[strings.ToLower(idxCol.Name)] @@ -890,9 +925,30 @@ func validateIndex(ctx *sql.Context, colMap map[string]*sql.Column, idxDef *sql. continue } - err := validatePrefixLength(ctx, idxCol.Name, idxCol.Length, schCol.Type, strictMySQLCompat, idxDef.IsUnique()) - if err != nil { - return err + if validatePrefixLengths { + err := validatePrefixLength(ctx, idxCol.Name, idxCol.Length, schCol.Type, strictMySQLCompat, idxDef.IsUnique()) + if err != nil { + return err + } + } + } + + // If we aren't validating user-specified prefix lengths, then we need to apply implicit prefix lengths for + // any TEXT columns. + if !validatePrefixLengths { + prefixLengths := make([]uint16, len(idxDef.Columns)) + for i, idxCol := range idxDef.Columns { + schCol, exists := colMap[strings.ToLower(idxCol.Name)] + if !exists { + return sql.ErrKeyColumnDoesNotExist.New(idxCol.Name) + } + if types.IsJSON(schCol.Type) { + return sql.ErrJSONIndex.New(schCol.Name) + } + if types.IsText(schCol.Type) { + prefixLengths[i] = defaultPrefixLength + } + idxDef.Columns[i].Length = int64(prefixLengths[i]) } } @@ -957,8 +1013,9 @@ func getTableIndexNames(ctx *sql.Context, _ *Analyzer, table sql.Node) ([]string return names, nil } -// validatePrimaryKey validates a primary key add or drop operation. -func validatePrimaryKey(ctx *sql.Context, initialSch, sch sql.Schema, ai *plan.AlterPK) (sql.Schema, error) { +// validatePrimaryKey validates a primary key add or drop operation. If |validatePrefixLengths| is true, then +// TEXT and BLOB columns will be checked for a valid prefix length. +func validatePrimaryKey(ctx *sql.Context, sch sql.Schema, ai *plan.AlterPK, validatePrefixLengths bool) (sql.Schema, error) { tableName := getTableName(ai.Table) switch ai.Action { case plan.PrimaryKeyAction_Create: @@ -976,7 +1033,7 @@ func validatePrimaryKey(ctx *sql.Context, initialSch, sch sql.Schema, ai *plan.A if err != nil { return nil, err } - err = validateIndex(ctx, colMap, idxDef, strictMySQLCompat) + err = validateIndex(ctx, colMap, idxDef, strictMySQLCompat, validatePrefixLengths) if err != nil { return nil, err } diff --git a/sql/engines.go b/sql/engines.go index 6b4d61315f..a49d0c438f 100644 --- a/sql/engines.go +++ b/sql/engines.go @@ -16,6 +16,15 @@ package sql import "fmt" +// EngineType indicates which type of database is being emulated. This is used to switch +// between MySQL behavior and PostgreSQL behavior. +type EngineType byte + +const ( + EngineType_MySql EngineType = iota + EngineType_Postgres +) + // Engine represents a sql engine. type Engine struct { Name string diff --git a/sql/index.go b/sql/index.go index 7b985d2012..c1eeb2d722 100644 --- a/sql/index.go +++ b/sql/index.go @@ -126,8 +126,10 @@ type Index interface { // Verifying that the expression's children match the index columns are done separately. CanSupportOrderBy(expr Expression) bool - // PrefixLengths returns the prefix lengths for each column in this index + // PrefixLengths returns the prefix lengths for each column in this index. PrefixLengths() []uint16 + // SetPrefixLengths sets the prefix lengths for each column in this index. + SetPrefixLengths([]uint16) } // ExtendedIndex is an extension of Index, that allows access to appended primary keys. MySQL internally represents an diff --git a/sql/index_builder_test.go b/sql/index_builder_test.go index 3d91c112b2..4addd506f9 100644 --- a/sql/index_builder_test.go +++ b/sql/index_builder_test.go @@ -225,4 +225,6 @@ func (testIndex) PrefixLengths() []uint16 { return nil } +func (i testIndex) SetPrefixLengths(uint16s []uint16) {} + var _ sql.Index = testIndex{} diff --git a/sql/index_registry_test.go b/sql/index_registry_test.go index ad8704c126..e05defd600 100644 --- a/sql/index_registry_test.go +++ b/sql/index_registry_test.go @@ -443,17 +443,18 @@ func (i dummyIdx) Expressions() []string { } return exprs } -func (i dummyIdx) ID() string { return i.id } -func (i dummyIdx) Database() string { return i.database } -func (i dummyIdx) Table() string { return i.table } -func (i dummyIdx) Driver() string { return "dummy" } -func (i dummyIdx) IsUnique() bool { return false } -func (i dummyIdx) IsSpatial() bool { return false } -func (i dummyIdx) IsFullText() bool { return false } -func (i dummyIdx) Comment() string { return "" } -func (i dummyIdx) IsGenerated() bool { return false } -func (i dummyIdx) IndexType() string { return "BTREE" } -func (i dummyIdx) PrefixLengths() []uint16 { return nil } +func (i dummyIdx) ID() string { return i.id } +func (i dummyIdx) Database() string { return i.database } +func (i dummyIdx) Table() string { return i.table } +func (i dummyIdx) Driver() string { return "dummy" } +func (i dummyIdx) IsUnique() bool { return false } +func (i dummyIdx) IsSpatial() bool { return false } +func (i dummyIdx) IsFullText() bool { return false } +func (i dummyIdx) Comment() string { return "" } +func (i dummyIdx) IsGenerated() bool { return false } +func (i dummyIdx) IndexType() string { return "BTREE" } +func (i dummyIdx) PrefixLengths() []uint16 { return nil } +func (i dummyIdx) SetPrefixLengths(uint16s []uint16) {} func (i dummyIdx) NewLookup(ctx *Context, ranges ...Range) (IndexLookup, error) { panic("not implemented") diff --git a/sql/memo/rel_props_test.go b/sql/memo/rel_props_test.go index e834df876b..2f032c5789 100644 --- a/sql/memo/rel_props_test.go +++ b/sql/memo/rel_props_test.go @@ -257,5 +257,6 @@ func (i dummyIndex) ColumnExpressionTypes() []sql.ColumnExpressionType { func (dummyIndex) PrefixLengths() []uint16 { return nil } +func (i dummyIndex) SetPrefixLengths(uint16s []uint16) {} var _ sql.Index = dummyIndex{}