Merge pull request #1713 from dolthub/fulghum/dolt-5735

Changing `MaxTextResponseByteLength()` to respect `character_set_results`

Author: fulghum

server/handler.go

@@ -114,7 +114,7 @@ func (h *Handler) ComPrepare(c *mysql.Conn, query string) ([]*query.Field, error
 	if types.IsOkResultSchema(analyzed.Schema()) {
 		return nil, nil
 	}
-	return schemaToFields(analyzed.Schema()), nil
+	return schemaToFields(ctx, analyzed.Schema()), nil
 }
 
 func (h *Handler) ComStmtExecute(c *mysql.Conn, prepare *mysql.PrepareData, callback func(*sqltypes.Result) error) error {
@@ -420,7 +420,7 @@ func (h *Handler) doQuery(
 		defer wg.Done()
 		for {
 			if r == nil {
-				r = &sqltypes.Result{Fields: schemaToFields(schema)}
+				r = &sqltypes.Result{Fields: schemaToFields(ctx, schema)}
 			}
 
 			if r.RowsAffected == rowsBatch {
@@ -691,7 +691,7 @@ func row2ToSQL(s sql.Schema, row sql.Row2) ([]sqltypes.Value, error) {
 	return o, nil
 }
 
-func schemaToFields(s sql.Schema) []*query.Field {
+func schemaToFields(ctx *sql.Context, s sql.Schema) []*query.Field {
 	fields := make([]*query.Field, len(s))
 	for i, c := range s {
 		var charset uint32 = mysql.CharacterSetUtf8
@@ -703,7 +703,7 @@ func schemaToFields(s sql.Schema) []*query.Field {
 			Name:         c.Name,
 			Type:         c.Type.Type(),
 			Charset:      charset,
-			ColumnLength: c.Type.MaxTextResponseByteLength(),
+			ColumnLength: c.Type.MaxTextResponseByteLength(ctx),
 		}
 	}
 

server/handler_test.go

@@ -35,10 +35,10 @@ import (
 	"github.com/dolthub/go-mysql-server/sql/analyzer"
 	"github.com/dolthub/go-mysql-server/sql/expression"
 	"github.com/dolthub/go-mysql-server/sql/types"
+	"github.com/dolthub/go-mysql-server/sql/variables"
 )
 
 func TestHandlerOutput(t *testing.T) {
-
 	e := setupMemDB(require.New(t))
 	dummyConn := newConn(1)
 	handler := &Handler{
@@ -655,14 +655,79 @@ func TestSchemaToFields(t *testing.T) {
 
 	require.Equal(len(schema), len(expected))
 
-	fields := schemaToFields(schema)
+	handler := &Handler{
+		e: setupMemDB(require),
+		sm: NewSessionManager(
+			testSessionBuilder,
+			sql.NoopTracer,
+			func(ctx *sql.Context, db string) bool { return db == "test" },
+			sql.NewMemoryManager(nil),
+			sqle.NewProcessList(),
+			"foo",
+		),
+		readTimeout: time.Second,
+	}
+
+	conn := newConn(1)
+	handler.NewConnection(conn)
+
+	ctx, err := handler.sm.NewContextWithQuery(conn, "SELECT 1")
+	require.NoError(err)
+
+	fields := schemaToFields(ctx, schema)
 	for i := 0; i < len(fields); i++ {
 		t.Run(schema[i].Name, func(t *testing.T) {
 			assert.Equal(t, expected[i], fields[i])
 		})
 	}
 }
 
+// TestHandlerMaxTextResponseBytes tests that the handler calculates the correct max text response byte
+// metadata for TEXT types, including honoring the character_set_results session variable. This is tested
+// here, instead of in string type unit tests, because of the dependency on system variables being loaded.
+func TestHandlerMaxTextResponseBytes(t *testing.T) {
+	variables.InitSystemVariables()
+	session := sql.NewBaseSession()
+	ctx := sql.NewContext(
+		context.Background(),
+		sql.WithSession(session),
+	)
+
+	tinyTextUtf8mb4 := types.MustCreateString(sqltypes.Text, types.TinyTextBlobMax, sql.Collation_Default)
+	textUtf8mb4 := types.MustCreateString(sqltypes.Text, types.TextBlobMax, sql.Collation_Default)
+	mediumTextUtf8mb4 := types.MustCreateString(sqltypes.Text, types.MediumTextBlobMax, sql.Collation_Default)
+	longTextUtf8mb4 := types.MustCreateString(sqltypes.Text, types.LongTextBlobMax, sql.Collation_Default)
+
+	// When character_set_results is set to utf8mb4, the multibyte character multiplier is 4
+	require.NoError(t, session.SetSessionVariable(ctx, "character_set_results", "utf8mb4"))
+	require.EqualValues(t, types.TinyTextBlobMax*4, tinyTextUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.TextBlobMax*4, textUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.MediumTextBlobMax*4, mediumTextUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.LongTextBlobMax, longTextUtf8mb4.MaxTextResponseByteLength(ctx))
+
+	// When character_set_results is set to utf8mb3, the multibyte character multiplier is 3
+	require.NoError(t, session.SetSessionVariable(ctx, "character_set_results", "utf8mb3"))
+	require.EqualValues(t, types.TinyTextBlobMax*3, tinyTextUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.TextBlobMax*3, textUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.MediumTextBlobMax*3, mediumTextUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.LongTextBlobMax, longTextUtf8mb4.MaxTextResponseByteLength(ctx))
+
+	// When character_set_results is set to utf8, the multibyte character multiplier is 3
+	require.NoError(t, session.SetSessionVariable(ctx, "character_set_results", "utf8"))
+	require.EqualValues(t, types.TinyTextBlobMax*3, tinyTextUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.TextBlobMax*3, textUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.MediumTextBlobMax*3, mediumTextUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.LongTextBlobMax, longTextUtf8mb4.MaxTextResponseByteLength(ctx))
+
+	// When character_set_results is set to NULL, the multibyte character multiplier is taken from
+	// the type's charset (4 in this case)
+	require.NoError(t, session.SetSessionVariable(ctx, "character_set_results", nil))
+	require.EqualValues(t, types.TinyTextBlobMax*4, tinyTextUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.TextBlobMax*4, textUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.MediumTextBlobMax*4, mediumTextUtf8mb4.MaxTextResponseByteLength(ctx))
+	require.EqualValues(t, types.LongTextBlobMax, longTextUtf8mb4.MaxTextResponseByteLength(ctx))
+}
+
 func TestHandlerTimeout(t *testing.T) {
 	require := require.New(t)
 

sql/analyzer/validate_create_table.go

@@ -159,7 +159,7 @@ func resolveAlterColumn(ctx *sql.Context, a *Analyzer, n sql.Node, scope *Scope,
 			if err != nil {
 				return nil, transform.SameTree, err
 			}
-			sch, err = validatePrimaryKey(initialSch, sch, n.(*plan.AlterPK))
+			sch, err = validatePrimaryKey(ctx, initialSch, sch, n.(*plan.AlterPK))
 			if err != nil {
 				return nil, transform.SameTree, err
 			}
@@ -397,7 +397,7 @@ func validateAlterIndex(ctx *sql.Context, initialSch, sch sql.Schema, ai *plan.A
 		if !ok {
 			return nil, sql.ErrKeyColumnDoesNotExist.New(badColName)
 		}
-		err := validateIndexType(ai.Columns, sch)
+		err := validateIndexType(ctx, ai.Columns, sch)
 		if err != nil {
 			return nil, err
 		}
@@ -455,7 +455,7 @@ func validateAlterIndex(ctx *sql.Context, initialSch, sch sql.Schema, ai *plan.A
 }
 
 // validatePrefixLength handles all errors related to creating indexes with prefix lengths
-func validatePrefixLength(schCol *sql.Column, idxCol sql.IndexColumn) error {
+func validatePrefixLength(ctx *sql.Context, schCol *sql.Column, idxCol sql.IndexColumn) error {
 	// Throw prefix length error for non-string types with prefixes
 	if idxCol.Length > 0 && !types.IsText(schCol.Type) {
 		return sql.ErrInvalidIndexPrefix.New(schCol.Name)
@@ -473,7 +473,7 @@ func validatePrefixLength(schCol *sql.Column, idxCol sql.IndexColumn) error {
 	}
 
 	// The specified prefix length is longer than the column
-	maxByteLength := int64(schCol.Type.MaxTextResponseByteLength())
+	maxByteLength := int64(schCol.Type.MaxTextResponseByteLength(ctx))
 	if prefixByteLength > maxByteLength {
 		return sql.ErrInvalidIndexPrefix.New(schCol.Name)
 	}
@@ -487,10 +487,10 @@ func validatePrefixLength(schCol *sql.Column, idxCol sql.IndexColumn) error {
 }
 
 // validateIndexType prevents creating invalid indexes
-func validateIndexType(cols []sql.IndexColumn, sch sql.Schema) error {
+func validateIndexType(ctx *sql.Context, cols []sql.IndexColumn, sch sql.Schema) error {
 	for _, idxCol := range cols {
 		schCol := sch[sch.IndexOfColName(idxCol.Name)]
-		err := validatePrefixLength(schCol, idxCol)
+		err := validatePrefixLength(ctx, schCol, idxCol)
 		if err != nil {
 			return err
 		}
@@ -610,7 +610,7 @@ func validateIndexes(ctx *sql.Context, tableSpec *plan.TableSpec) error {
 			if !ok {
 				return sql.ErrUnknownIndexColumn.New(idxCol.Name, idx.IndexName)
 			}
-			err := validatePrefixLength(schCol, idxCol)
+			err := validatePrefixLength(ctx, schCol, idxCol)
 			if err != nil {
 				return err
 			}
@@ -683,7 +683,7 @@ func getTableIndexNames(ctx *sql.Context, a *Analyzer, table sql.Node) ([]string
 }
 
 // validatePrimaryKey validates a primary key add or drop operation.
-func validatePrimaryKey(initialSch, sch sql.Schema, ai *plan.AlterPK) (sql.Schema, error) {
+func validatePrimaryKey(ctx *sql.Context, initialSch, sch sql.Schema, ai *plan.AlterPK) (sql.Schema, error) {
 	tableName := getTableName(ai.Table)
 	switch ai.Action {
 	case plan.PrimaryKeyAction_Create:
@@ -698,7 +698,7 @@ func validatePrimaryKey(initialSch, sch sql.Schema, ai *plan.AlterPK) (sql.Schem
 
 		for _, idxCol := range ai.Columns {
 			schCol := sch[sch.IndexOf(idxCol.Name, tableName)]
-			err := validatePrefixLength(schCol, idxCol)
+			err := validatePrefixLength(ctx, schCol, idxCol)
 			if err != nil {
 				return nil, err
 			}

sql/information_schema/columns_table.go

@@ -209,7 +209,7 @@ func getRowFromColumn(ctx *sql.Context, curOrdPos int, col *sql.Column, dbName,
 		}
 	}
 
-	charName, collName, charMaxLen, charOctetLen := getCharAndCollNamesAndCharMaxAndOctetLens(col.Type)
+	charName, collName, charMaxLen, charOctetLen := getCharAndCollNamesAndCharMaxAndOctetLens(ctx, col.Type)
 
 	numericPrecision, numericScale := getColumnPrecisionAndScale(col.Type)
 	if types.IsDatetimeType(col.Type) || types.IsTimestampType(col.Type) {
@@ -539,7 +539,7 @@ func getColumnPrecisionAndScale(colType sql.Type) (interface{}, interface{}) {
 	}
 }
 
-func getCharAndCollNamesAndCharMaxAndOctetLens(colType sql.Type) (interface{}, interface{}, interface{}, interface{}) {
+func getCharAndCollNamesAndCharMaxAndOctetLens(ctx *sql.Context, colType sql.Type) (interface{}, interface{}, interface{}, interface{}) {
 	var (
 		charName     interface{}
 		collName     interface{}
@@ -551,8 +551,8 @@ func getCharAndCollNamesAndCharMaxAndOctetLens(colType sql.Type) (interface{}, i
 		collName = colColl.Name()
 		charName = colColl.CharacterSet().String()
 		if types.IsEnum(colType) || types.IsSet(colType) {
-			charOctetLen = int64(colType.MaxTextResponseByteLength())
-			charMaxLen = int64(colType.MaxTextResponseByteLength()) / colColl.CharacterSet().MaxLength()
+			charOctetLen = int64(colType.MaxTextResponseByteLength(ctx))
+			charMaxLen = int64(colType.MaxTextResponseByteLength(ctx)) / colColl.CharacterSet().MaxLength()
 		}
 	}
 	if st, ok := colType.(sql.StringType); ok {

sql/information_schema/routines_table.go

@@ -285,7 +285,7 @@ func parametersRowIter(ctx *Context, c Catalog, p map[string][]*plan.Procedure)
 					parameterMode = "OUT"
 				}
 
-				charName, collName, charMaxLen, charOctetLen := getCharAndCollNamesAndCharMaxAndOctetLens(param.Type)
+				charName, collName, charMaxLen, charOctetLen := getCharAndCollNamesAndCharMaxAndOctetLens(ctx, param.Type)
 				numericPrecision, numericScale := getColumnPrecisionAndScale(param.Type)
 				// float types get nil for numericScale, but it gets 0 for this table
 				if _, ok := param.Type.(NumberType); ok {

sql/type.go

@@ -77,8 +77,10 @@ type Type interface {
 	// Equals returns whether the given type is equivalent to the calling type. All parameters are included in the
 	// comparison, so ENUM("a", "b") is not equivalent to ENUM("a", "b", "c").
 	Equals(otherType Type) bool
-	// MaxTextResponseByteLength returns the maximum number of bytes needed to serialize an instance of this type as a string in a response over the wire for MySQL's text protocol – in other words, this is the maximum bytes needed to serialize any value of this type as human-readable text, NOT in a more compact, binary representation.
-	MaxTextResponseByteLength() uint32
+	// MaxTextResponseByteLength returns the maximum number of bytes needed to serialize an instance of this type
+	// as a string in a response over the wire for MySQL's text protocol – in other words, this is the maximum bytes
+	// needed to serialize any value of this type as human-readable text, NOT in a more compact, binary representation.
+	MaxTextResponseByteLength(ctx *Context) uint32
 	// Promote will promote the current type to the largest representing type of the same kind, such as Int8 to Int64.
 	Promote() Type
 	// SQL returns the sqltypes.Value for the given value.

sql/types/bit.go

@@ -72,7 +72,7 @@ func MustCreateBitType(numOfBits uint8) BitType {
 }
 
 // MaxTextResponseByteLength implements Type interface
-func (t BitType_) MaxTextResponseByteLength() uint32 {
+func (t BitType_) MaxTextResponseByteLength(_ *sql.Context) uint32 {
 	// Because this is a text serialization format, each bit requires one byte in the text response format
 	return uint32(t.numOfBits)
 }

sql/types/datetime.go

@@ -323,7 +323,7 @@ func (t datetimeType) Equals(otherType sql.Type) bool {
 }
 
 // MaxTextResponseByteLength implements the Type interface
-func (t datetimeType) MaxTextResponseByteLength() uint32 {
+func (t datetimeType) MaxTextResponseByteLength(_ *sql.Context) uint32 {
 	switch t.baseType {
 	case sqltypes.Date:
 		return uint32(len(sql.DateLayout))

sql/types/decimal.go

@@ -267,7 +267,7 @@ func (t DecimalType_) Equals(otherType sql.Type) bool {
 }
 
 // MaxTextResponseByteLength implements the Type interface
-func (t DecimalType_) MaxTextResponseByteLength() uint32 {
+func (t DecimalType_) MaxTextResponseByteLength(_ *sql.Context) uint32 {
 	if t.scale == 0 {
 		// if no digits are reserved for the right-hand side of the decimal point,
 		// just return precision plus one byte for sign

sql/types/deferred.go

@@ -54,7 +54,7 @@ func (t deferredType) Convert(v interface{}) (interface{}, sql.ConvertInRange, e
 }
 
 // MaxTextResponseByteLength implements the Type interface
-func (t deferredType) MaxTextResponseByteLength() uint32 {
+func (t deferredType) MaxTextResponseByteLength(_ *sql.Context) uint32 {
 	// deferredType is never actually sent over the wire
 	return 0
 }