Merge pull request #2394 from dolthub/daylon/binary-wire-tests

Implemented the binary wire format

Author: fulghum

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/sirupsen/logrus v1.8.3
 	github.com/stretchr/testify v1.11.1
 	github.com/twpayne/go-geom v1.3.6
+	github.com/xdg-go/scram v1.2.0
 	github.com/xdg-go/stringprep v1.0.4
 	golang.org/x/crypto v0.47.0
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
@@ -184,6 +185,7 @@ require (
 	github.com/tklauser/numcpus v0.11.0 // indirect
 	github.com/twpayne/go-kml v1.5.2-0.20200728095708-9f2fd4dfcbfe // indirect
 	github.com/vbauerster/mpb/v8 v8.0.2 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xitongsys/parquet-go v1.6.1 // indirect
 	github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371 // indirect
 	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect

go.sum

@@ -798,6 +798,10 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/vbauerster/mpb/v8 v8.0.2 h1:alVQG69Jg5+Ku9Hu1dakDx50uACEHnIzS7i356NQ/Vs=
 github.com/vbauerster/mpb/v8 v8.0.2/go.mod h1:Z9VJYIzXls7xZwirZjShGsi+14enzJhQfGyb/XZK0ZQ=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
+github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=

server/cast/bit.go

@@ -73,7 +73,7 @@ func bitImplicit() {
 			if err != nil {
 				return nil, err
 			}
-			expectedLength := pgtypes.GetCharLengthFromTypmod(targetType.GetAttTypMod())
+			expectedLength := targetType.GetAttTypMod()
 			if array.BitLen() != uint(expectedLength) {
 				return nil, pgtypes.ErrWrongLengthBit.New(len(input), expectedLength)
 			}
@@ -91,9 +91,8 @@ func bitImplicit() {
 			}
 			atttypmod := targetType.GetAttTypMod()
 			if atttypmod != -1 {
-				maxLength := pgtypes.GetCharLengthFromTypmod(atttypmod)
-				if int32(array.BitLen()) > maxLength {
-					return nil, pgtypes.ErrVarBitLengthExceeded.New(maxLength)
+				if int32(array.BitLen()) > atttypmod {
+					return nil, pgtypes.ErrVarBitLengthExceeded.New(atttypmod)
 				}
 			}
 			return tree.AsStringWithFlags(array, tree.FmtPgwireText), nil

server/cast/varbit.go

@@ -39,7 +39,7 @@ func varBitImplicit() {
 			if err != nil {
 				return nil, err
 			}
-			expectedLength := pgtypes.GetCharLengthFromTypmod(targetType.GetAttTypMod())
+			expectedLength := targetType.GetAttTypMod()
 			if array.BitLen() != uint(expectedLength) {
 				return nil, pgtypes.ErrWrongLengthBit.New(len(input), expectedLength)
 			}
@@ -57,9 +57,8 @@ func varBitImplicit() {
 			}
 			atttypmod := targetType.GetAttTypMod()
 			if atttypmod != -1 {
-				maxLength := pgtypes.GetCharLengthFromTypmod(atttypmod)
-				if int32(array.BitLen()) > maxLength {
-					return nil, pgtypes.ErrVarBitLengthExceeded.New(maxLength)
+				if int32(array.BitLen()) > atttypmod {
+					return nil, pgtypes.ErrVarBitLengthExceeded.New(atttypmod)
 				}
 			}
 			return tree.AsStringWithFlags(array, tree.FmtPgwireText), nil

server/config/parameters_list.go

@@ -2990,7 +2990,7 @@ var postgresConfigParameters = map[string]sql.SystemVariable{
 	// schema on the path here.
 	"search_path": &Parameter{
 		Name:      "search_path",
-		Default:   "\"$user\", public,",
+		Default:   "\"$user\", public",
 		Category:  "Client Connection Defaults / Statement Behavior",
 		ShortDesc: "Sets the schema search order for names that are not schema-qualified.",
 		Context:   ParameterContextUser,

server/connection_data.go

@@ -23,7 +23,6 @@ import (
 	"github.com/dolthub/go-mysql-server/sql/expression"
 	"github.com/dolthub/go-mysql-server/sql/plan"
 	"github.com/dolthub/go-mysql-server/sql/transform"
-	"github.com/dolthub/vitess/go/vt/proto/query"
 	vitess "github.com/dolthub/vitess/go/vt/sqlparser"
 	"github.com/jackc/pgx/v5/pgproto3"
 	"github.com/lib/pq/oid"
@@ -94,6 +93,7 @@ type PortalData struct {
 	IsEmptyQuery bool
 	Fields       []pgproto3.FieldDescription
 	BoundPlan    sql.Node
+	FormatCodes  []int16
 }
 
 type PreparedStatementData struct {
@@ -133,15 +133,15 @@ func extractBindVarTypes(queryPlan sql.Node) ([]uint32, error) {
 				case *plan.Offset:
 					typOid = uint32(oid.T_int4)
 				default:
-					typOid, err = VitessTypeToObjectID(e.Type().Type())
+					typOid, err = VitessTypeToObjectID(e.Type())
 					if err != nil {
 						err = errors.Errorf("could not determine OID for placeholder %s: %e", e.Name, err)
 						return false
 					}
 				}
 			} else {
 				// TODO: should remove usage non doltgres type
-				typOid, err = VitessTypeToObjectID(e.Type().Type())
+				typOid, err = VitessTypeToObjectID(e.Type())
 				if err != nil {
 					err = errors.Errorf("could not determine OID for placeholder %s: %e", e.Name, err)
 					return false
@@ -158,7 +158,7 @@ func extractBindVarTypes(queryPlan sql.Node) ([]uint32, error) {
 				if doltgresType, ok := bindVar.Type().(*pgtypes.DoltgresType); ok {
 					typOid = id.Cache().ToOID(doltgresType.ID.AsId())
 				} else {
-					typOid, err = VitessTypeToObjectID(e.Type().Type())
+					typOid, err = VitessTypeToObjectID(e.Type())
 					if err != nil {
 						err = errors.Errorf("could not determine OID for placeholder %s: %e", bindVar.Name, err)
 						return false
@@ -175,7 +175,7 @@ func extractBindVarTypes(queryPlan sql.Node) ([]uint32, error) {
 		case *expression.Convert:
 			if bindVar, ok := e.Child.(*expression.BindVar); ok {
 				var typOid uint32
-				typOid, err = VitessTypeToObjectID(e.Type().Type())
+				typOid, err = VitessTypeToObjectID(e.Type())
 				if err != nil {
 					err = errors.Errorf("could not determine OID for placeholder %s: %e", bindVar.Name, err)
 					return false
@@ -218,65 +218,10 @@ func extractBindVarTypes(queryPlan sql.Node) ([]uint32, error) {
 
 // VitessTypeToObjectID returns a type, as defined by Vitess, into a type as defined by Postgres.
 // OIDs can be obtained with the following query: `SELECT oid, typname FROM pg_type ORDER BY 1;`
-func VitessTypeToObjectID(typ query.Type) (uint32, error) {
-	switch typ {
-	case query.Type_INT8:
-		// Postgres doesn't make use of a small integer type for integer returns, which presents a bit of a conundrum.
-		// GMS defines boolean operations as the smallest integer type, while Postgres has an explicit bool type.
-		// We can't always assume that `INT8` means bool, since it could just be a small integer. As a result, we'll
-		// always return this as though it's an `INT16`, which also means that we can't support bools right now.
-		// OIDs 16 (bool) and 18 (char, ASCII only?) are the only single-byte types as far as I'm aware.
-		return uint32(oid.T_int2), nil
-	case query.Type_INT16:
-		// The technically correct OID is 21 (2-byte integer), however it seems like some clients don't actually expect
-		// this, so I'm not sure when it's actually used by Postgres. Because of this, we'll just pretend it's an `INT32`.
-		return uint32(oid.T_int2), nil
-	case query.Type_INT24:
-		// Postgres doesn't have a 3-byte integer type, so just pretend it's `INT32`.
-		return uint32(oid.T_int4), nil
-	case query.Type_INT32:
-		return uint32(oid.T_int4), nil
-	case query.Type_INT64:
-		return uint32(oid.T_int8), nil
-	case query.Type_UINT8:
-		return uint32(oid.T_int4), nil
-	case query.Type_UINT16:
-		return uint32(oid.T_int4), nil
-	case query.Type_UINT24:
-		return uint32(oid.T_int4), nil
-	case query.Type_UINT32:
-		// Since this has an upperbound greater than `INT32`, we'll treat it as `INT64`
-		return uint32(oid.T_oid), nil
-	case query.Type_UINT64:
-		// Since this has an upperbound greater than `INT64`, we'll treat it as `NUMERIC`
-		return uint32(oid.T_numeric), nil
-	case query.Type_FLOAT32:
-		return uint32(oid.T_float4), nil
-	case query.Type_FLOAT64:
-		return uint32(oid.T_float8), nil
-	case query.Type_DECIMAL:
-		return uint32(oid.T_numeric), nil
-	case query.Type_CHAR:
-		return uint32(oid.T_char), nil
-	case query.Type_VARCHAR:
-		return uint32(oid.T_varchar), nil
-	case query.Type_TEXT:
-		return uint32(oid.T_text), nil
-	case query.Type_BLOB:
-		return uint32(oid.T_bytea), nil
-	case query.Type_JSON:
-		return uint32(oid.T_json), nil
-	case query.Type_TIMESTAMP, query.Type_DATETIME:
-		return uint32(oid.T_timestamp), nil
-	case query.Type_DATE:
-		return uint32(oid.T_date), nil
-	case query.Type_NULL_TYPE:
-		return uint32(oid.T_text), nil // NULL is treated as TEXT on the wire
-	case query.Type_ENUM:
-		return uint32(oid.T_text), nil // TODO: temporary solution until we support CREATE TYPE
-	case query.Type_EXPRESSION:
-		return uint32(oid.T_text), nil // this most closely matches the behavior in postgres, which treats any unresolved type as a string (unless it has special handling)
-	default:
-		return 0, errors.Errorf("unsupported type: %s", typ)
+func VitessTypeToObjectID(typ sql.Type) (uint32, error) {
+	doltgresType, err := pgtypes.FromGmsTypeToDoltgresType(typ)
+	if err != nil {
+		return 0, err
 	}
+	return id.Cache().ToOID(doltgresType.ID.AsId()), nil
 }

server/connection_handler.go

@@ -634,7 +634,8 @@ func (h *ConnectionHandler) handleBind(message *pgproto3.Bind) error {
 			varTypes:    preparedData.BindVarTypes,
 			formatCodes: message.ParameterFormatCodes,
 			parameters:  message.Parameters,
-		})
+		},
+		message.ResultFormatCodes)
 	if err != nil {
 		return err
 	}
@@ -644,10 +645,15 @@ func (h *ConnectionHandler) handleBind(message *pgproto3.Bind) error {
 		return errors.Errorf("expected a sql.Node, got %T", analyzedPlan)
 	}
 
+	resultFormatCodes, err := extendFormatCodes(len(fields), message.ResultFormatCodes)
+	if err != nil {
+		return err
+	}
 	h.portals[message.DestinationPortal] = PortalData{
-		Query:     preparedData.Query,
-		Fields:    fields,
-		BoundPlan: boundPlan,
+		Query:       preparedData.Query,
+		Fields:      fields,
+		BoundPlan:   boundPlan,
+		FormatCodes: resultFormatCodes,
 	}
 	return h.send(&pgproto3.BindComplete{})
 }
@@ -679,7 +685,7 @@ func (h *ConnectionHandler) handleExecute(message *pgproto3.Execute) error {
 	rowsAffected := int32(0)
 
 	callback := h.spoolRowsCallback(query, &rowsAffected, true)
-	err = h.doltgresHandler.ComExecuteBound(context.Background(), h.mysqlConn, query.String, portalData.BoundPlan, callback)
+	err = h.doltgresHandler.ComExecuteBound(context.Background(), h.mysqlConn, query.String, portalData.BoundPlan, portalData.FormatCodes, callback)
 	if err != nil {
 		return err
 	}
@@ -839,7 +845,7 @@ func (h *ConnectionHandler) handleCopyDataHelper(copyState *copyFromStdinState,
 	}
 
 	callback := func(_ *sql.Context, _ *Result) error { return nil }
-	err = h.doltgresHandler.ComExecuteBound(sqlCtx, h.mysqlConn, "COPY FROM", copyState.insertNode, callback)
+	err = h.doltgresHandler.ComExecuteBound(sqlCtx, h.mysqlConn, "COPY FROM", copyState.insertNode, nil, callback)
 	if err != nil {
 		return false, false, err
 	}