Do not convert keys if key type is incompatible with column type

enginetest/queries/script_queries.go

@@ -1176,10 +1176,8 @@ FROM task_instance INNER JOIN job ON job.id = task_instance.queued_by_job_id INN
 				Expected:              []sql.Row{{"11"}},
 				ExpectedWarningsCount: 0,
 			},
-
 			{
 				// https://github.com/dolthub/dolt/issues/9739
-				Skip:    true,
 				Dialect: "mysql",
 				Query:   "select * from test01 where pk in (11)",
 				Expected: []sql.Row{
@@ -1193,25 +1191,22 @@ FROM task_instance INNER JOIN job ON job.id = task_instance.queued_by_job_id INN
 			},
 			{
 				// https://github.com/dolthub/dolt/issues/9739
-				Skip:    true,
+				Skip:    true, // this passes in gms but not dolt
 				Dialect: "mysql",
 				Query:   "select * from test01 where pk=3",
 				Expected: []sql.Row{
 					{"  3 12 4"},
-					{"  3. 12 4"},
 					{"3. 12 4"},
 				},
 				ExpectedWarningsCount: 12,
 				ExpectedWarning:       mysql.ERTruncatedWrongValue,
 			},
 			{
 				// https://github.com/dolthub/dolt/issues/9739
-				Skip:    true,
 				Dialect: "mysql",
 				Query:   "select * from test01 where pk>=3 and pk < 4",
 				Expected: []sql.Row{
 					{"  3 12 4"},
-					{"  3. 12 4"},
 					{"  3.2 12 4"},
 					{"+3.1234"},
 					{"3. 12 4"},
@@ -1235,6 +1230,50 @@ FROM task_instance INNER JOIN job ON job.id = task_instance.queued_by_job_id INN
 			},
 		},
 	},
+	{
+		// https://github.com/dolthub/dolt/issues/9936
+		Name:    "invisible hash index with different key types",
+		Dialect: "mysql",
+		SetUpScript: []string{
+			"create table t0(c0 varchar(500))",
+			"insert into t0(c0) values (77367106)",
+			"create index i0 using hash on t0(c0) invisible",
+		},
+		Assertions: []ScriptTestAssertion{
+			{
+				Query:    "select c0 from t0 where 1630944823 >= t0.c0",
+				Expected: []sql.Row{{"77367106"}},
+			},
+			{
+				Query:    "select c0 from t0 where 1630944823 <= t0.c0",
+				Expected: []sql.Row{},
+			},
+			{
+				Query:    "select c0 from t0 where '1630944823' >= t0.c0",
+				Expected: []sql.Row{},
+			},
+			{
+				Query:    "select c0 from t0 where '1630944823' <= t0.c0",
+				Expected: []sql.Row{{"77367106"}},
+			},
+			{
+				Query:    "select c0 from t0 where 1630944823.2 >= t0.c0",
+				Expected: []sql.Row{{"77367106"}},
+			},
+			{
+				Query:    "select c0 from t0 where 1630944823.2 <= t0.c0",
+				Expected: []sql.Row{},
+			},
+			{
+				Query:    "select c0 from t0 where '1630944823.2' >= t0.c0",
+				Expected: []sql.Row{},
+			},
+			{
+				Query:    "select c0 from t0 where '1630944823.2' <= t0.c0",
+				Expected: []sql.Row{{"77367106"}},
+			},
+		},
+	},
 	{
 		// https://github.com/dolthub/dolt/issues/9821
 		Name: "strings with boolean operators",

sql/index_builder.go

@@ -224,11 +224,23 @@ func (b *MySQLIndexBuilder) GreaterThan(ctx *Context, colExpr string, keyType Ty
 	return b
 }
 
+// isConvertibleKeyType checks if the key can be converted into the column type
+func isConvertibleKeyType(colType Type, keyType Type) bool {
+	if IsStringType(colType) {
+		return !(IsNumberType(keyType) || IsDecimalType(keyType))
+	}
+	// TODO: check other types
+	return true
+}
+
 // convertKey converts the given key from keyType to colType, returning an error if the conversion fails.
 func (b *MySQLIndexBuilder) convertKey(ctx *Context, colType Type, keyType Type, key interface{}) (interface{}, error) {
 	if et, ok := colType.(ExtendedType); ok {
 		return et.ConvertToType(ctx, keyType.(ExtendedType), key)
 	} else {
+		if !isConvertibleKeyType(colType, keyType) {
+			return nil, ErrInvalidValueType.New(key, colType)
+		}
 		k, _, err := colType.Convert(ctx, key)
 		if err != nil && !ErrTruncatedIncorrect.Is(err) {
 			return nil, err

sql/type.go

@@ -172,6 +172,11 @@ type NumberType interface {
 	DisplayWidth() int
 }
 
+func IsNumberType(t Type) bool {
+	_, ok := t.(NumberType)
+	return ok
+}
+
 // RoundingNumberType represents Number Types that implement an additional interface
 // that supports rounding when converting rather than the default truncation.
 type RoundingNumberType interface {
@@ -197,6 +202,11 @@ type StringType interface {
 	Length() int64
 }
 
+func IsStringType(t Type) bool {
+	_, ok := t.(StringType)
+	return ok
+}
+
 // DatetimeType represents DATE, DATETIME, and TIMESTAMP.
 // https://dev.mysql.com/doc/refman/8.0/en/datetime.html
 // The type of the returned value is time.Time.
@@ -277,6 +287,11 @@ type DecimalType interface {
 	Scale() uint8
 }
 
+func IsDecimalType(t Type) bool {
+	_, ok := t.(DecimalType)
+	return ok
+}
+
 type Type2 interface {
 	Type