MDEV-36545: Possible crash in row_raw_format_str()

row_raw_format_str(): Treat the invalid value charset_coll==0 as binary.
This could be invoked on FTS_%_CONFIG.key or SYS_FOREIGN.ID
or possible other key columns.

dtype_is_utf8(): Merge to its only caller row_raw_format_str().

row_raw_format(): Add debug assertions and comments to document
when dtype_get_charset_coll(prtype) may be 0.

Author: dr-m

storage/innobase/include/data0type.h

@@ -353,16 +353,6 @@ dtype_form_prtype(ulint old_prtype, ulint charset_coll)
 	return(uint32_t(old_prtype + (charset_coll << 16)));
 }
 
-/*********************************************************************//**
-Determines if a MySQL string type is a subset of UTF-8.  This function
-may return false negatives, in case further character-set collation
-codes are introduced in MySQL later.
-@return whether a subset of UTF-8 */
-UNIV_INLINE
-bool
-dtype_is_utf8(
-/*==========*/
-	ulint	prtype);/*!< in: precise data type */
 /*********************************************************************//**
 Gets the type length.
 @return fixed length of the type, in bytes, or 0 if variable-length */

storage/innobase/include/data0type.inl

@@ -27,31 +27,6 @@ Created 1/16/1996 Heikki Tuuri
 #include "mach0data.h"
 #include "ha_prototypes.h"
 
-/*********************************************************************//**
-Determines if a MySQL string type is a subset of UTF-8.  This function
-may return false negatives, in case further character-set collation
-codes are introduced in MySQL later.
-@return whether a subset of UTF-8 */
-UNIV_INLINE
-bool
-dtype_is_utf8(
-/*==========*/
-	ulint	prtype)	/*!< in: precise data type */
-{
-	/* These codes have been copied from strings/ctype-extra.c
-	and strings/ctype-utf8.c. */
-	switch (dtype_get_charset_coll(prtype)) {
-	case 11: /* ascii_general_ci */
-	case 65: /* ascii_bin */
-	case 33: /* utf8_general_ci */
-	case 83: /* utf8_bin */
-	case 254: /* utf8_general_cs */
-		return true;
-	}
-
-	return false;
-}
-
 /*********************************************************************//**
 Gets the MySQL type code from a dtype.
 @return MySQL type code; this is NOT an InnoDB type code! */

storage/innobase/row/row0row.cc

@@ -1407,18 +1407,18 @@ row_raw_format_str(
 
 	charset_coll = dtype_get_charset_coll(prtype);
 
-	if (UNIV_LIKELY(dtype_is_utf8(prtype))) {
-
+	switch (charset_coll) {
+	case 11: /* ascii_general_ci */
+	case 65: /* ascii_bin */
+	case 33: /* utf8_general_ci */
+	case 83: /* utf8_bin */
+	case 254: /* utf8_general_cs */
 		return(ut_str_sql_format(data, data_len, buf, buf_size));
-	}
-	/* else */
-
-	if (charset_coll == DATA_MYSQL_BINARY_CHARSET_COLL) {
-
+	case 0:
+	case DATA_MYSQL_BINARY_CHARSET_COLL:
 		*format_in_hex = TRUE;
 		return(0);
 	}
-	/* else */
 
 	return(innobase_raw_format(data, data_len, charset_coll,
 					  buf, buf_size));
@@ -1479,9 +1479,18 @@ row_raw_format(
 		break;
 	case DATA_CHAR:
 	case DATA_VARCHAR:
+		/* FTS_%_CONFIG.key are incorrectly created with prtype==0.
+		The DATA_ENGLISH is being used for CHAR columns of the
+		InnoDB internal SQL parser, such as SYS_FOREIGN.ID.
+		For these, we will eventually goto format_in_hex. */
+		ut_ad(dtype_get_charset_coll(prtype) == 8
+		      || (mtype == DATA_VARCHAR
+			  && (prtype == 0 || prtype == DATA_ENGLISH)));
+		goto format_str;
 	case DATA_MYSQL:
 	case DATA_VARMYSQL:
-
+		ut_ad(dtype_get_charset_coll(prtype));
+	format_str:
 		ret = row_raw_format_str(data, data_len, prtype,
 					 buf, buf_size, &format_in_hex);
 		if (format_in_hex) {