MDEV-37947 Item_func_hex doesn't check for max_allowed_packet

Author: vuvova

mysql-test/main/func_str.result

@@ -1,4 +1,3 @@
-drop table if exists t1,t2;
 set @save_max_allowed_packet=@@global.max_allowed_packet;
 set global max_allowed_packet=1048576;
 connect  conn1,localhost,root,,;
@@ -5415,5 +5414,11 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 DROP TABLE t1;
 #
-# End of 10.6 tests
+# MDEV-37947 Item_func_hex doesn't check for max_allowed_packet
 #
+select hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex('\\'))))))))))))))))))))))))))))))))))))))))))))) as x;
+x
+NULL
+Warnings:
+Warning	1301	Result of hex() was larger than max_allowed_packet (16777216) - truncated
+# End of 10.6 tests

mysql-test/main/func_str.test

@@ -2,10 +2,6 @@
 # -----------
 # Testing string functions
 
---disable_warnings
-drop table if exists t1,t2;
---enable_warnings
-
 set @save_max_allowed_packet=@@global.max_allowed_packet;
 set global max_allowed_packet=1048576;
 connect (conn1,localhost,root,,);
@@ -2455,5 +2451,8 @@ SHOW CREATE TABLE t1;
 DROP TABLE t1;
 
 --echo #
---echo # End of 10.6 tests
+--echo # MDEV-37947 Item_func_hex doesn't check for max_allowed_packet
 --echo #
+select hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex(hex('\\'))))))))))))))))))))))))))))))))))))))))))))) as x;
+
+--echo # End of 10.6 tests

sql/item_strfunc.cc

@@ -3828,9 +3828,21 @@ String *Item_func_hex::val_str_ascii_from_val_str(String *str)
 {
   DBUG_ASSERT(&tmp_value != str);
   String *res= args[0]->val_str(&tmp_value);
-  DBUG_ASSERT(res != str);
+  THD *thd= current_thd;
+
   if ((null_value= (res == NULL)))
     return NULL;
+
+  if (res->length()*2 > thd->variables.max_allowed_packet)
+  {
+    push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
+                        ER_WARN_ALLOWED_PACKET_OVERFLOWED,
+                        ER_THD(thd, ER_WARN_ALLOWED_PACKET_OVERFLOWED),
+                        func_name(), thd->variables.max_allowed_packet);
+    null_value= true;
+    return NULL;
+  }
+
   return str->set_hex(res->ptr(), res->length()) ? make_empty_result(str) : str;
 }