MDEV-37935 : Assertion `0' failed in int wsrep::transaction::before_rollback()

Problem was executable comment in REPLACE that caused
consistency check that is not supported for RSU and
!InnoDB tables.

Fix is to check is Online Schema Upgrade (OSU) method
something else than TOI and if it is then table storage
engine must be InnoDB or error is produced and execution
returns and no consistency check is done.

Author: janlindstrom

mysql-test/suite/galera/r/MDEV-37935.result

@@ -0,0 +1,40 @@
+connection node_2;
+connection node_1;
+SET sql_mode='';
+SET SESSION autocommit=0;
+SET SESSION enforce_storage_engine=Aria;
+SET WSREP_OSU_METHOD = RSU;
+CREATE TABLE t1 (a TEXT) ;
+Warnings:
+Note	1266	Using storage engine Aria for table 't1'
+Note	1266	Using storage engine Aria for table 't1'
+CREATE TABLE t2 (a TEXT) ;
+Warnings:
+Note	1266	Using storage engine Aria for table 't2'
+Note	1266	Using storage engine Aria for table 't2'
+SET SESSION autocommit=1;
+REPLACE INTO t2 (a) SELECT a from t1;
+ERROR 42000: This version of MariaDB doesn't yet support 'RSU on this table engine'
+REPLACE INTO t2 (a) SELECT /*!99997 */ a from t1;
+ERROR 42000: This version of MariaDB doesn't yet support 'RSU on this table engine'
+SET WSREP_OSU_METHOD = TOI;
+DROP TABLE t1,t2;
+SET sql_mode='';
+SET SESSION autocommit=0;
+SET SESSION enforce_storage_engine=MyISAM;
+SET WSREP_OSU_METHOD = RSU;
+CREATE TABLE t1 (a TEXT) ;
+Warnings:
+Note	1266	Using storage engine MyISAM for table 't1'
+Note	1266	Using storage engine MyISAM for table 't1'
+CREATE TABLE t2 (a TEXT) ;
+Warnings:
+Note	1266	Using storage engine MyISAM for table 't2'
+Note	1266	Using storage engine MyISAM for table 't2'
+SET SESSION autocommit=1;
+REPLACE INTO t2 (a) SELECT a from t1;
+ERROR 42000: This version of MariaDB doesn't yet support 'RSU on this table engine'
+REPLACE INTO t2 (a) SELECT /*!99997 */ a from t1;
+ERROR 42000: This version of MariaDB doesn't yet support 'RSU on this table engine'
+SET WSREP_OSU_METHOD = TOI;
+DROP TABLE t1,t2;

mysql-test/suite/galera/t/MDEV-37935.test

@@ -0,0 +1,30 @@
+--source include/galera_cluster.inc
+--source include/have_aria.inc
+
+SET sql_mode='';
+SET SESSION autocommit=0;
+SET SESSION enforce_storage_engine=Aria;
+SET WSREP_OSU_METHOD = RSU;
+CREATE TABLE t1 (a TEXT) ;
+CREATE TABLE t2 (a TEXT) ;
+SET SESSION autocommit=1;
+--error ER_NOT_SUPPORTED_YET
+REPLACE INTO t2 (a) SELECT a from t1;
+--error ER_NOT_SUPPORTED_YET
+REPLACE INTO t2 (a) SELECT /*!99997 */ a from t1;
+SET WSREP_OSU_METHOD = TOI;
+DROP TABLE t1,t2;
+
+SET sql_mode='';
+SET SESSION autocommit=0;
+SET SESSION enforce_storage_engine=MyISAM;
+SET WSREP_OSU_METHOD = RSU;
+CREATE TABLE t1 (a TEXT) ;
+CREATE TABLE t2 (a TEXT) ;
+SET SESSION autocommit=1;
+--error ER_NOT_SUPPORTED_YET
+REPLACE INTO t2 (a) SELECT a from t1;
+--error ER_NOT_SUPPORTED_YET
+REPLACE INTO t2 (a) SELECT /*!99997 */ a from t1;
+SET WSREP_OSU_METHOD = TOI;
+DROP TABLE t1,t2;

sql/sql_parse.cc

@@ -4724,10 +4724,21 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt)
 #ifdef WITH_WSREP
       if (wsrep && !first_table->view)
       {
-        const legacy_db_type db_type= first_table->table->file->partition_ht()->db_type;
+        const handlerton *hton= first_table->table->file->partition_ht() ?
+          first_table->table->file->partition_ht() :
+          first_table->table->file->ht;
+
+        const legacy_db_type db_type= hton->db_type;
         // For InnoDB we don't need to worry about anything here:
         if (db_type != DB_TYPE_INNODB)
         {
+          /* Only TOI allowed to !InnoDB tables */
+          if (thd->variables.wsrep_OSU_method != WSREP_OSU_TOI)
+          {
+            my_error(ER_NOT_SUPPORTED_YET, MYF(0), "RSU on this table engine");
+            break;
+          }
+
           // For consistency check inserted table needs to be InnoDB
           if (thd->wsrep_consistency_check != NO_CONSISTENCY_CHECK)
           {
@@ -4737,16 +4748,10 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt)
                                 " for InnoDB tables.");
             thd->wsrep_consistency_check= NO_CONSISTENCY_CHECK;
           }
-          /* Only TOI allowed to !InnoDB tables */
-          if (wsrep_OSU_method_get(thd) != WSREP_OSU_TOI)
-          {
-            my_error(ER_NOT_SUPPORTED_YET, MYF(0), "RSU on this table engine");
-            break;
-          }
           // For !InnoDB we start TOI if it is not yet started and hope for the best
           if (!wsrep_toi)
           {
-            /* Currently we support TOI for MyISAM only. */
+            /* Currently we support TOI for MyISAM && Aria only. */
             if ((db_type == DB_TYPE_MYISAM && wsrep_check_mode(WSREP_MODE_REPLICATE_MYISAM)) ||
                 (db_type == DB_TYPE_ARIA   && wsrep_check_mode(WSREP_MODE_REPLICATE_ARIA)))
             {