Bug#36880863 Innodb startup takes longer time in

             Fil_system::prepare_open_for_business()
             after the fix for Bug 36808732

Analysis
--------
1. After the fix for Bug 36808732, the cost of checking
   the dd  metadata  was transferred to the
   prepare_open_for_business().
2. The fix for Bug 36808732 categorized all tables whose
   data directories are outside the default data directory
   as moved and these tables needs to be processed in
   prepare_open_for_business() which adds to the performance
   degradation.
3. In prepare_open_for_business() we are updating the dd
   metadata even when there is no update, which also
   causes performance degradation during startup.

Fix
---
1. Rollback the fix for Bug 36808732 partially.
2. Register the work threads launched during Validating
   files with MySQL so that dd flag can be accessed.

Change-Id: I523038d3a7760184efb64f5c894be8219a9c487d

Author: Aditya A

mysql-test/suite/innodb/r/alter_datadir_path.result

@@ -232,53 +232,78 @@ DROP TABLE dbtest.t7c;
 CREATE TABLE `t8a` (a INT, b INT) PARTITION BY RANGE(a) (PARTITION p0 VALUES LESS THAN (10) DATA DIRECTORY='MYSQL_TMP_DIR/external_dir', PARTITION p1 VALUES LESS THAN (20) DATA DIRECTORY='MYSQL_TMP_DIR/external_dir2');
 CREATE TABLE `t8b` (a INT, b INT) PARTITION BY RANGE(a) (PARTITION p0 VALUES LESS THAN (10) DATA DIRECTORY='MYSQL_TMP_DIR/external_dir', PARTITION p1 VALUES LESS THAN (20) DATA DIRECTORY='MYSQL_TMP_DIR/external_dir2');
 CREATE TABLE `t8c` (a INT, b INT) PARTITION BY RANGE(a) (PARTITION p0 VALUES LESS THAN (10) DATA DIRECTORY='MYSQL_TMP_DIR/external_dir', PARTITION p1 VALUES LESS THAN (20) DATA DIRECTORY='MYSQL_TMP_DIR/external_dir2');
+# Partition table with one partition in external directory and one in default directory
+CREATE TABLE `t8d` (a INT, b INT) PARTITION BY RANGE(a) (PARTITION p0 VALUES LESS THAN (10) DATA DIRECTORY='MYSQL_TMP_DIR/external_dir', PARTITION p1 VALUES LESS THAN (20) );
 # Shutdown server
-# Check that for partition p0 for t8a, t8b, t8c exist in EXTERNAL_DIR and partition p1 for for t8a, t8b, t8c exist in EXTERNAL_DIR2
+# Check that for partition p0 for t8a, t8b, t8c and t8d exist in EXTERNAL_DIR and partition p1 for for t8a, t8b, t8c exist in EXTERNAL_DIR2
 # Listing EXTERNAL_DIR
 t8a#p#p0.ibd
 t8b#p#p0.ibd
 t8c#p#p0.ibd
+t8d#p#p0.ibd
 # Listing EXTERNAL_DIR2
 t8a#p#p1.ibd
 t8b#p#p1.ibd
 t8c#p#p1.ibd
+# List default directory. t8d's p1 partition should be present
+t8d#p#p1.ibd
 # Restart the server
 # restart: --innodb-directories=MYSQL_TMP_DIR/external_dir;MYSQL_TMP_DIR/external_dir2
 # Alter table
 ALTER TABLE dbtest.t8a FORCE;
 ALTER TABLE dbtest.t8b FORCE, ALGORITHM=INPLACE;
 ALTER TABLE dbtest.t8c FORCE, ALGORITHM=COPY;
+ALTER TABLE dbtest.t8d FORCE, ALGORITHM=COPY;
 # Check that for partition p0 for t8a, t8b, t8c still exist in EXTERNAL_DIR and partition p1 for for t8a, t8b, t8c still exist in EXTERNAL_DIR2
 # Listing EXTERNAL_DIR
 t8a#p#p0.ibd
 t8b#p#p0.ibd
 t8c#p#p0.ibd
+t8d#p#p0.ibd
 # Listing EXTERNAL_DIR2
 t8a#p#p1.ibd
 t8b#p#p1.ibd
 t8c#p#p1.ibd
+# List default directory. t8d's p1 partition should be present
+t8d#p#p1.ibd
 # Shutdown server
-# Now lets move all partitioned tables p0 of t8a, t8b, t8c from EXTERNAL_DIR and partitioned tables p1 of t8a, t8b, t8c from EXTERNAL_DIR2 to default directory
+# Now lets move all partitioned tables p0 of t8a, t8b, t8c, t8d from EXTERNAL_DIR and partitioned tables p1 of t8a, t8b, t8c from EXTERNAL_DIR2 to default directory
 # Restart the server
 # restart: --innodb-directories=MYSQL_TMP_DIR/external_dir;MYSQL_TMP_DIR/external_dir2
 # Alter table
 ALTER TABLE dbtest.t8a FORCE;
 ALTER TABLE dbtest.t8b FORCE, ALGORITHM=INPLACE;
 ALTER TABLE dbtest.t8c FORCE, ALGORITHM=COPY;
-# Check that all the partitioned tables for t8a, t8b, t8c now exist in default directory
+# Check that all the partitioned tables for t8a, t8b, t8c and t8d now exist in default directory
 t8a#p#p0.ibd
 t8a#p#p1.ibd
 t8b#p#p0.ibd
 t8b#p#p1.ibd
 t8c#p#p0.ibd
 t8c#p#p1.ibd
+t8d#p#p0.ibd
+t8d#p#p1.ibd
+# Check that EXTERNAL_DIR and EXTERNAL_DIR2 are empty
+# Listing EXTERNAL_DIR
+# Listing EXTERNAL_DIR2
+ALTER TABLE dbtest.t8d FORCE, ALGORITHM=COPY;
+# Check that all the partitioned tables for t8a, t8b, t8c and t8d now exist in default directory
+t8a#p#p0.ibd
+t8a#p#p1.ibd
+t8b#p#p0.ibd
+t8b#p#p1.ibd
+t8c#p#p0.ibd
+t8c#p#p1.ibd
+t8d#p#p0.ibd
+t8d#p#p1.ibd
 # Check that EXTERNAL_DIR and EXTERNAL_DIR2 are empty
 # Listing EXTERNAL_DIR
 # Listing EXTERNAL_DIR2
 # Drop tables
 DROP TABLE dbtest.t8a;
 DROP TABLE dbtest.t8b;
 DROP TABLE dbtest.t8c;
+DROP TABLE dbtest.t8d;
 # Clean up for other tests
 # 2. Creating partitioned table without data directory clause but moving each partition to different external directories
 # Partition tables

mysql-test/suite/innodb/t/alter_datadir_path.test

@@ -28,6 +28,7 @@
 #Table t8a: Covers scenario create partitioned table with data directory clause and both partitions in different directories, move to default dir and alter table force
 #Table t8b: Covers scenario create partitioned table with data directory clause and both partitions in different directories, move to default dir and alter table force algo=inplace
 #Table t8c: Covers scenario create partitioned table with data directory clause and both partitions in different directories, move to default dir and alter table force algo=copy
+#Table t8d: Covers scenario create partitioned table with one partition in data directory clause and the other partition in default data dir path, move to default dir and alter table force algo=copy
 #Table t9a: Covers scenario create partitioned table without data directory clause but moving each partition to different external directories and alter table force
 #Table t9b: Covers scenario create partitioned table without data directory clause but moving each partition to different external directories and alter table force algo=inplace
 #Table t9c: Covers scenario create partitioned table without data directory clause but moving each partition to different external directories and alter table force algo=copy
@@ -296,18 +297,24 @@ eval CREATE TABLE `t8a` (a INT, b INT) PARTITION BY RANGE(a) (PARTITION p0 VALUE
 eval CREATE TABLE `t8b` (a INT, b INT) PARTITION BY RANGE(a) (PARTITION p0 VALUES LESS THAN (10) $data_directory_clause, PARTITION p1 VALUES LESS THAN (20) $data_directory_clause2);
 --replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR
 eval CREATE TABLE `t8c` (a INT, b INT) PARTITION BY RANGE(a) (PARTITION p0 VALUES LESS THAN (10) $data_directory_clause, PARTITION p1 VALUES LESS THAN (20) $data_directory_clause2);
+--echo # Partition table with one partition in external directory and one in default directory
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR
+eval CREATE TABLE `t8d` (a INT, b INT) PARTITION BY RANGE(a) (PARTITION p0 VALUES LESS THAN (10) $data_directory_clause, PARTITION p1 VALUES LESS THAN (20) );
 
 --echo # Shutdown server
 --source include/shutdown_mysqld.inc
 
---echo # Check that for partition p0 for t8a, t8b, t8c exist in EXTERNAL_DIR and partition p1 for for t8a, t8b, t8c exist in EXTERNAL_DIR2
+--echo # Check that for partition p0 for t8a, t8b, t8c and t8d exist in EXTERNAL_DIR and partition p1 for for t8a, t8b, t8c exist in EXTERNAL_DIR2
 
 --echo # Listing EXTERNAL_DIR
 --list_files $EXTERNAL_DIR/dbtest/ *
 
 --echo # Listing EXTERNAL_DIR2
 --list_files $EXTERNAL_DIR2/dbtest/ *
 
+--echo # List default directory. t8d's p1 partition should be present
+--list_files $MYSQLD_DATADIR/dbtest *
+
 --echo # Restart the server
 --let $innodb_dirs=$EXTERNAL_DIR;$EXTERNAL_DIR2
 let restart_parameters = restart: --innodb-directories=$innodb_dirs;
@@ -318,7 +325,7 @@ let restart_parameters = restart: --innodb-directories=$innodb_dirs;
 ALTER TABLE dbtest.t8a FORCE;
 ALTER TABLE dbtest.t8b FORCE, ALGORITHM=INPLACE;
 ALTER TABLE dbtest.t8c FORCE, ALGORITHM=COPY;
-
+ALTER TABLE dbtest.t8d FORCE, ALGORITHM=COPY;
 --echo # Check that for partition p0 for t8a, t8b, t8c still exist in EXTERNAL_DIR and partition p1 for for t8a, t8b, t8c still exist in EXTERNAL_DIR2
 
 --echo # Listing EXTERNAL_DIR
@@ -327,10 +334,13 @@ ALTER TABLE dbtest.t8c FORCE, ALGORITHM=COPY;
 --echo # Listing EXTERNAL_DIR2
 --list_files $EXTERNAL_DIR2/dbtest/ *
 
+--echo # List default directory. t8d's p1 partition should be present
+--list_files $MYSQLD_DATADIR/dbtest *
+
 --echo # Shutdown server
 --source include/shutdown_mysqld.inc
 
---echo # Now lets move all partitioned tables p0 of t8a, t8b, t8c from EXTERNAL_DIR and partitioned tables p1 of t8a, t8b, t8c from EXTERNAL_DIR2 to default directory
+--echo # Now lets move all partitioned tables p0 of t8a, t8b, t8c, t8d from EXTERNAL_DIR and partitioned tables p1 of t8a, t8b, t8c from EXTERNAL_DIR2 to default directory
 --copy_files_wildcard $EXTERNAL_DIR/dbtest/ $MYSQLD_DATADIR/dbtest/ *
 --remove_files_wildcard $EXTERNAL_DIR/dbtest/ *
 --copy_files_wildcard $EXTERNAL_DIR2/dbtest/ $MYSQLD_DATADIR/dbtest/ *
@@ -347,7 +357,19 @@ ALTER TABLE dbtest.t8a FORCE;
 ALTER TABLE dbtest.t8b FORCE, ALGORITHM=INPLACE;
 ALTER TABLE dbtest.t8c FORCE, ALGORITHM=COPY;
 
---echo # Check that all the partitioned tables for t8a, t8b, t8c now exist in default directory
+--echo # Check that all the partitioned tables for t8a, t8b, t8c and t8d now exist in default directory
+--list_files $MYSQLD_DATADIR/dbtest *
+
+--echo # Check that EXTERNAL_DIR and EXTERNAL_DIR2 are empty
+--echo # Listing EXTERNAL_DIR
+--list_files $EXTERNAL_DIR/dbtest/ *
+
+--echo # Listing EXTERNAL_DIR2
+--list_files $EXTERNAL_DIR2/dbtest/ *
+
+ALTER TABLE dbtest.t8d FORCE, ALGORITHM=COPY;
+
+--echo # Check that all the partitioned tables for t8a, t8b, t8c and t8d now exist in default directory
 --list_files $MYSQLD_DATADIR/dbtest *
 
 --echo # Check that EXTERNAL_DIR and EXTERNAL_DIR2 are empty
@@ -361,6 +383,7 @@ ALTER TABLE dbtest.t8c FORCE, ALGORITHM=COPY;
 DROP TABLE dbtest.t8a;
 DROP TABLE dbtest.t8b;
 DROP TABLE dbtest.t8c;
+DROP TABLE dbtest.t8d;
 
 --echo # Clean up for other tests
 --remove_files_wildcard $EXTERNAL_DIR/dbtest/ *

share/messages_to_error_log.txt

@@ -12301,8 +12301,8 @@ ER_IB_INDEX_PART_TOO_LONG
 ER_DD_UPDATE_DATADIR_FLAG_FAIL
   eng "Failed to update data directory flag for dd table with tablespace id '%llu' when ibd file for '%s' is moved to new location '%s' during server shutdown"
 
-ER_IB_MSG_FIL_STATE_MOVED_PREV_OR_HAS_DATADIR
-  eng "%s DD ID: %llu - Tablespace %u, name '%s', found at '%s' outside default data directory. It is either moved or is created with 'DATA DIRECTORY'"
+ER_IB_MSG_FIL_STATE_MOVED_PREV
+  eng "%s DD ID: %llu - Tablespace %u, name '%s', found at '%s' outside default data directory, but DD misses info about DATA DIRECTORY" 
 
 ER_RPL_KILL_OLD_DUMP_THREAD_ENCOUNTERED
   eng "Upon reconnection with the replica, while initializing the dump thread for %s <%s>, an existing dump thread with the same %s was detected. The source is terminating the previous dump thread (%u), which is normal and expected."

sql/dd/impl/types/table_impl.cc

@@ -783,6 +783,23 @@ Partition *Table_impl::get_partition(Object_id partition_id) {
   return nullptr;
 }
 
+///////////////////////////////////////////////////////////////////////////
+Partition *Table_impl::get_leaf_partition(const std::string &part_name) {
+  return const_cast<Partition *>(
+      std::as_const(*this).get_leaf_partition(part_name));
+}
+
+const Partition *Table_impl::get_leaf_partition(
+    const std::string &part_name) const {
+  for (Partition *leaf : m_leaf_partitions) {
+    if (!my_strcasecmp(system_charset_info, leaf->name().c_str(),
+                       part_name.c_str())) {
+      return leaf;
+    }
+  }
+  return nullptr;
+}
+
 ///////////////////////////////////////////////////////////////////////////
 // Trigger collection.
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/table_impl.h

@@ -402,6 +402,11 @@ class Table_impl : public Abstract_table_impl, virtual public Table {
     return &m_leaf_partitions;
   }
 
+  Partition *get_leaf_partition(const std::string &part_name) override;
+
+  const Partition *get_leaf_partition(
+      const std::string &part_name) const override;
+
   // non-virtual
   void add_leaf_partition(Partition *p) { m_leaf_partitions.push_back(p); }
 

sql/dd/types/table.h

@@ -290,6 +290,11 @@ class Table : virtual public Abstract_table {
 
   virtual Partition_leaf_vector *leaf_partitions() = 0;
 
+  virtual Partition *get_leaf_partition(const std::string &part_name) = 0;
+
+  virtual const Partition *get_leaf_partition(
+      const std::string &part_name) const = 0;
+
   /////////////////////////////////////////////////////////////////////////
   // Trigger collection.
   /////////////////////////////////////////////////////////////////////////

storage/innobase/dict/dict0dd.cc

@@ -35,12 +35,14 @@ Data dictionary interface */
 #include <sql_backup_lock.h>
 #include <sql_class.h>
 #include <sql_thd_internal_api.h>
+
 #include "item.h"
 #else /* !UNIV_HOTBACKUP */
 #include <my_base.h>
 #endif /* !UNIV_HOTBACKUP */
 
 #include <dd/properties.h>
+
 #include "dict0crea.h"
 #include "dict0dd.h"
 #include "dict0dict.h"
@@ -1191,69 +1193,41 @@ dberr_t dd_update_table_and_partitions_after_dir_change(dd::Object_id object_id,
     ut_o(return DB_ERROR);
   }
 
-  ut_ad(dd_table != nullptr);
   std::string dd_table_name{dd_table->table().name()};
   Fil_path fpath{path};
 
-  std::string part_name = (!table_info.subpartition.empty())
-                              ? table_info.subpartition
-                              : table_info.partition;
-  to_lower(part_name);
-
-  /* This function may be hit by those tables which are marked as
-  Fil_state::MOVED_PREV_OR_HAS_DATADIR which includes tables that are moved
-  before 8.0.38/8.4.1/9.0.0 and tables that are created using data directory
-  clause. We want to set the flag for tables moved before 8.0.38/8.4.1/9.0.0
-  only and ignore those tables which are created using data directory clause
-  as the dd_table data dir flag is already set for them. Additionally, we
-  explicitly remove the flag if moved from external to default data dir. We
-  do this because dd_table data dir flag should not exist if the ibd file is
-  located in default dir */
-  if (part_name.empty()) {
-    /* The table is non-partitioned table */
-    bool dd_flag = dd_table->se_private_data().exists(
-        dd_table_key_strings[DD_TABLE_DATA_DIRECTORY]);
-    bool set_true = !(MySQL_datadir_path.is_same_as(fpath) ||
-                      MySQL_datadir_path.is_ancestor(fpath)) &&
-                    !dd_flag;
-    bool set_false = MySQL_datadir_path.is_same_as(fpath) ||
-                     MySQL_datadir_path.is_ancestor(fpath);
+  bool set_true = !MySQL_datadir_path.is_ancestor(fpath);
+  if (!dd_table_is_partitioned(*dd_table)) {
+    /* Set the DATA DIRECTORY FLAG to true for dd table if ibd file is moved to
+    directory other than default data dir. Remove the flag if moved from
+    external to default data dir.*/
     if (set_true) {
       dd_table->se_private_data().set(
           dd_table_key_strings[DD_TABLE_DATA_DIRECTORY], set_true);
-    }
-    if (set_false) {
+    } else {
       dd_table->se_private_data().remove(
           dd_table_key_strings[DD_TABLE_DATA_DIRECTORY]);
     }
   } else {
-    /* The table is partitioned table */
-    for (dd::Partition *part_obj : *dd_table->leaf_partitions()) {
-      std::string part_obj_name{part_obj->name()};
-      to_lower(part_obj_name);
-      if (part_obj_name == part_name) {
-        dd::Properties &part_options = part_obj->options();
-        bool dd_flag = part_obj->se_private_data().exists(
-            dd_table_key_strings[DD_TABLE_DATA_DIRECTORY]);
-        bool set_true = !(MySQL_datadir_path.is_same_as(fpath) ||
-                          MySQL_datadir_path.is_ancestor(fpath)) &&
-                        !dd_flag;
-        bool set_false = MySQL_datadir_path.is_same_as(fpath) ||
-                         MySQL_datadir_path.is_ancestor(fpath);
-        if (set_true) {
-          part_obj->se_private_data().set(
-              dd_table_key_strings[DD_TABLE_DATA_DIRECTORY], set_true);
-          /* Update data_file_name for dd::partition as we do not set data
-          directory for whole partitioned table. We acquire dd::partition later
-          and read from it*/
-          part_options.set(data_file_name_key, table_info.directory.c_str());
-        }
-        if (set_false) {
-          part_obj->se_private_data().remove(
-              dd_table_key_strings[DD_TABLE_DATA_DIRECTORY]);
-          part_options.remove(data_file_name_key);
-        }
-      }
+    dd::Partition *part_obj = dd_table->get_leaf_partition(
+        (!table_info.subpartition.empty()) ? table_info.subpartition
+                                           : table_info.partition);
+    if (!part_obj) {
+      ut_d(ut_error);
+      ut_o(return DB_ERROR);
+    }
+    dd::Properties &part_options = part_obj->options();
+    if (set_true) {
+      part_obj->se_private_data().set(
+          dd_table_key_strings[DD_TABLE_DATA_DIRECTORY], true);
+      /* Update data_file_name for dd::partition as we do not set data
+      directory for whole partitioned table. We acquire dd::partition later
+      and read from it*/
+      part_options.set(data_file_name_key, table_info.directory.c_str());
+    } else {
+      part_obj->se_private_data().remove(
+          dd_table_key_strings[DD_TABLE_DATA_DIRECTORY]);
+      part_options.remove(data_file_name_key);
     }
   }
 
@@ -7544,16 +7518,27 @@ std::optional<table_name_components> parse_tablespace_path(std::string path) {
   table_info.table_name = temp_table.substr(0, hash_pos);
   file_to_table(table_info.table_name, false);
 
+  std::string part_seperator(PART_SEPARATOR);
+  std::string sub_part_seperator(SUB_PART_SEPARATOR);
+
   // Check for partitions and subpartitions
   bool has_partitions = temp_table.find(PART_SEPARATOR) != std::string::npos;
+  if (!has_partitions) {
+    has_partitions = temp_table.find(ALT_PART_SEPARATOR) != std::string::npos;
+    if (has_partitions) {
+      part_seperator = ALT_PART_SEPARATOR;
+      sub_part_seperator = ALT_SUB_PART_SEPARATOR;
+    }
+  }
+
   bool has_subpartitions =
-      temp_table.find(SUB_PART_SEPARATOR) != std::string::npos;
+      temp_table.find(sub_part_seperator) != std::string::npos;
 
   if (has_partitions) {
     // Extract partition name
     size_t part_start =
-        temp_table.find(PART_SEPARATOR) + std::string(PART_SEPARATOR).length();
-    size_t part_end = has_subpartitions ? temp_table.find(SUB_PART_SEPARATOR)
+        temp_table.find(part_seperator) + std::string(part_seperator).length();
+    size_t part_end = has_subpartitions ? temp_table.find(sub_part_seperator)
                                         : temp_table.find('.');
 
     ut_ad(part_end != std::string::npos);
@@ -7565,8 +7550,8 @@ std::optional<table_name_components> parse_tablespace_path(std::string path) {
 
   if (has_subpartitions) {
     // Extract subpartition name
-    size_t sub_part_start = temp_table.find(SUB_PART_SEPARATOR) +
-                            std::string(SUB_PART_SEPARATOR).length();
+    size_t sub_part_start = temp_table.find(sub_part_seperator) +
+                            std::string(sub_part_seperator).length();
     size_t sub_part_end = temp_table.find('.');
     ut_ad(sub_part_end != std::string::npos);
     std::string temp_subpartition =