Support disabling indexes for MySQL and MariaDB

For MySQL version 8.0.0 and above, and MariaDB version 10.6.0 and above
Allows to disable an index to be ignored by queries while still
being maintained.

Author: mtaner

activerecord/CHANGELOG.md

@@ -1,3 +1,41 @@
+*   Support disabling indexes for MySQL v8.0.0+ and MariaDB v10.6.0+
+
+    MySQL 8.0.0 added an option to disable indexes from being used by the query
+    optimizer by making them "invisible". This allows the index to still be maintained
+    and updated but no queries will be permitted to use it. This can be useful for adding
+    new invisible indexes or making existing indexes invisible before dropping them
+    to ensure queries are not negatively affected.
+    See https://dev.mysql.com/blog-archive/mysql-8-0-invisible-indexes/ for more details.
+
+    MariaDB 10.6.0 also added support for this feature by allowing indexes to be "ignored"
+    in queries. See https://mariadb.com/kb/en/ignored-indexes/ for more details.
+
+    Active Record now supports this option for MySQL 8.0.0+ and MariaDB 10.6.0+ for
+    index creation and alteration where the new index option `enabled: true/false` can be
+    passed to column and index methods as below:
+
+    ```ruby
+    add_index :users, :email, enabled: false
+    enable_index :users, :email
+    add_column :users, :dob, :string, index: { enabled: false }
+
+    change_table :users do |t|
+      t.index :name, enabled: false
+      t.index :dob
+      t.disable_index :dob
+      t.column :username, :string, index: { enabled: false }
+      t.references :account, index: { enabled: false }
+    end
+
+    create_table :users do |t|
+      t.string :name, index: { enabled: false }
+      t.string :email
+      t.index :email, enabled: false
+    end
+    ```
+
+    *Merve Taner*
+
 *   Respect `implicit_order_column` in `ActiveRecord::Relation#reverse_order`.
 
     *Joshua Young*

activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb

@@ -912,6 +912,19 @@ def rename_column(table_name, column_name, new_column_name)
       # Concurrently adding an index is not supported in a transaction.
       #
       # For more information see the {"Transactional Migrations" section}[rdoc-ref:Migration].
+      #
+      # ====== Creating an index that is not used by queries
+      #
+      #   add_index(:developers, :name, enabled: false)
+      #
+      # generates:
+      #
+      #   CREATE INDEX index_developers_on_name ON developers (name) INVISIBLE -- MySQL
+      #
+      #   CREATE INDEX index_developers_on_name ON developers (name) IGNORED -- MariaDB
+      #
+      # Note: only supported by MySQL version 8.0.0 and greater, and MariaDB version 10.6.0 and greater.
+      #
       def add_index(table_name, column_name, **options)
         create_index = build_create_index_definition(table_name, column_name, **options)
         execute schema_creation.accept(create_index)
@@ -1475,7 +1488,7 @@ def update_table_definition(table_name, base) # :nodoc:
       end
 
       def add_index_options(table_name, column_name, name: nil, if_not_exists: false, internal: false, **options) # :nodoc:
-        options.assert_valid_keys(:unique, :length, :order, :opclass, :where, :type, :using, :comment, :algorithm, :include, :nulls_not_distinct)
+        options.assert_valid_keys(valid_index_options)
 
         column_names = index_column_names(column_name)
 
@@ -1484,7 +1497,7 @@ def add_index_options(table_name, column_name, name: nil, if_not_exists: false,
 
         validate_index_length!(table_name, index_name, internal)
 
-        index = IndexDefinition.new(
+        index = create_index_definition(
           table_name, index_name,
           options[:unique],
           column_names,
@@ -1539,6 +1552,20 @@ def change_column_comment(table_name, column_name, comment_or_changes)
         raise NotImplementedError, "#{self.class} does not support changing column comments"
       end
 
+      # Enables an index to be used by queries.
+      #
+      #   enable_index(:users, :email)
+      def enable_index(table_name, index_name)
+        raise NotImplementedError, "#{self.class} does not support enabling indexes"
+      end
+
+      # Prevents an index from being used by queries.
+      #
+      #   disable_index(:users, :email)
+      def disable_index(table_name, index_name)
+        raise NotImplementedError, "#{self.class} does not support disabling indexes"
+      end
+
       def create_schema_dumper(options) # :nodoc:
         SchemaDumper.create(self, options)
       end
@@ -1627,6 +1654,10 @@ def add_index_sort_order(quoted_columns, **options)
           end
         end
 
+        def valid_index_options
+          [:unique, :length, :order, :opclass, :where, :type, :using, :comment, :algorithm, :include, :nulls_not_distinct]
+        end
+
         def options_for_index_columns(options)
           if options.is_a?(Hash)
             options.symbolize_keys
@@ -1703,6 +1734,10 @@ def create_table_definition(name, **options)
           TableDefinition.new(self, name, **options)
         end
 
+        def create_index_definition(table_name, name, unique, columns, **options)
+          IndexDefinition.new(table_name, name, unique, columns, **options)
+        end
+
         def create_alter_table(name)
           AlterTable.new create_table_definition(name)
         end

activerecord/lib/active_record/connection_adapters/abstract_adapter.rb

@@ -555,6 +555,10 @@ def supports_nulls_not_distinct?
         false
       end
 
+      def supports_disabling_indexes?
+        false
+      end
+
       def return_value_after_insert?(column) # :nodoc:
         column.auto_populated?
       end

activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb

@@ -178,6 +178,16 @@ def return_value_after_insert?(column) # :nodoc:
         supports_insert_returning? ? column.auto_populated? : column.auto_increment?
       end
 
+      # See https://dev.mysql.com/doc/refman/8.0/en/invisible-indexes.html for more details on MySQL feature.
+      # See https://mariadb.com/kb/en/ignored-indexes/ for more details on the MariaDB feature.
+      def supports_disabling_indexes?
+        if mariadb?
+          database_version >= "10.6.0"
+        else
+          database_version >= "8.0.0"
+        end
+      end
+
       def get_advisory_lock(lock_name, timeout = 0) # :nodoc:
         query_value("SELECT GET_LOCK(#{quote(lock_name.to_s)}, #{timeout})") == 1
       end
@@ -457,6 +467,24 @@ def build_create_index_definition(table_name, column_name, **options) # :nodoc:
         CreateIndexDefinition.new(index, algorithm)
       end
 
+      def enable_index(table_name, index_name) # :nodoc:
+        raise NotImplementedError unless supports_disabling_indexes?
+
+        query = <<~SQL
+          ALTER TABLE #{quote_table_name(table_name)} ALTER INDEX #{index_name} #{mariadb? ? "NOT IGNORED" : "VISIBLE"}
+        SQL
+        execute(query)
+      end
+
+      def disable_index(table_name, index_name) # :nodoc:
+        raise NotImplementedError unless supports_disabling_indexes?
+
+        query = <<~SQL
+          ALTER TABLE #{quote_table_name(table_name)} ALTER INDEX #{index_name} #{mariadb? ? "IGNORED" : "INVISIBLE"}
+        SQL
+        execute(query)
+      end
+
       def add_sql_comment!(sql, comment) # :nodoc:
         sql << " COMMENT #{quote(comment)}" if comment.present?
         sql

activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb

@@ -49,6 +49,8 @@ def visit_IndexDefinition(o, create = false)
             sql << "USING #{o.using}" if o.using
             sql << "ON #{quote_table_name(o.table)}" if create
             sql << "(#{quoted_columns(o)})"
+            sql << "INVISIBLE" if o.disabled? && !mariadb?
+            sql << "IGNORED" if o.disabled? && mariadb?
 
             add_sql_comment!(sql.join(" "), o.comment)
           end

activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb

@@ -50,6 +50,25 @@ module ColumnMethods
         deprecate :unsigned_float, :unsigned_decimal, deprecator: ActiveRecord.deprecator
       end
 
+      # = Active Record MySQL Adapter \Index Definition
+      class IndexDefinition < ActiveRecord::ConnectionAdapters::IndexDefinition
+        attr_accessor :enabled
+
+        def initialize(*args, **kwargs)
+          @enabled = kwargs.key?(:enabled) ? kwargs.delete(:enabled) : true
+          super
+        end
+
+        def defined_for?(columns = nil, name: nil, unique: nil, valid: nil, include: nil, nulls_not_distinct: nil, enabled: nil, **options)
+          super(columns, name:, unique:, valid:, include:, nulls_not_distinct:, **options) &&
+            (enabled.nil? || self.enabled == enabled)
+        end
+
+        def disabled?
+          !@enabled
+        end
+      end
+
       # = Active Record MySQL Adapter \Table Definition
       class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
         include ColumnMethods
@@ -99,6 +118,28 @@ def integer_like_primary_key_type(type, options)
       # = Active Record MySQL Adapter \Table
       class Table < ActiveRecord::ConnectionAdapters::Table
         include ColumnMethods
+
+        # Enables an index to be used by query optimizers.
+        #
+        #   t.enable_index(:email)
+        #
+        # Note: only supported by MySQL version 8.0.0 and greater, and MariaDB version 10.6.0 and greater.
+        #
+        # See {connection.enable_index}[rdoc-ref:SchemaStatements#enable_index]
+        def enable_index(index_name)
+          @base.enable_index(name, index_name)
+        end
+
+        # Disables an index not to be used by query optimizers.
+        #
+        #   t.disable_index(:email)
+        #
+        # Note: only supported by MySQL version 8.0.0 and greater, and MariaDB version 10.6.0 and greater.
+        #
+        # See {connection.disable_index}[rdoc-ref:SchemaStatements#disable_index]
+        def disable_index(index_name)
+          @base.disable_index(name, index_name)
+        end
       end
     end
   end

activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb

@@ -21,7 +21,7 @@ def indexes(table_name)
                 index_using = mysql_index_type
               end
 
-              indexes << [
+              index = [
                 row["Table"],
                 row["Key_name"],
                 row["Non_unique"].to_i == 0,
@@ -30,8 +30,14 @@ def indexes(table_name)
                 orders: {},
                 type: index_type,
                 using: index_using,
-                comment: row["Index_comment"].presence
+                comment: row["Index_comment"].presence,
               ]
+
+              if supports_disabling_indexes?
+                index[-1][:enabled] = mariadb? ? row["Ignored"] == "NO" : row["Visible"] == "YES"
+              end
+
+              indexes << index
             end
 
             if expression = row["Expression"]
@@ -63,8 +69,7 @@ def indexes(table_name)
                 columns, order: orders, length: lengths
               ).values.join(", ")
             end
-
-            IndexDefinition.new(*index, **options)
+            MySQL::IndexDefinition.new(*index, **options)
           end
         rescue StatementInvalid => e
           if e.message.match?(/Table '.+' doesn't exist/)
@@ -74,6 +79,16 @@ def indexes(table_name)
           end
         end
 
+        def create_index_definition(table_name, name, unique, columns, **options)
+          MySQL::IndexDefinition.new(table_name, name, unique, columns, **options)
+        end
+
+        def add_index_options(table_name, column_name, name: nil, if_not_exists: false, internal: false, **options) # :nodoc:
+          index, algorithm, if_not_exists = super
+          index.enabled = options[:enabled] unless options[:enabled].nil?
+          [index, algorithm, if_not_exists]
+        end
+
         def remove_column(table_name, column_name, type = nil, **options)
           if foreign_key_exists?(table_name, column: column_name)
             remove_foreign_key(table_name, column: column_name)
@@ -234,6 +249,12 @@ def add_index_length(quoted_columns, **options)
             end
           end
 
+          def valid_index_options
+            index_options = super
+            index_options << :enabled if supports_disabling_indexes?
+            index_options
+          end
+
           def add_options_for_index_columns(quoted_columns, **options)
             quoted_columns = add_index_length(quoted_columns, **options)
             super

activerecord/lib/active_record/migration/command_recorder.rb

@@ -44,6 +44,8 @@ class Migration
     # * rename_enum_value (must supply a +:from+ and +:to+ option)
     # * rename_index
     # * rename_table
+    # * enable_index
+    # * disable_index
     class CommandRecorder
       ReversibleAndIrreversibleMethods = [
         :create_table, :create_join_table, :rename_table, :add_column, :remove_column,
@@ -58,7 +60,8 @@ class CommandRecorder
         :add_unique_constraint, :remove_unique_constraint,
         :create_enum, :drop_enum, :rename_enum, :add_enum_value, :rename_enum_value,
         :create_schema, :drop_schema,
-        :create_virtual_table, :drop_virtual_table
+        :create_virtual_table, :drop_virtual_table,
+        :enable_index, :disable_index
       ]
       include JoinTable
 
@@ -183,6 +186,16 @@ def invert_#{method}(args, &block)    # def invert_create_table(args, &block)
 
         include StraightReversions
 
+        def invert_enable_index(args)
+          table_name, index_name = args
+          [:disable_index, [table_name, index_name]]
+        end
+
+        def invert_disable_index(args)
+          table_name, index_name = args
+          [:enable_index, [table_name, index_name]]
+        end
+
         def invert_transaction(args, &block)
           sub_recorder = CommandRecorder.new(delegate)
           sub_recorder.revert(&block)

activerecord/lib/active_record/schema_dumper.rb

@@ -277,6 +277,7 @@ def index_parts(index)
         index_parts << "nulls_not_distinct: #{index.nulls_not_distinct.inspect}" if index.nulls_not_distinct
         index_parts << "type: #{index.type.inspect}" if index.type
         index_parts << "comment: #{index.comment.inspect}" if index.comment
+        index_parts << "enabled: #{index.enabled.inspect}" if @connection.supports_disabling_indexes? && index.disabled?
         index_parts
       end
 

activerecord/test/cases/active_record_schema_test.rb

@@ -18,6 +18,7 @@ class ActiveRecordSchemaTest < ActiveRecord::TestCase
     @connection.drop_table :fruits rescue nil
     @connection.drop_table :has_timestamps rescue nil
     @connection.drop_table :multiple_indexes rescue nil
+    @connection.drop_table :disabled_index rescue nil
     @schema_migration.delete_all_versions
     ActiveRecord::Migration.verbose = @original_verbose
   end
@@ -120,6 +121,20 @@ def test_schema_load_with_multiple_indexes_for_column_of_different_names
     assert_equal ["multiple_indexes_foo_1", "multiple_indexes_foo_2"], indexes.collect(&:name).sort
   end
 
+  if ActiveRecord::Base.lease_connection.supports_disabling_indexes?
+    def test_schema_load_for_index_visibility
+      ActiveRecord::Schema.define do
+        create_table :disabled_index do |t|
+          t.string "foo"
+          t.index ["foo"], name: "disabled_foo_index", enabled: false
+        end
+      end
+
+      indexes = @connection.indexes("disabled_index").find { |index| index.name == "disabled_foo_index" }
+      assert_predicate indexes, :disabled?
+    end
+  end
+
   if current_adapter?(:PostgreSQLAdapter)
     def test_timestamps_with_and_without_zones
       ActiveRecord::Schema.define do