Merge pull request rails#54332 from Shopify/mt/support-mysql-index-visibility

Add support for index visibility for MySQL v8.0.0+ and MariaDB v10.6.0+

Author: gmcgibbon

activerecord/CHANGELOG.md

@@ -1,3 +1,41 @@
+*   Support disabling indexes for MySQL v8.0.0+ and MariaDB v10.6.0+
+
+    MySQL 8.0.0 added an option to disable indexes from being used by the query
+    optimizer by making them "invisible". This allows the index to still be maintained
+    and updated but no queries will be permitted to use it. This can be useful for adding
+    new invisible indexes or making existing indexes invisible before dropping them
+    to ensure queries are not negatively affected.
+    See https://dev.mysql.com/blog-archive/mysql-8-0-invisible-indexes/ for more details.
+
+    MariaDB 10.6.0 also added support for this feature by allowing indexes to be "ignored"
+    in queries. See https://mariadb.com/kb/en/ignored-indexes/ for more details.
+
+    Active Record now supports this option for MySQL 8.0.0+ and MariaDB 10.6.0+ for
+    index creation and alteration where the new index option `enabled: true/false` can be
+    passed to column and index methods as below:
+
+    ```ruby
+    add_index :users, :email, enabled: false
+    enable_index :users, :email
+    add_column :users, :dob, :string, index: { enabled: false }
+
+    change_table :users do |t|
+      t.index :name, enabled: false
+      t.index :dob
+      t.disable_index :dob
+      t.column :username, :string, index: { enabled: false }
+      t.references :account, index: { enabled: false }
+    end
+
+    create_table :users do |t|
+      t.string :name, index: { enabled: false }
+      t.string :email
+      t.index :email, enabled: false
+    end
+    ```
+
+    *Merve Taner*
+
 *   Respect `implicit_order_column` in `ActiveRecord::Relation#reverse_order`.
 
     *Joshua Young*

activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb

@@ -912,6 +912,19 @@ def rename_column(table_name, column_name, new_column_name)
       # Concurrently adding an index is not supported in a transaction.
       #
       # For more information see the {"Transactional Migrations" section}[rdoc-ref:Migration].
+      #
+      # ====== Creating an index that is not used by queries
+      #
+      #   add_index(:developers, :name, enabled: false)
+      #
+      # generates:
+      #
+      #   CREATE INDEX index_developers_on_name ON developers (name) INVISIBLE -- MySQL
+      #
+      #   CREATE INDEX index_developers_on_name ON developers (name) IGNORED -- MariaDB
+      #
+      # Note: only supported by MySQL version 8.0.0 and greater, and MariaDB version 10.6.0 and greater.
+      #
       def add_index(table_name, column_name, **options)
         create_index = build_create_index_definition(table_name, column_name, **options)
         execute schema_creation.accept(create_index)
@@ -1475,7 +1488,7 @@ def update_table_definition(table_name, base) # :nodoc:
       end
 
       def add_index_options(table_name, column_name, name: nil, if_not_exists: false, internal: false, **options) # :nodoc:
-        options.assert_valid_keys(:unique, :length, :order, :opclass, :where, :type, :using, :comment, :algorithm, :include, :nulls_not_distinct)
+        options.assert_valid_keys(valid_index_options)
 
         column_names = index_column_names(column_name)
 
@@ -1484,7 +1497,7 @@ def add_index_options(table_name, column_name, name: nil, if_not_exists: false,
 
         validate_index_length!(table_name, index_name, internal)
 
-        index = IndexDefinition.new(
+        index = create_index_definition(
           table_name, index_name,
           options[:unique],
           column_names,
@@ -1539,6 +1552,20 @@ def change_column_comment(table_name, column_name, comment_or_changes)
         raise NotImplementedError, "#{self.class} does not support changing column comments"
       end
 
+      # Enables an index to be used by queries.
+      #
+      #   enable_index(:users, :email)
+      def enable_index(table_name, index_name)
+        raise NotImplementedError, "#{self.class} does not support enabling indexes"
+      end
+
+      # Prevents an index from being used by queries.
+      #
+      #   disable_index(:users, :email)
+      def disable_index(table_name, index_name)
+        raise NotImplementedError, "#{self.class} does not support disabling indexes"
+      end
+
       def create_schema_dumper(options) # :nodoc:
         SchemaDumper.create(self, options)
       end
@@ -1627,6 +1654,10 @@ def add_index_sort_order(quoted_columns, **options)
           end
         end
 
+        def valid_index_options
+          [:unique, :length, :order, :opclass, :where, :type, :using, :comment, :algorithm, :include, :nulls_not_distinct]
+        end
+
         def options_for_index_columns(options)
           if options.is_a?(Hash)
             options.symbolize_keys
@@ -1703,6 +1734,10 @@ def create_table_definition(name, **options)
           TableDefinition.new(self, name, **options)
         end
 
+        def create_index_definition(table_name, name, unique, columns, **options)
+          IndexDefinition.new(table_name, name, unique, columns, **options)
+        end
+
         def create_alter_table(name)
           AlterTable.new create_table_definition(name)
         end

activerecord/lib/active_record/connection_adapters/abstract_adapter.rb

@@ -555,6 +555,10 @@ def supports_nulls_not_distinct?
         false
       end
 
+      def supports_disabling_indexes?
+        false
+      end
+
       def return_value_after_insert?(column) # :nodoc:
         column.auto_populated?
       end

activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb

@@ -178,6 +178,16 @@ def return_value_after_insert?(column) # :nodoc:
         supports_insert_returning? ? column.auto_populated? : column.auto_increment?
       end
 
+      # See https://dev.mysql.com/doc/refman/8.0/en/invisible-indexes.html for more details on MySQL feature.
+      # See https://mariadb.com/kb/en/ignored-indexes/ for more details on the MariaDB feature.
+      def supports_disabling_indexes?
+        if mariadb?
+          database_version >= "10.6.0"
+        else
+          database_version >= "8.0.0"
+        end
+      end
+
       def get_advisory_lock(lock_name, timeout = 0) # :nodoc:
         query_value("SELECT GET_LOCK(#{quote(lock_name.to_s)}, #{timeout})") == 1
       end
@@ -457,6 +467,24 @@ def build_create_index_definition(table_name, column_name, **options) # :nodoc:
         CreateIndexDefinition.new(index, algorithm)
       end
 
+      def enable_index(table_name, index_name) # :nodoc:
+        raise NotImplementedError unless supports_disabling_indexes?
+
+        query = <<~SQL
+          ALTER TABLE #{quote_table_name(table_name)} ALTER INDEX #{index_name} #{mariadb? ? "NOT IGNORED" : "VISIBLE"}
+        SQL
+        execute(query)
+      end
+
+      def disable_index(table_name, index_name) # :nodoc:
+        raise NotImplementedError unless supports_disabling_indexes?
+
+        query = <<~SQL
+          ALTER TABLE #{quote_table_name(table_name)} ALTER INDEX #{index_name} #{mariadb? ? "IGNORED" : "INVISIBLE"}
+        SQL
+        execute(query)
+      end
+
       def add_sql_comment!(sql, comment) # :nodoc:
         sql << " COMMENT #{quote(comment)}" if comment.present?
         sql

activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb

@@ -49,6 +49,8 @@ def visit_IndexDefinition(o, create = false)
             sql << "USING #{o.using}" if o.using
             sql << "ON #{quote_table_name(o.table)}" if create
             sql << "(#{quoted_columns(o)})"
+            sql << "INVISIBLE" if o.disabled? && !mariadb?
+            sql << "IGNORED" if o.disabled? && mariadb?
 
             add_sql_comment!(sql.join(" "), o.comment)
           end

activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb

@@ -50,6 +50,25 @@ module ColumnMethods
         deprecate :unsigned_float, :unsigned_decimal, deprecator: ActiveRecord.deprecator
       end
 
+      # = Active Record MySQL Adapter \Index Definition
+      class IndexDefinition < ActiveRecord::ConnectionAdapters::IndexDefinition
+        attr_accessor :enabled
+
+        def initialize(*args, **kwargs)
+          @enabled = kwargs.key?(:enabled) ? kwargs.delete(:enabled) : true
+          super
+        end
+
+        def defined_for?(columns = nil, name: nil, unique: nil, valid: nil, include: nil, nulls_not_distinct: nil, enabled: nil, **options)
+          super(columns, name:, unique:, valid:, include:, nulls_not_distinct:, **options) &&
+            (enabled.nil? || self.enabled == enabled)
+        end
+
+        def disabled?
+          !@enabled
+        end
+      end
+
       # = Active Record MySQL Adapter \Table Definition
       class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
         include ColumnMethods
@@ -99,6 +118,28 @@ def integer_like_primary_key_type(type, options)
       # = Active Record MySQL Adapter \Table
       class Table < ActiveRecord::ConnectionAdapters::Table
         include ColumnMethods
+
+        # Enables an index to be used by query optimizers.
+        #
+        #   t.enable_index(:email)
+        #
+        # Note: only supported by MySQL version 8.0.0 and greater, and MariaDB version 10.6.0 and greater.
+        #
+        # See {connection.enable_index}[rdoc-ref:SchemaStatements#enable_index]
+        def enable_index(index_name)
+          @base.enable_index(name, index_name)
+        end
+
+        # Disables an index not to be used by query optimizers.
+        #
+        #   t.disable_index(:email)
+        #
+        # Note: only supported by MySQL version 8.0.0 and greater, and MariaDB version 10.6.0 and greater.
+        #
+        # See {connection.disable_index}[rdoc-ref:SchemaStatements#disable_index]
+        def disable_index(index_name)
+          @base.disable_index(name, index_name)
+        end
       end
     end
   end

activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb

@@ -21,7 +21,7 @@ def indexes(table_name)
                 index_using = mysql_index_type
               end
 
-              indexes << [
+              index = [
                 row["Table"],
                 row["Key_name"],
                 row["Non_unique"].to_i == 0,
@@ -30,8 +30,14 @@ def indexes(table_name)
                 orders: {},
                 type: index_type,
                 using: index_using,
-                comment: row["Index_comment"].presence
+                comment: row["Index_comment"].presence,
               ]
+
+              if supports_disabling_indexes?
+                index[-1][:enabled] = mariadb? ? row["Ignored"] == "NO" : row["Visible"] == "YES"
+              end
+
+              indexes << index
             end
 
             if expression = row["Expression"]
@@ -63,8 +69,7 @@ def indexes(table_name)
                 columns, order: orders, length: lengths
               ).values.join(", ")
             end
-
-            IndexDefinition.new(*index, **options)
+            MySQL::IndexDefinition.new(*index, **options)
           end
         rescue StatementInvalid => e
           if e.message.match?(/Table '.+' doesn't exist/)
@@ -74,6 +79,16 @@ def indexes(table_name)
           end
         end
 
+        def create_index_definition(table_name, name, unique, columns, **options)
+          MySQL::IndexDefinition.new(table_name, name, unique, columns, **options)
+        end
+
+        def add_index_options(table_name, column_name, name: nil, if_not_exists: false, internal: false, **options) # :nodoc:
+          index, algorithm, if_not_exists = super
+          index.enabled = options[:enabled] unless options[:enabled].nil?
+          [index, algorithm, if_not_exists]
+        end
+
         def remove_column(table_name, column_name, type = nil, **options)
           if foreign_key_exists?(table_name, column: column_name)
             remove_foreign_key(table_name, column: column_name)
@@ -234,6 +249,12 @@ def add_index_length(quoted_columns, **options)
             end
           end
 
+          def valid_index_options
+            index_options = super
+            index_options << :enabled if supports_disabling_indexes?
+            index_options
+          end
+
           def add_options_for_index_columns(quoted_columns, **options)
             quoted_columns = add_index_length(quoted_columns, **options)
             super

activerecord/lib/active_record/migration/command_recorder.rb

@@ -44,6 +44,8 @@ class Migration
     # * rename_enum_value (must supply a +:from+ and +:to+ option)
     # * rename_index
     # * rename_table
+    # * enable_index
+    # * disable_index
     class CommandRecorder
       ReversibleAndIrreversibleMethods = [
         :create_table, :create_join_table, :rename_table, :add_column, :remove_column,
@@ -58,7 +60,8 @@ class CommandRecorder
         :add_unique_constraint, :remove_unique_constraint,
         :create_enum, :drop_enum, :rename_enum, :add_enum_value, :rename_enum_value,
         :create_schema, :drop_schema,
-        :create_virtual_table, :drop_virtual_table
+        :create_virtual_table, :drop_virtual_table,
+        :enable_index, :disable_index
       ]
       include JoinTable
 
@@ -183,6 +186,16 @@ def invert_#{method}(args, &block)    # def invert_create_table(args, &block)
 
         include StraightReversions
 
+        def invert_enable_index(args)
+          table_name, index_name = args
+          [:disable_index, [table_name, index_name]]
+        end
+
+        def invert_disable_index(args)
+          table_name, index_name = args
+          [:enable_index, [table_name, index_name]]
+        end
+
         def invert_transaction(args, &block)
           sub_recorder = CommandRecorder.new(delegate)
           sub_recorder.revert(&block)

activerecord/lib/active_record/schema_dumper.rb

@@ -277,6 +277,7 @@ def index_parts(index)
         index_parts << "nulls_not_distinct: #{index.nulls_not_distinct.inspect}" if index.nulls_not_distinct
         index_parts << "type: #{index.type.inspect}" if index.type
         index_parts << "comment: #{index.comment.inspect}" if index.comment
+        index_parts << "enabled: #{index.enabled.inspect}" if @connection.supports_disabling_indexes? && index.disabled?
         index_parts
       end
 

activerecord/test/cases/active_record_schema_test.rb

@@ -18,6 +18,7 @@ class ActiveRecordSchemaTest < ActiveRecord::TestCase
     @connection.drop_table :fruits rescue nil
     @connection.drop_table :has_timestamps rescue nil
     @connection.drop_table :multiple_indexes rescue nil
+    @connection.drop_table :disabled_index rescue nil
     @schema_migration.delete_all_versions
     ActiveRecord::Migration.verbose = @original_verbose
   end
@@ -120,6 +121,20 @@ def test_schema_load_with_multiple_indexes_for_column_of_different_names
     assert_equal ["multiple_indexes_foo_1", "multiple_indexes_foo_2"], indexes.collect(&:name).sort
   end
 
+  if ActiveRecord::Base.lease_connection.supports_disabling_indexes?
+    def test_schema_load_for_index_visibility
+      ActiveRecord::Schema.define do
+        create_table :disabled_index do |t|
+          t.string "foo"
+          t.index ["foo"], name: "disabled_foo_index", enabled: false
+        end
+      end
+
+      indexes = @connection.indexes("disabled_index").find { |index| index.name == "disabled_foo_index" }
+      assert_predicate indexes, :disabled?
+    end
+  end
+
   if current_adapter?(:PostgreSQLAdapter)
     def test_timestamps_with_and_without_zones
       ActiveRecord::Schema.define do