Bigint migration step3 for remaining tables (#4591)

jochenehret · web-flow · commit ccfba1e0762a · 2025-10-13T08:03:51.000+02:00
* step 3a and b for remaining tables: jobs, delayed_jobs, app_usage_events, service_usage_events * see also #4406
diff --git a/db/migrations/20250930135451_bigint_migration_delayed_jobs_step3a.rb b/db/migrations/20250930135451_bigint_migration_delayed_jobs_step3a.rb
@@ -0,0 +1,6 @@
+require 'database/bigint_migration_step3ab'
+
+Sequel.migration do
+  no_transaction
+  VCAP::BigintMigration.step3a_migration(self, :delayed_jobs)
+end
diff --git a/db/migrations/20250930135459_bigint_migration_delayed_jobs_step3b.rb b/db/migrations/20250930135459_bigint_migration_delayed_jobs_step3b.rb
@@ -0,0 +1,6 @@
+require 'database/bigint_migration_step3ab'
+
+Sequel.migration do
+  no_transaction
+  VCAP::BigintMigration.step3b_migration(self, :delayed_jobs)
+end
diff --git a/db/migrations/20250930135517_bigint_migration_jobs_step3a.rb b/db/migrations/20250930135517_bigint_migration_jobs_step3a.rb
@@ -0,0 +1,6 @@
+require 'database/bigint_migration_step3ab'
+
+Sequel.migration do
+  no_transaction
+  VCAP::BigintMigration.step3a_migration(self, :jobs)
+end
diff --git a/db/migrations/20250930135527_bigint_migration_jobs_step3b.rb b/db/migrations/20250930135527_bigint_migration_jobs_step3b.rb
@@ -0,0 +1,6 @@
+require 'database/bigint_migration_step3ab'
+
+Sequel.migration do
+  no_transaction
+  VCAP::BigintMigration.step3b_migration(self, :jobs)
+end
diff --git a/db/migrations/20250930135548_bigint_migration_app_usage_events_step3a.rb b/db/migrations/20250930135548_bigint_migration_app_usage_events_step3a.rb
@@ -0,0 +1,6 @@
+require 'database/bigint_migration_step3ab'
+
+Sequel.migration do
+  no_transaction
+  VCAP::BigintMigration.step3a_migration(self, :app_usage_events)
+end
diff --git a/db/migrations/20250930135554_bigint_migration_app_usage_events_step3b.rb b/db/migrations/20250930135554_bigint_migration_app_usage_events_step3b.rb
@@ -0,0 +1,6 @@
+require 'database/bigint_migration_step3ab'
+
+Sequel.migration do
+  no_transaction
+  VCAP::BigintMigration.step3b_migration(self, :app_usage_events)
+end
diff --git a/db/migrations/20250930135612_bigint_migration_service_usage_events_step3a.rb b/db/migrations/20250930135612_bigint_migration_service_usage_events_step3a.rb
@@ -0,0 +1,6 @@
+require 'database/bigint_migration_step3ab'
+
+Sequel.migration do
+  no_transaction
+  VCAP::BigintMigration.step3a_migration(self, :service_usage_events)
+end
diff --git a/db/migrations/20250930135619_bigint_migration_service_usage_events_step3b.rb b/db/migrations/20250930135619_bigint_migration_service_usage_events_step3b.rb
@@ -0,0 +1,6 @@
+require 'database/bigint_migration_step3ab'
+
+Sequel.migration do
+  no_transaction
+  VCAP::BigintMigration.step3b_migration(self, :service_usage_events)
+end
diff --git a/lib/database/bigint_migration_step3ab.rb b/lib/database/bigint_migration_step3ab.rb
@@ -0,0 +1,78 @@
+require 'database/bigint_migration'
+
+module VCAP::BigintMigration
+  class << self
+    def step3a_migration(migration, table)
+      migration.up do
+        if database_type == :postgres &&
+           !VCAP::BigintMigration.migration_completed?(self, table) &&
+           !VCAP::BigintMigration.migration_skipped?(self, table)
+          transaction { VCAP::BigintMigration.add_check_constraint(self, table) }
+          begin
+            VCAP::Migration.with_concurrent_timeout(self) do
+              VCAP::BigintMigration.validate_check_constraint(self, table)
+            end
+          rescue Sequel::CheckConstraintViolation
+            raise "Failed to add check constraint on '#{table}' table!\n" \
+                  "There are rows where 'id_bigint' does not match 'id', " \
+                  "thus step 3 of the bigint migration cannot be executed.\n" \
+                  "Consider running rake task 'db:bigint_backfill[#{table}]'."
+          end
+        end
+      end
+
+      migration.down do
+        transaction { VCAP::BigintMigration.drop_check_constraint(self, table) if database_type == :postgres }
+      end
+    end
+
+    def step3b_migration(migration, table)
+      migration.up do
+        if database_type == :postgres && VCAP::BigintMigration.has_check_constraint?(self, table)
+          transaction do
+            # Drop check constraint and trigger function
+            VCAP::BigintMigration.drop_check_constraint(self, table)
+            VCAP::BigintMigration.drop_trigger_function(self, table)
+
+            # Drop old id column
+            VCAP::BigintMigration.drop_pk_column(self, table)
+
+            # Switch id_bigint -> id
+            VCAP::BigintMigration.rename_bigint_column(self, table)
+            VCAP::BigintMigration.add_pk_constraint(self, table)
+            VCAP::BigintMigration.set_pk_as_identity_with_correct_start_value(self, table)
+          end
+        end
+      end
+
+      migration.down do
+        if database_type == :postgres && VCAP::BigintMigration.migration_completed?(self, table)
+          transaction do
+            # Revert id -> id_bigint
+            VCAP::BigintMigration.drop_identity(self, table)
+            VCAP::BigintMigration.drop_timestamp_pk_index(self, table)
+            VCAP::BigintMigration.drop_pk_constraint(self, table)
+            VCAP::BigintMigration.revert_bigint_column_name(self, table)
+
+            # Restore old id column
+            VCAP::BigintMigration.add_id_column(self, table)
+
+            # To restore the previous state it is necessary to backfill the id column. In case there is a lot of data in the
+            # table this might be problematic, e.g. take a longer time.
+            #
+            # Ideally this down migration SHOULD NEVER BE EXECUTED IN A PRODUCTION SYSTEM! (It's there for proper integration
+            # testing of the bigint migration steps.)
+            VCAP::BigintMigration.backfill_id(self, table)
+
+            VCAP::BigintMigration.add_pk_constraint(self, table)
+            VCAP::BigintMigration.set_pk_as_identity_with_correct_start_value(self, table)
+
+            # Recreate trigger function and check constraint
+            VCAP::BigintMigration.create_trigger_function(self, table)
+            VCAP::BigintMigration.add_check_constraint(self, table)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/migrations/20250729143100_bigint_migration_service_usage_events_step3a_spec.rb b/spec/migrations/20250729143100_bigint_migration_service_usage_events_step3a_spec.rb
@@ -0,0 +1,19 @@
+require 'spec_helper'
+require 'migrations/helpers/bigint_migration_step3_shared_context'
+
+RSpec.describe 'bigint migration - service_usage_events table - step3a', isolation: :truncation, type: :migration do
+  include_context 'bigint migration step3a' do
+    let(:migration_filename_step1) { '20250729143100_bigint_migration_service_usage_events_step1.rb' }
+    let(:migration_filename_step3a) { '20250930135612_bigint_migration_service_usage_events_step3a.rb' }
+    let(:table) { :service_usage_events }
+    let(:insert) do
+      lambda do |db|
+        db[:service_usage_events].insert(guid: SecureRandom.uuid, created_at: Time.now.utc,
+                                         state: 'teststate', org_guid: SecureRandom.uuid,
+                                         space_guid: SecureRandom.uuid, space_name: 'testspace',
+                                         service_instance_guid: SecureRandom.uuid, service_instance_name: 'testinstance',
+                                         service_instance_type: 'testtype')
+      end
+    end
+  end
+end
diff --git a/spec/migrations/20250930135451_bigint_migration_delayed_jobs_step3a_spec.rb b/spec/migrations/20250930135451_bigint_migration_delayed_jobs_step3a_spec.rb
@@ -0,0 +1,15 @@
+require 'spec_helper'
+require 'migrations/helpers/bigint_migration_step3_shared_context'
+
+RSpec.describe 'bigint migration - delayed jobs table - step3a', isolation: :truncation, type: :migration do
+  include_context 'bigint migration step3a' do
+    let(:migration_filename_step1) { '20250729142800_bigint_migration_delayed_jobs_step1.rb' }
+    let(:migration_filename_step3a) { '20250930135451_bigint_migration_delayed_jobs_step3a_spec.rb' }
+    let(:table) { :delayed_jobs }
+    let(:insert) do
+      lambda do |db|
+        db[:delayed_jobs].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, updated_at: Time.now.utc)
+      end
+    end
+  end
+end
diff --git a/spec/migrations/20250930135459_bigint_migration_delayed_jobs_step3b_spec.rb b/spec/migrations/20250930135459_bigint_migration_delayed_jobs_step3b_spec.rb
@@ -0,0 +1,16 @@
+require 'spec_helper'
+require 'migrations/helpers/bigint_migration_step3_shared_context'
+
+RSpec.describe 'bigint migration - delayed jobs table - step3b', isolation: :truncation, type: :migration do
+  include_context 'bigint migration step3b' do
+    let(:migration_filename_step1) { '20250729142800_bigint_migration_delayed_jobs_step1.rb' }
+    let(:migration_filename_step3a) { '20250930135451_bigint_migration_delayed_jobs_step3a.rb' }
+    let(:migration_filename_step3b) { '20250930135459_bigint_migration_delayed_jobs_step3b.rb' }
+    let(:table) { :delayed_jobs }
+    let(:insert) do
+      lambda do |db|
+        db[:delayed_jobs].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, updated_at: Time.now.utc)
+      end
+    end
+  end
+end
diff --git a/spec/migrations/20250930135517_bigint_migration_jobs_step3a_spec.rb b/spec/migrations/20250930135517_bigint_migration_jobs_step3a_spec.rb
@@ -0,0 +1,15 @@
+require 'spec_helper'
+require 'migrations/helpers/bigint_migration_step3_shared_context'
+
+RSpec.describe 'bigint migration - jobs table - step3a', isolation: :truncation, type: :migration do
+  include_context 'bigint migration step3a' do
+    let(:migration_filename_step1) { '20250729142900_bigint_migration_jobs_step1.rb' }
+    let(:migration_filename_step3a) { '20250930135517_bigint_migration_jobs_step3a.rb' }
+    let(:table) { :jobs }
+    let(:insert) do
+      lambda do |db|
+        db[:jobs].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, updated_at: Time.now.utc)
+      end
+    end
+  end
+end
diff --git a/spec/migrations/20250930135527_bigint_migration_jobs_step3b_spec.rb b/spec/migrations/20250930135527_bigint_migration_jobs_step3b_spec.rb
@@ -0,0 +1,16 @@
+require 'spec_helper'
+require 'migrations/helpers/bigint_migration_step3_shared_context'
+
+RSpec.describe 'bigint migration - jobs table - step3b', isolation: :truncation, type: :migration do
+  include_context 'bigint migration step3b' do
+    let(:migration_filename_step1) { '20250729142900_bigint_migration_jobs_step1.rb' }
+    let(:migration_filename_step3a) { '20250930135517_bigint_migration_jobs_step3a.rb' }
+    let(:migration_filename_step3b) { '20250930135527_bigint_migration_jobs_step3b.rb' }
+    let(:table) { :jobs }
+    let(:insert) do
+      lambda do |db|
+        db[:jobs].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, updated_at: Time.now.utc)
+      end
+    end
+  end
+end
diff --git a/spec/migrations/20250930135548_bigint_migration_app_usage_events_step3a_spec.rb b/spec/migrations/20250930135548_bigint_migration_app_usage_events_step3a_spec.rb
@@ -0,0 +1,19 @@
+require 'spec_helper'
+require 'migrations/helpers/bigint_migration_step3_shared_context'
+
+RSpec.describe 'bigint migration - app_usage_events table - step3a', isolation: :truncation, type: :migration do
+  include_context 'bigint migration step3a' do
+    let(:migration_filename_step1) { '20250729143000_bigint_migration_app_usage_events_step1.rb' }
+    let(:migration_filename_step3a) { '20250930135548_bigint_migration_app_usage_events_step3a.rb' }
+    let(:table) { :app_usage_events }
+    let(:insert) do
+      lambda do |db|
+        db[:app_usage_events].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, instance_count: 1,
+                                     memory_in_mb_per_instance: 512, state: 'teststate',
+                                     app_guid: SecureRandom.uuid, app_name: 'testapp',
+                                     space_guid: SecureRandom.uuid, space_name: 'testspace',
+                                     org_guid: SecureRandom.uuid)
+      end
+    end
+  end
+end
diff --git a/spec/migrations/20250930135554_bigint_migration_app_usage_events_step3b_spec.rb b/spec/migrations/20250930135554_bigint_migration_app_usage_events_step3b_spec.rb
@@ -0,0 +1,20 @@
+require 'spec_helper'
+require 'migrations/helpers/bigint_migration_step3_shared_context'
+
+RSpec.describe 'bigint migration - app_usage_events table - step3b', isolation: :truncation, type: :migration do
+  include_context 'bigint migration step3b' do
+    let(:migration_filename_step1) { '20250729143000_bigint_migration_app_usage_events_step1.rb' }
+    let(:migration_filename_step3a) { '20250930135548_bigint_migration_app_usage_events_step3a.rb' }
+    let(:migration_filename_step3b) { '20250930135554_bigint_migration_app_usage_events_step3b.rb' }
+    let(:table) { :app_usage_events }
+    let(:insert) do
+      lambda do |db|
+        db[:app_usage_events].insert(guid: SecureRandom.uuid, created_at: Time.now.utc, instance_count: 1,
+                                     memory_in_mb_per_instance: 512, state: 'teststate',
+                                     app_guid: SecureRandom.uuid, app_name: 'testapp',
+                                     space_guid: SecureRandom.uuid, space_name: 'testspace',
+                                     org_guid: SecureRandom.uuid)
+      end
+    end
+  end
+end
diff --git a/spec/migrations/20250930135619_bigint_migration_service_usage_events_step3b_spec.rb b/spec/migrations/20250930135619_bigint_migration_service_usage_events_step3b_spec.rb
@@ -0,0 +1,20 @@
+require 'spec_helper'
+require 'migrations/helpers/bigint_migration_step3_shared_context'
+
+RSpec.describe 'bigint migration - service_usage_events table - step3b', isolation: :truncation, type: :migration do
+  include_context 'bigint migration step3b' do
+    let(:migration_filename_step1) { '20250729143100_bigint_migration_service_usage_events_step1.rb' }
+    let(:migration_filename_step3a) { '20250930135612_bigint_migration_service_usage_events_step3a.rb' }
+    let(:migration_filename_step3b) { '20250930135619_bigint_migration_service_usage_events_step3b.rb' }
+    let(:table) { :service_usage_events }
+    let(:insert) do
+      lambda do |db|
+        db[:service_usage_events].insert(guid: SecureRandom.uuid, created_at: Time.now.utc,
+                                         state: 'teststate', org_guid: SecureRandom.uuid,
+                                         space_guid: SecureRandom.uuid, space_name: 'testspace',
+                                         service_instance_guid: SecureRandom.uuid, service_instance_name: 'testinstance',
+                                         service_instance_type: 'testtype')
+      end
+    end
+  end
+end
diff --git a/spec/migrations/helpers/bigint_migration_step3_shared_context.rb b/spec/migrations/helpers/bigint_migration_step3_shared_context.rb
@@ -51,7 +51,7 @@
           it 'fails ...' do
             expect do
               run_migration_step3a
-            end.to raise_error(/Failed to add check constraint on 'events' table!/)
+            end.to raise_error(/Failed to add check constraint on '#{table}' table!/)
           end
         end
       end
@@ -178,11 +178,14 @@
       end
 
       it 'has an index on timestamp + (bigint) id column' do
-        expect(db).to have_table_with_index_on_columns(table, %i[timestamp id])
+        if db.schema(table).any? { |col| col[0] == :timestamp }
 
-        run_migration_step3b
+          expect(db).to have_table_with_index_on_columns(table, %i[timestamp id])
+
+          run_migration_step3b
 
-        expect(db).to have_table_with_index_on_columns(table, %i[timestamp id])
+          expect(db).to have_table_with_index_on_columns(table, %i[timestamp id])
+        end
       end
 
       it 'uses an identity with correct start value for the (bigint) id column' do
@@ -218,12 +221,15 @@
       end
 
       it 'has an index on timestamp + (integer) id column' do
-        expect(db).to have_table_with_index_on_columns(table, %i[timestamp id])
+        if db.schema(table).any? { |col| col[0] == :timestamp }
 
-        run_rollback_step3b
+          expect(db).to have_table_with_index_on_columns(table, %i[timestamp id])
+
+          run_rollback_step3b
 
-        expect(db).to have_table_with_index_on_columns(table, %i[timestamp id])
-        expect(db).not_to have_table_with_index_on_columns(table, %i[timestamp id_bigint])
+          expect(db).to have_table_with_index_on_columns(table, %i[timestamp id])
+          expect(db).not_to have_table_with_index_on_columns(table, %i[timestamp id_bigint])
+        end
       end
 
       it 'uses the (integer) id column as primary key' do
