Speed up concise_* tables calculation

db/migrate/20260127202420_add_lookup_data_to_records_lookup.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class AddLookupDataToRecordsLookup < ActiveRecord::Migration[8.1]
+  def change
+    up_only do
+      # Upon reversing, the table can just stay as-is
+      #   because the schema changes will simply be dropped
+      truncate_tables :regional_records_lookup
+    end
+
+    # rubocop:disable Rails/NotNullColumn
+    #   It is okay to introduce a non-null column without default value here,
+    #   because we've made sure above that the table will definitely be empty.
+    change_table :regional_records_lookup, bulk: true do |t|
+      t.string :person_id, after: :result_id, null: false
+      t.integer :competition_reg_year, after: :competition_end_date, null: false
+      t.string :continent_id, after: :country_id, null: false
+    end
+    # rubocop:enable Rails/NotNullColumn
+
+    up_only do
+      # Don't need a `down` because the `change_table` above will just delete the whole column altogether.
+      say_with_time("Recomputing RRL index with augmented columns") do
+        CheckRegionalRecords.add_to_lookup_table
+      end
+    end
+
+    change_table :regional_records_lookup, bulk: true do |t|
+      t.index %i[person_id country_id event_id competition_reg_year best result_id], name: :concise_single_speedup
+      t.index %i[person_id country_id event_id competition_reg_year average result_id], name: :concise_average_speedup
+    end
+  end
+end

db/schema.rb

@@ -1058,10 +1058,15 @@
     t.integer "average", default: 0, null: false
     t.integer "best", default: 0, null: false
     t.date "competition_end_date", null: false
+    t.integer "competition_reg_year", null: false
+    t.string "continent_id", null: false
     t.string "country_id", null: false
     t.string "event_id", null: false
+    t.string "person_id", null: false
     t.index ["event_id", "country_id", "average", "competition_end_date"], name: "idx_on_eventId_countryId_average_competitionEndDate_b424c59953"
     t.index ["event_id", "country_id", "best", "competition_end_date"], name: "idx_on_eventId_countryId_best_competitionEndDate_4e01b1ae38"
+    t.index ["person_id", "country_id", "event_id", "competition_reg_year", "average", "result_id"], name: "concise_average_speedup"
+    t.index ["person_id", "country_id", "event_id", "competition_reg_year", "best", "result_id"], name: "concise_single_speedup"
   end
 
   create_table "registration_competition_events", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|

lib/auxiliary_data_computation.rb

@@ -2,9 +2,9 @@
 
 module AuxiliaryDataComputation
   def self.compute_everything
+    self.insert_regional_records_lookup
     self.compute_concise_results
     self.compute_rank_tables
-    self.insert_regional_records_lookup
   end
 
   ## Build 'concise results' tables.
@@ -16,26 +16,23 @@ def self.compute_concise_results
       DbHelper.with_temp_table(table_name) do |temp_table_name|
         ActiveRecord::Base.connection.execute <<~SQL.squish
           INSERT INTO #{temp_table_name} (id, #{field}, value_and_id, person_id, event_id, country_id, continent_id, reg_year)
+          WITH concise_agg AS (
+            SELECT MIN(#{field} * 1000000000 + result_id) value_and_id
+            FROM regional_records_lookup
+            WHERE #{field} > 0
+            GROUP BY person_id, country_id, event_id, competition_reg_year
+          )
           SELECT
-            results.id,
-            #{field},
-            valueAndId,
-            person_id,
-            event_id,
-            countries.id country_id,
-            continent_id,
-            YEAR(start_date) reg_year
-          FROM (
-              SELECT MIN(#{field} * 1000000000 + results.id) valueAndId
-              FROM results
-              JOIN competitions ON competitions.id = competition_id
-              WHERE #{field} > 0
-              GROUP BY person_id, results.country_id, event_id, YEAR(start_date)
-            ) MinValuesWithId
-            JOIN results ON results.id = valueAndId % 1000000000
-            JOIN competitions ON competitions.id = results.competition_id
-            JOIN countries ON countries.id = results.country_id
-            JOIN events ON events.id = results.event_id
+            rrl.result_id id,
+            rrl.#{field},
+            concise_agg.value_and_id,
+            rrl.person_id,
+            rrl.event_id,
+            rrl.country_id,
+            rrl.continent_id,
+            rrl.competition_reg_year `reg_year`
+          FROM concise_agg
+            INNER JOIN regional_records_lookup rrl ON rrl.result_id = (concise_agg.value_and_id % 1000000000)
         SQL
       end
     end

lib/check_regional_records.rb

@@ -7,15 +7,19 @@ module CheckRegionalRecords
   def self.add_to_lookup_table(competition_id = nil, table_name: LOOKUP_TABLE_NAME)
     ActiveRecord::Base.connection.execute <<~SQL.squish
       INSERT INTO #{table_name}
-      (result_id, country_id, event_id, competition_end_date, best, average)
-      SELECT results.id, results.country_id, results.event_id, competitions.end_date, results.best, results.average
+      (result_id, person_id, country_id, continent_id, event_id, competition_end_date, competition_reg_year, best, average)
+      SELECT results.id, results.person_id, results.country_id, countries.continent_id, results.event_id, competitions.end_date, YEAR(competitions.start_date), results.best, results.average
       FROM results
       INNER JOIN competitions ON results.competition_id = competitions.id
+      INNER JOIN countries ON results.country_id = countries.id
       #{"WHERE results.competition_id = '#{competition_id}'" if competition_id.present?}
       ON DUPLICATE KEY UPDATE
+        person_id = results.person_id,
         country_id = results.country_id,
+        continent_id = countries.continent_id,
         event_id = results.event_id,
         competition_end_date = competitions.end_date,
+        competition_reg_year = YEAR(competitions.start_date),
         best = results.best,
         average = results.average
     SQL

spec/lib/auxiliary_data_computation_spec.rb

@@ -17,7 +17,7 @@
       create(:result, event_id: "333", best: 700, average: 850, competition: competition_2016, person: person, round_type_id: "f", round: round_333_f)
       create(:result, event_id: "333", best: 800, average: 900, competition: competition_2017, person: person)
       create(:result, event_id: "222", best: 100, average: 150, competition: competition_2017, person: person)
-      AuxiliaryDataComputation.compute_concise_results
+      AuxiliaryDataComputation.compute_everything
       # Concise single results
       concise_single_results = ActiveRecord::Base.connection.execute "SELECT event_id, person_id, reg_year, best FROM concise_single_results"
       expect(concise_single_results).to contain_exactly(["333", person.wca_id, 2016, 700], ["333", person.wca_id, 2017, 800], ["222", person.wca_id, 2017, 100])
@@ -30,7 +30,7 @@
       create(:result, event_id: "333", best: 700, average: 800, competition: competition_2016, person: person)
       person.update_using_sub_id! country_id: "Chile"
       create(:result, event_id: "333", best: 750, average: 850, competition: next_competition_2016, person: person)
-      AuxiliaryDataComputation.compute_concise_results
+      AuxiliaryDataComputation.compute_everything
       # Concise single results
       concise_single_results = ActiveRecord::Base.connection.execute "SELECT event_id, person_id, country_id, reg_year, best FROM concise_single_results"
       expect(concise_single_results).to contain_exactly(["333", person.wca_id, "China", 2016, 700], ["333", person.wca_id, "Chile", 2016, 750])
@@ -58,8 +58,7 @@ def rank_333(person, ranks_type)
     end
 
     it "computes world, continental, and national ranking position" do
-      AuxiliaryDataComputation.compute_concise_results # Rank tables computation require concise results to be present.
-      AuxiliaryDataComputation.compute_rank_tables
+      AuxiliaryDataComputation.compute_everything
       %w[ranks_single ranks_average].each do |ranks_type|
         expect(rank_333(australian, ranks_type)).to include(world_rank: 1, continent_rank: 1, country_rank: 1)
         expect(rank_333(american_1, ranks_type)).to include(world_rank: 2, continent_rank: 1, country_rank: 1)
@@ -74,8 +73,7 @@ def rank_333(person, ranks_type)
       australian.update_using_sub_id! country_id: "France"
       new_french = australian
       create(:result, event_id: "333", best: 900, average: 1000, person: new_canadian)
-      AuxiliaryDataComputation.compute_concise_results # Rank tables computation require concise results to be present.
-      AuxiliaryDataComputation.compute_rank_tables
+      AuxiliaryDataComputation.compute_everything
       %w[ranks_single ranks_average].each do |ranks_type|
         # NOTE: this person hasn't got any results in Europe/France yet.
         expect(rank_333(new_french, ranks_type)).to include(world_rank: 1, continent_rank: 0, country_rank: 0)

spec/requests/api_spec.rb

@@ -10,7 +10,7 @@
 
     it "renders current records", :clean_db_with_truncation do
       # Compute necessary data.
-      AuxiliaryDataComputation.compute_concise_results
+      AuxiliaryDataComputation.compute_everything
 
       get api_v0_records_path
       expect(response).to have_http_status :ok