rework lb generation (#467)

Author: 3kh0

app/controllers/leaderboards_controller.rb

@@ -78,35 +78,31 @@ def find_or_generate_leaderboard
   def generate_regional_leaderboard
     return nil unless current_user&.timezone_utc_offset
 
-    LeaderboardGenerator.generate_timezone_offset_leaderboard(
-      start_date, current_user.timezone_utc_offset, @period_type
+    LeaderboardService.get(
+      period: @period_type,
+      date: start_date,
+      offset: current_user.timezone_utc_offset
     )
   end
 
   def generate_timezone_leaderboard
     return nil unless current_user&.timezone
 
-    LeaderboardGenerator.generate_timezone_leaderboard(
-      start_date, current_user.timezone, @period_type
+    offset = current_user.timezone_utc_offset
+    return nil unless offset
+
+    LeaderboardService.get(
+      period: @period_type,
+      date: start_date,
+      offset: offset
     )
   end
 
   def find_or_generate_global_leaderboard
-    cache_key = "leaderboard_#{@period_type}_#{start_date}"
-
-    leaderboard = Rails.cache.fetch(cache_key, expires_in: 1.minute) do
-      Leaderboard.where.not(finished_generating_at: nil)
-                 .find_by(start_date: start_date, period_type: @period_type, deleted_at: nil)
-    end
-
-    Rails.cache.delete(cache_key) if leaderboard.nil?
-
-    if leaderboard.nil?
-      LeaderboardUpdateJob.perform_later(@period_type)
-      nil
-    else
-      leaderboard
-    end
+    LeaderboardService.get(
+      period: @period_type,
+      date: start_date
+    )
   end
 
   def start_date

app/controllers/static_pages_controller.rb

@@ -89,31 +89,23 @@ def mini_leaderboard
 
     if use_timezone_leaderboard && current_user&.timezone_utc_offset
       # we now doing it by default wooo
-      @leaderboard = LeaderboardGenerator.generate_timezone_offset_leaderboard(
-        Date.current, current_user.timezone_utc_offset, :daily
+      @leaderboard = LeaderboardService.get(
+        period: :daily,
+        date: Date.current,
+        offset: current_user.timezone_utc_offset
       )
 
       if @leaderboard&.entries&.empty?
         Rails.logger.warn "[MiniLeaderboard] Regional leaderboard empty for offset #{current_user.timezone_utc_offset}"
+        @leaderboard = nil
       end
-    else
-      # Use global leaderboard
-      @leaderboard = Leaderboard.where.associated(:entries)
-                                .where(start_date: Date.current)
-                                .where(deleted_at: nil)
-                                .where(period_type: :daily)
-                                .distinct
-                                .first
     end
 
-    if @leaderboard.nil? || @leaderboard.entries.empty?
-      Rails.logger.info "[MiniLeaderboard] Falling back to global leaderboard"
-      @leaderboard = Leaderboard.where.associated(:entries)
-                                .where(start_date: Date.current)
-                                .where(deleted_at: nil)
-                                .where(period_type: :daily)
-                                .distinct
-                                .first
+    if @leaderboard.nil?
+      @leaderboard = LeaderboardService.get(
+        period: :daily,
+        date: Date.current
+      )
     end
 
     @active_projects = Cache::ActiveProjectsJob.perform_now

app/jobs/leaderboard_update_job.rb

@@ -1,79 +1,111 @@
 class LeaderboardUpdateJob < ApplicationJob
   queue_as :latency_10s
 
-  BATCH_SIZE = 1000
-
   include GoodJob::ActiveJobExtensions::Concurrency
 
-  # Limits concurrency to 1 job per date
+  # Limits concurrency to 1 job per period/date combination
   good_job_control_concurrency_with(
-    key: -> { "#{arguments[0] || 'daily'}_#{arguments[1] || Date.current.to_s}" },
+    key: -> { "leaderboard_#{arguments[0] || 'daily'}_#{arguments[1] || Date.current.to_s}" },
     total: 1,
     drop: true
   )
 
-  def perform(period_type = :daily, date = Date.current)
-    parsed_date = date.is_a?(Date) ? date : Date.parse(date.to_s)
+  def perform(period = :daily, date = Date.current)
+    date = LeaderboardDateRange.normalize_date(date, period)
+
+    Rails.logger.info "Starting leaderboard generation for #{period} on #{date}"
+
+    board = build_global(date, period)
+    build_timezones(date, period)
 
-    parsed_date = parsed_date.beginning_of_week if period_type == :weekly
+    Rails.logger.info "Completed leaderboard generation for #{period} on #{date}"
+
+    board
+  rescue => e
+    Rails.logger.error "Failed to update leaderboard: #{e.message}"
+    Honeybadger.notify(e, context: { period: period, date: date })
+    raise
+  end
 
-    leaderboard = Leaderboard.create!(
-      start_date: parsed_date,
-      period_type: period_type
-    )
+  private
 
-    date_range = case period_type
-    when :weekly
-      (parsed_date.beginning_of_day...(parsed_date + 7.days).beginning_of_day)
-    when :last_7_days
-      ((parsed_date - 6.days).beginning_of_day...parsed_date.end_of_day)
-    else
-      parsed_date.all_day
+  def build_global(date, period)
+    range = LeaderboardDateRange.calculate(date, period)
+    board = ::Leaderboard.find_or_create_by!(
+      start_date: date,
+      period_type: period,
+      timezone_offset: nil
+    ) do |lb|
+      lb.finished_generating_at = nil
     end
 
-    Rails.logger.info "Starting leaderboard generation for #{period_type} on #{parsed_date}"
+    return board if board.finished_generating_at.present?
 
     ActiveRecord::Base.transaction do
-      entries_data = Heartbeat.where(time: date_range)
-                              .coding_only
-                              .with_valid_timestamps
-                              .joins(:user)
-                              .where.not(users: { github_uid: nil })
-                              .group(:user_id)
-                              .duration_seconds
+      board.entries.delete_all
+      data = Heartbeat.where(time: range)
+                     .with_valid_timestamps
+                     .joins(:user)
+                     .coding_only
+                     .where.not(users: { github_uid: nil })
+                     .group(:user_id)
+                     .duration_seconds
 
-      entries_data = entries_data.filter { |_, total_seconds| total_seconds > 60 }
+      data = data.filter { |_, seconds| seconds > 60 }
 
-      convicted_users = User.where(trust_level: User.trust_levels[:red]).pluck(:id)
-      entries_data = entries_data.reject { |user_id, _| convicted_users.include?(user_id) }
+      convicted = User.where(trust_level: User.trust_levels[:red]).pluck(:id)
+      data = data.reject { |user_id, _| convicted.include?(user_id) }
 
-      streaks = Heartbeat.daily_streaks_for_users(entries_data.keys)
+      streaks = Heartbeat.daily_streaks_for_users(data.keys)
 
-      entries_data = entries_data.map do |user_id, total_seconds|
+      entries = data.map do |user_id, seconds|
         {
-          leaderboard_id: leaderboard.id,
+          leaderboard_id: board.id,
           user_id: user_id,
-          total_seconds: total_seconds,
-          streak_count: streaks[user_id] || 0
+          total_seconds: seconds,
+          streak_count: streaks[user_id] || 0,
+          created_at: Time.current,
+          updated_at: Time.current
         }
       end
 
-      LeaderboardEntry.insert_all!(entries_data) if entries_data.any?
+      LeaderboardEntry.insert_all!(entries) if entries.any?
+      board.update!(finished_generating_at: Time.current)
     end
 
-    leaderboard.finished_generating_at = Time.current
-    leaderboard.save!
+    key = LeaderboardCache.global_key(period, date)
+    LeaderboardCache.write(key, board)
 
-    Leaderboard.where.not(id: leaderboard.id)
-               .where(start_date: parsed_date, period_type: period_type)
-               .where(deleted_at: nil)
-               .update_all(deleted_at: Time.current)
+    board
+  end
 
-    leaderboard
-  rescue => e
-    Rails.logger.error "Failed to update current leaderboard: #{e.message}"
-    raise
-  rescue Date::Error
-    raise ArgumentError, "Invalid date format provided"
+  def build_timezones(date, period)
+    range = LeaderboardDateRange.calculate(date, period)
+
+    offsets = User.joins(:heartbeats)
+                  .where(heartbeats: { time: range })
+                  .where.not(timezone_utc_offset: nil)
+                  .distinct
+                  .pluck(:timezone_utc_offset)
+                  .compact
+
+    Rails.logger.info "Generating timezone leaderboards for #{offsets.size} active UTC offsets"
+
+    offsets.each do |offset|
+      build_timezone(date, period, offset)
+    end
+  end
+
+  def build_timezone(date, period, offset)
+    key = LeaderboardCache.timezone_key(offset, date, period)
+
+    data = LeaderboardCache.fetch(key) do
+      users = User.users_in_timezone_offset(offset).not_convicted
+      LeaderboardBuilder.build_for_users(users, date, "UTC#{offset >= 0 ? '+' : ''}#{offset}", period)
+    end
+
+    Rails.logger.debug "Cached timezone leaderboard for UTC#{offset >= 0 ? '+' : ''}#{offset} with #{data&.entries&.size || 0} entries"
+
+    data
   end
 end

app/jobs/timezone_leaderboard_job.rb

@@ -0,0 +1,41 @@
+class TimezoneLeaderboardJob < ApplicationJob
+  queue_as :latency_5m
+
+  include GoodJob::ActiveJobExtensions::Concurrency
+
+  # Limits concurrency to 1 job per timezone/period/date combination
+  good_job_control_concurrency_with(
+    key: -> { "timezone_#{arguments[0]}_#{arguments[1]}_#{arguments[2]}" },
+    total: 1,
+    drop: true
+  )
+
+  def perform(period = :daily, date = Date.current, offset = 0)
+    date = LeaderboardDateRange.normalize_date(date, period)
+
+    Rails.logger.info "Generating timezone leaderboard for UTC#{offset >= 0 ? '+' : ''}#{offset} (#{period}, #{date})"
+
+    key = LeaderboardCache.timezone_key(offset, date, period)
+
+    # Generate the leaderboard
+    board = build_timezone(date, period, offset)
+
+    # Cache it for 10 minutes
+    LeaderboardCache.write(key, board)
+
+    Rails.logger.info "Cached timezone leaderboard for UTC#{offset >= 0 ? '+' : ''}#{offset} with #{board&.entries&.size || 0} entries"
+
+    board
+  rescue => e
+    Rails.logger.error "Failed to generate timezone leaderboard for UTC#{offset}: #{e.message}"
+    Honeybadger.notify(e, context: { period: period, date: date, offset: offset })
+    raise
+  end
+
+  private
+
+  def build_timezone(date, period, offset)
+    users = User.users_in_timezone_offset(offset).not_convicted
+    LeaderboardBuilder.build_for_users(users, date, "UTC#{offset >= 0 ? '+' : ''}#{offset}", period)
+  end
+end

app/jobs/warm_mini_leaderboard_cache_job.rb

@@ -0,0 +1,49 @@
+module LeaderboardBuilder
+  module_function
+
+  def build_for_users(users, date, scope, period)
+    date = Date.current if date.blank?
+
+    board = ::Leaderboard.new(
+      start_date: date,
+      period_type: period,
+      finished_generating_at: Time.current
+    )
+
+    ids = users.pluck(:id)
+    return board if ids.empty?
+
+    users_map = users.index_by(&:id)
+
+    range = LeaderboardDateRange.calculate(date, period)
+
+    beats = Heartbeat.where(user_id: ids, time: range)
+                    .coding_only
+                    .with_valid_timestamps
+                    .joins(:user)
+                    .where.not(users: { github_uid: nil })
+
+    totals = beats.group(:user_id).duration_seconds
+    totals = totals.filter { |_, seconds| seconds > 60 }
+
+    streak_ids = totals.keys
+    streaks = streak_ids.any? ? Heartbeat.daily_streaks_for_users(streak_ids, start_date: 30.days.ago) : {}
+
+    entries = totals.map do |user_id, seconds|
+      entry = LeaderboardEntry.new(
+        leaderboard: board,
+        user_id: user_id,
+        total_seconds: seconds,
+        streak_count: streaks[user_id] || 0
+      )
+
+      entry.user = users_map[user_id]
+      entry
+    end.sort_by(&:total_seconds).reverse
+
+    board.define_singleton_method(:entries) { entries }
+    board.define_singleton_method(:scope_name) { scope }
+
+    board
+  end
+end

app/services/leaderboard_builder.rb

@@ -0,0 +1,25 @@
+module LeaderboardCache
+  CACHE_EXPIRATION = 10.minutes
+
+  module_function
+
+  def global_key(period, date)
+    "leaderboard_#{period}_#{date}"
+  end
+
+  def timezone_key(offset, date, period)
+    "tz_leaderboard_#{offset}_#{date}_#{period}"
+  end
+
+  def write(key, data)
+    Rails.cache.write(key, data, expires_in: CACHE_EXPIRATION)
+  end
+
+  def read(key)
+    Rails.cache.read(key)
+  end
+
+  def fetch(key, &block)
+    Rails.cache.fetch(key, expires_in: CACHE_EXPIRATION, &block)
+  end
+end