Backend: Introduce MAX_LEADERBOARD_QUERY_LIMIT setting to optimize leaderboard data queries and update filtering logic in calculate_distinct_sorted_leaderboard_data function to enhance performance. (#4994)

Author: RishabhJain2018

apps/jobs/utils.py

@@ -8,6 +8,7 @@
 from base.utils import get_model_object, suppress_autotime
 from challenges.models import ChallengePhaseSplit, LeaderboardData
 from challenges.utils import get_challenge_phase_model
+from django.conf import settings
 from django.db.models import ExpressionWrapper, F, FloatField, Q, fields
 from django.db.models.expressions import RawSQL
 from django.utils import timezone
@@ -364,10 +365,14 @@ def calculate_distinct_sorted_leaderboard_data(
         response_data = {"error": "Sorry, the leaderboard is not public!"}
         return response_data, status.HTTP_400_BAD_REQUEST
 
-    leaderboard_data = LeaderboardData.objects.exclude(
-        Q(submission__created_by__email__in=challenge_hosts_emails)
-        & Q(submission__is_baseline=False)
-    ).filter(is_disabled=False)
+    leaderboard_data = LeaderboardData.objects.filter(is_disabled=False)
+    # Only exclude host submissions when there are host emails (avoids
+    # auth_user join)
+    if challenge_hosts_emails:
+        leaderboard_data = leaderboard_data.exclude(
+            Q(submission__created_by__email__in=challenge_hosts_emails)
+            & Q(submission__is_baseline=False)
+        )
 
     # Get all the successful submissions related to the challenge phase split
     all_valid_submission_status = [Submission.FINISHED]
@@ -457,6 +462,10 @@ def calculate_distinct_sorted_leaderboard_data(
 
     all_banned_participant_team = []
 
+    # Apply query limit to prevent slow queries on popular challenges
+    max_limit = getattr(settings, "MAX_LEADERBOARD_QUERY_LIMIT", 10000)
+    leaderboard_data = leaderboard_data[:max_limit]
+
     # Convert to list to allow multiple iterations
     leaderboard_data = list(leaderboard_data)
 

settings/common.py

@@ -156,6 +156,9 @@
 
 SITE_ID = 1
 
+# Maximum number of leaderboard rows to load per query (prevents slow queries)
+MAX_LEADERBOARD_QUERY_LIMIT = 10000
+
 REST_FRAMEWORK = {
     "DEFAULT_PAGINATION_CLASS": (
         "rest_framework.pagination.LimitOffsetPagination"

tests/unit/jobs/test_utils.py

@@ -49,7 +49,11 @@ def test_calculate_distinct_sorted_leaderboard_data(
         mock_challenge_phase_split = MagicMock()
         mock_leaderboard_data = MagicMock()
         mock_filter.return_value = mock_leaderboard_data
-        mock_exclude.return_value = mock_leaderboard_data
+        mock_leaderboard_data.exclude.return_value = mock_leaderboard_data
+        mock_leaderboard_data.filter.return_value = mock_leaderboard_data
+        mock_leaderboard_data.order_by.return_value = mock_leaderboard_data
+        mock_leaderboard_data.annotate.return_value = mock_leaderboard_data
+        mock_leaderboard_data.values.return_value = []
 
         (
             leaderboard_data,
@@ -302,7 +306,11 @@ def test_calculate_distinct_sorted_leaderboard_data_order_by_in_labels(
         mock_challenge_phase_split = MagicMock()
         mock_leaderboard_data = MagicMock()
         mock_filter.return_value = mock_leaderboard_data
-        mock_exclude.return_value = mock_leaderboard_data
+        mock_leaderboard_data.filter.return_value = mock_leaderboard_data
+        mock_leaderboard_data.order_by.return_value = mock_leaderboard_data
+        mock_leaderboard_data.annotate.return_value = mock_leaderboard_data
+        mock_leaderboard_data.values.return_value = []
+        # When challenge_hosts_emails is [], exclude is not called
 
         mock_leaderboard = MagicMock()
         mock_leaderboard.schema = {
@@ -541,9 +549,11 @@ def test_banned_email_ids(
         ]
 
         # Set up chainable mock for:
-        # .exclude().filter().filter().order_by().annotate().values()
+        # .filter().exclude().filter().filter().order_by().annotate().values()
         mock_qs = MagicMock()
-        mock_leaderboard_data_objects.exclude.return_value = mock_qs
+        mock_filter_result = MagicMock()
+        mock_leaderboard_data_objects.filter.return_value = mock_filter_result
+        mock_filter_result.exclude.return_value = mock_qs
         mock_qs.filter.return_value = mock_qs
         mock_qs.order_by.return_value = mock_qs
         mock_qs.annotate.return_value = mock_qs
@@ -594,9 +604,11 @@ def test_team_in_all_banned_participant_team(
         ]
 
         # Set up chainable mock for:
-        # .exclude().filter().filter().order_by().annotate().values()
+        # .filter().exclude().filter().filter().order_by().annotate().values()
         mock_qs = MagicMock()
-        mock_leaderboard_data_objects.exclude.return_value = mock_qs
+        mock_filter_result = MagicMock()
+        mock_leaderboard_data_objects.filter.return_value = mock_filter_result
+        mock_filter_result.exclude.return_value = mock_qs
         mock_qs.filter.return_value = mock_qs
         mock_qs.order_by.return_value = mock_qs
         mock_qs.annotate.return_value = mock_qs
@@ -664,9 +676,11 @@ def test_bulk_prefetch_avoids_n_plus_one_queries(
         ]
 
         # Set up chainable mock for:
-        # .exclude().filter().filter().order_by().annotate().values()
+        # .filter().exclude().filter().filter().order_by().annotate().values()
         mock_qs = MagicMock()
-        mock_leaderboard_data_objects.exclude.return_value = mock_qs
+        mock_filter_result = MagicMock()
+        mock_leaderboard_data_objects.filter.return_value = mock_filter_result
+        mock_filter_result.exclude.return_value = mock_qs
         mock_qs.filter.return_value = mock_qs
         mock_qs.order_by.return_value = mock_qs
         mock_qs.annotate.return_value = mock_qs
@@ -741,9 +755,11 @@ def test_multiple_banned_participants_in_team(
         ]
 
         # Set up chainable mock for:
-        # .exclude().filter().filter().order_by().annotate().values()
+        # .filter().exclude().filter().filter().order_by().annotate().values()
         mock_qs = MagicMock()
-        mock_leaderboard_data_objects.exclude.return_value = mock_qs
+        mock_filter_result = MagicMock()
+        mock_leaderboard_data_objects.filter.return_value = mock_filter_result
+        mock_filter_result.exclude.return_value = mock_qs
         mock_qs.filter.return_value = mock_qs
         mock_qs.order_by.return_value = mock_qs
         mock_qs.annotate.return_value = mock_qs