Backend: Refactor calculate_distinct_sorted_leaderboard_data to use sets for banned email IDs and participant teams, improving performance and ensuring distinct team listings. Add unit tests to verify behavior with empty and None banned email IDs. (#4997)

Author: RishabhJain2018

apps/jobs/utils.py

@@ -460,7 +460,10 @@ def calculate_distinct_sorted_leaderboard_data(
             "submission__is_verified_by_host",
         )
 
-    all_banned_participant_team = []
+    all_banned_participant_team = set()
+    all_banned_email_ids_set = (
+        set(all_banned_email_ids) if all_banned_email_ids else set()
+    )
 
     # Apply query limit to prevent slow queries on popular challenges
     max_limit = getattr(settings, "MAX_LEADERBOARD_QUERY_LIMIT", 10000)
@@ -489,8 +492,8 @@ def calculate_distinct_sorted_leaderboard_data(
             participant_team_id, []
         )
         for participant_email in all_participants_email_ids:
-            if participant_email in all_banned_email_ids:
-                all_banned_participant_team.append(participant_team_id)
+            if participant_email in all_banned_email_ids_set:
+                all_banned_participant_team.add(participant_team_id)
                 break
         if leaderboard_item["error"] is None:
             leaderboard_item.update(filtering_error=0)
@@ -508,7 +511,7 @@ def calculate_distinct_sorted_leaderboard_data(
             reverse=True if is_leaderboard_order_descending else False,
         )
     distinct_sorted_leaderboard_data = []
-    team_list = []
+    team_list = set()
     for data in sorted_leaderboard_data:
         if (
             data["submission__participant_team__team_name"] in team_list
@@ -520,7 +523,7 @@ def calculate_distinct_sorted_leaderboard_data(
             distinct_sorted_leaderboard_data.append(data)
         else:
             distinct_sorted_leaderboard_data.append(data)
-            team_list.append(data["submission__participant_team__team_name"])
+            team_list.add(data["submission__participant_team__team_name"])
 
     leaderboard_labels = challenge_phase_split.leaderboard.schema["labels"]
     show_scores = challenge_phase_split.show_scores_on_leaderboard

tests/unit/jobs/test_utils.py

@@ -524,6 +524,7 @@ def setUp(self):
         self.challenge_phase_split.show_leaderboard_by_latest_submission = (
             False
         )
+        self.challenge_phase_split.show_scores_on_leaderboard = True
 
     @patch("jobs.utils.ParticipantTeam")
     @patch("challenges.models.LeaderboardData.objects")
@@ -799,6 +800,330 @@ def test_multiple_banned_participants_in_team(
         self.assertEqual(len(result), 1)
         self.assertEqual(result[0]["submission__participant_team"], 2)
 
+    @patch("jobs.utils.ParticipantTeam")
+    @patch("challenges.models.LeaderboardData.objects")
+    @patch("hosts.utils.is_user_a_staff_or_host")
+    def test_empty_banned_email_ids_includes_all_teams(
+        self,
+        mock_is_user_a_staff_or_host,
+        mock_leaderboard_data_objects,
+        mock_participant_team,
+    ):
+        """Test that empty banned_email_ids excludes no teams."""
+        mock_is_user_a_staff_or_host.return_value = False
+        self.challenge_obj.banned_email_ids = []
+
+        test_data = [
+            {
+                "submission__participant_team": 1,
+                "submission__participant_team__team_name": "Team1",
+                "submission__is_baseline": False,
+                "error": None,
+                "filtering_score": 10,
+                "result": {"score": 10, "time": 5},
+            },
+            {
+                "submission__participant_team": 2,
+                "submission__participant_team__team_name": "Team2",
+                "submission__is_baseline": False,
+                "error": None,
+                "filtering_score": 9,
+                "result": {"score": 9, "time": 4},
+            },
+        ]
+
+        self._create_mock_leaderboard_chain(
+            mock_leaderboard_data_objects, test_data
+        )
+
+        mock_team1 = Mock()
+        mock_team1.id = 1
+        mock_p1 = Mock()
+        mock_p1.user.email = "user1@example.com"
+        mock_team1.participants.all.return_value = [mock_p1]
+
+        mock_team2 = Mock()
+        mock_team2.id = 2
+        mock_p2 = Mock()
+        mock_p2.user.email = "user2@example.com"
+        mock_team2.participants.all.return_value = [mock_p2]
+
+        mock_participant_team.objects.filter.return_value.prefetch_related.return_value = [
+            mock_team1,
+            mock_team2,
+        ]
+
+        result, status_code = calculate_distinct_sorted_leaderboard_data(
+            self.user,
+            self.challenge_obj,
+            self.challenge_phase_split,
+            False,
+            "score",
+        )
+
+        self.assertEqual(status_code, 200)
+        self.assertEqual(len(result), 2)
+
+    @patch("jobs.utils.ParticipantTeam")
+    @patch("challenges.models.LeaderboardData.objects")
+    @patch("hosts.utils.is_user_a_staff_or_host")
+    def test_none_banned_email_ids_includes_all_teams(
+        self,
+        mock_is_user_a_staff_or_host,
+        mock_leaderboard_data_objects,
+        mock_participant_team,
+    ):
+        """Test that None banned_email_ids excludes no teams (set conversion)."""
+        mock_is_user_a_staff_or_host.return_value = False
+        self.challenge_obj.banned_email_ids = None
+
+        test_data = [
+            {
+                "submission__participant_team": 1,
+                "submission__participant_team__team_name": "Team1",
+                "submission__is_baseline": False,
+                "error": None,
+                "filtering_score": 10,
+                "result": {"score": 10, "time": 5},
+            },
+        ]
+
+        self._create_mock_leaderboard_chain(
+            mock_leaderboard_data_objects, test_data
+        )
+
+        mock_team1 = Mock()
+        mock_team1.id = 1
+        mock_p1 = Mock()
+        mock_p1.user.email = "user1@example.com"
+        mock_team1.participants.all.return_value = [mock_p1]
+
+        mock_participant_team.objects.filter.return_value.prefetch_related.return_value = [
+            mock_team1,
+        ]
+
+        result, status_code = calculate_distinct_sorted_leaderboard_data(
+            self.user,
+            self.challenge_obj,
+            self.challenge_phase_split,
+            False,
+            "score",
+        )
+
+        self.assertEqual(status_code, 200)
+        self.assertEqual(len(result), 1)
+
+        test_data = [
+            {
+                "submission__participant_team": 1,
+                "submission__participant_team__team_name": "Team1",
+                "submission__is_baseline": False,
+                "error": None,
+                "filtering_score": 10,
+                "result": {"score": 10, "time": 5},
+            },
+            {
+                "submission__participant_team": 2,
+                "submission__participant_team__team_name": "Team2",
+                "submission__is_baseline": False,
+                "error": None,
+                "filtering_score": 9,
+                "result": {"score": 9, "time": 4},
+            },
+        ]
+
+        self._create_mock_leaderboard_chain(
+            mock_leaderboard_data_objects, test_data
+        )
+
+        mock_team1 = Mock()
+        mock_team1.id = 1
+        mock_p1 = Mock()
+        mock_p1.user.email = "user1@example.com"
+        mock_team1.participants.all.return_value = [mock_p1]
+
+        mock_team2 = Mock()
+        mock_team2.id = 2
+        mock_p2 = Mock()
+        mock_p2.user.email = "user2@example.com"
+        mock_team2.participants.all.return_value = [mock_p2]
+
+        mock_participant_team.objects.filter.return_value.prefetch_related.return_value = [
+            mock_team1,
+            mock_team2,
+        ]
+
+        result, status_code = calculate_distinct_sorted_leaderboard_data(
+            self.user,
+            self.challenge_obj,
+            self.challenge_phase_split,
+            False,
+            "score",
+        )
+
+        self.assertEqual(status_code, 200)
+        self.assertEqual(len(result), 2)
+
+    @patch("jobs.utils.ParticipantTeam")
+    @patch("challenges.models.LeaderboardData.objects")
+    @patch("hosts.utils.is_user_a_staff_or_host")
+    def test_distinct_team_list_with_many_duplicates(
+        self,
+        mock_is_user_a_staff_or_host,
+        mock_leaderboard_data_objects,
+        mock_participant_team,
+    ):
+        """Test set-based team_list gives correct distinct result with many entries."""
+        mock_is_user_a_staff_or_host.return_value = False
+
+        # Many entries from same teams - Team1 appears 5x, Team2 appears 3x
+        test_data = [
+            {
+                "submission__participant_team": 1,
+                "submission__participant_team__team_name": "Team1",
+                "submission__is_baseline": False,
+                "error": None,
+                "filtering_score": float(i),
+                "result": {"score": str(i), "time": "0"},
+            }
+            for i in range(10, 5, -1)
+        ] + [
+            {
+                "submission__participant_team": 2,
+                "submission__participant_team__team_name": "Team2",
+                "submission__is_baseline": False,
+                "error": None,
+                "filtering_score": float(i),
+                "result": {"score": str(i), "time": "0"},
+            }
+            for i in range(9, 6, -1)
+        ]
+
+        self._create_mock_leaderboard_chain(
+            mock_leaderboard_data_objects, test_data
+        )
+
+        mock_team1 = Mock()
+        mock_team1.id = 1
+        mock_p1 = Mock()
+        mock_p1.user.email = "user1@example.com"
+        mock_team1.participants.all.return_value = [mock_p1]
+
+        mock_team2 = Mock()
+        mock_team2.id = 2
+        mock_p2 = Mock()
+        mock_p2.user.email = "user2@example.com"
+        mock_team2.participants.all.return_value = [mock_p2]
+
+        mock_participant_team.objects.filter.return_value.prefetch_related.return_value = [
+            mock_team1,
+            mock_team2,
+        ]
+
+        result, status_code = calculate_distinct_sorted_leaderboard_data(
+            self.user,
+            self.challenge_obj,
+            self.challenge_phase_split,
+            False,
+            "score",
+        )
+
+        self.assertEqual(status_code, 200)
+        # Only 2 distinct teams (best entry per team)
+        self.assertEqual(len(result), 2)
+        self.assertEqual(
+            result[0]["submission__participant_team__team_name"], "Team1"
+        )
+        self.assertEqual(
+            result[1]["submission__participant_team__team_name"], "Team2"
+        )
+
+    @patch("jobs.utils.ParticipantTeam")
+    @patch("challenges.models.LeaderboardData.objects")
+    @patch("hosts.utils.is_user_a_staff_or_host")
+    def test_baseline_entries_always_included(
+        self,
+        mock_is_user_a_staff_or_host,
+        mock_leaderboard_data_objects,
+        mock_participant_team,
+    ):
+        """Test baseline entries are always included regardless of team_list."""
+        mock_is_user_a_staff_or_host.return_value = False
+
+        test_data = [
+            {
+                "submission__participant_team": 1,
+                "submission__participant_team__team_name": "Team1",
+                "submission__is_baseline": False,
+                "error": None,
+                "filtering_score": 10,
+                "result": {"score": "10", "time": "0"},
+            },
+            {
+                "submission__participant_team": 2,
+                "submission__participant_team__team_name": "Baseline",
+                "submission__is_baseline": True,
+                "error": None,
+                "filtering_score": 5,
+                "result": {"score": "5", "time": "0"},
+            },
+            {
+                "submission__participant_team": 2,
+                "submission__participant_team__team_name": "Baseline",
+                "submission__is_baseline": True,
+                "error": None,
+                "filtering_score": 3,
+                "result": {"score": "3", "time": "0"},
+            },
+        ]
+
+        self._create_mock_leaderboard_chain(
+            mock_leaderboard_data_objects, test_data
+        )
+
+        mock_team1 = Mock()
+        mock_team1.id = 1
+        mock_p1 = Mock()
+        mock_p1.user.email = "user1@example.com"
+        mock_team1.participants.all.return_value = [mock_p1]
+
+        mock_team2 = Mock()
+        mock_team2.id = 2
+        mock_p2 = Mock()
+        mock_p2.user.email = "baseline@example.com"
+        mock_team2.participants.all.return_value = [mock_p2]
+
+        mock_participant_team.objects.filter.return_value.prefetch_related.return_value = [
+            mock_team1,
+            mock_team2,
+        ]
+
+        result, status_code = calculate_distinct_sorted_leaderboard_data(
+            self.user,
+            self.challenge_obj,
+            self.challenge_phase_split,
+            False,
+            "score",
+        )
+
+        self.assertEqual(status_code, 200)
+        # Team1 (1) + 2 baseline entries = 3
+        self.assertEqual(len(result), 3)
+
+    def _create_mock_leaderboard_chain(
+        self, mock_leaderboard_data_objects, test_data
+    ):
+        """Helper to create mock chain for LeaderboardData queryset."""
+        mock_qs = MagicMock()
+        mock_filter_result = MagicMock()
+        mock_leaderboard_data_objects.filter.return_value = mock_filter_result
+        mock_filter_result.exclude.return_value = mock_qs
+        mock_qs.filter.return_value = mock_qs
+        mock_qs.order_by.return_value = mock_qs
+        mock_qs.annotate.return_value = mock_qs
+        mock_qs.values.return_value = test_data
+        return mock_qs
+
     def test_all_comparisons(self):
         def dummy_comparator(a, b):
             return a - b