optimize CatalogueView with improved query efficiency and filtering

Author: shenghann

src/api/serializers.py

@@ -79,6 +79,7 @@ class AuthorHistorySerializer(serializers.ModelSerializer):
     author = serializers.PrimaryKeyRelatedField(
         queryset=Author.objects.all(), write_only=True
     )
+    area_name = serializers.CharField(source="area.name", read_only=True, default=None)
 
     class Meta:
         model = AuthorHistory
@@ -87,6 +88,7 @@ class Meta:
             "author",
             "party",
             "area",
+            "area_name",
             "exec_posts",
             "service_posts",
             "start_date",

src/api/views.py

@@ -459,6 +459,8 @@ def get(self, request) -> JsonResponse:
 
         Query Parameters:
         house: 0 for Dewan Rakyat, 1 for Dewan Negara
+        dropdown: If present, only returns the structure without sitting details
+        term: Filter by specific term (optional)
 
         Response Format:
         {
@@ -485,71 +487,108 @@ def get(self, request) -> JsonResponse:
                     ...
                 },
                 ...
-            }
+            },
+            "total_count": <total_count>
         }
         """
         # get query parameters, default DR
         house_type = request.GET.get("house", "dewan-rakyat")
+        house_int = ParliamentaryCycle.get_integer_value(house_type)
+
+        # Filter parameters (optional)
+        term_filter = request.GET.get("term")
 
-        # for dropdown menu
+        # For dropdown menu
         is_dropdown = request.GET.get("dropdown", None)
 
-        # filter Sitting model based on query parameters
-        sittings = Sitting.objects.filter(
-            cycle__house=ParliamentaryCycle.get_integer_value(house_type)
-        ).order_by("date")
+        # Start with base query
+        cycles_query = ParliamentaryCycle.objects.filter(house=house_int)
 
-        # build nested term -> session -> meeting structure
-        cycles = ParliamentaryCycle.objects.filter(
-            house=ParliamentaryCycle.get_integer_value(house_type)
-        )
-        terms = cycles.values("term").annotate(
+        # Apply term filter if provided
+        if term_filter:
+            cycles_query = cycles_query.filter(term=term_filter)
+
+        # Build nested term -> session -> meeting structure more efficiently
+        terms = cycles_query.values("term").annotate(
             start_date=Min("start_date"),
             end_date=Max("end_date"),
         )
-        cycle_map = {
-            term["term"]: {
+
+        cycle_map = {}
+        for term in terms:
+            term_id = term["term"]
+            cycle_map[term_id] = {
                 "start_date": term["start_date"],
                 "end_date": term["end_date"],
             }
-            for term in terms
-        }
 
-        sessions = cycles.values("term", "session").annotate(
-            start_date=Min("start_date"),
-            end_date=Max("end_date"),
-        )
-        for session in sessions:
-            cycle_map[session["term"]][session["session"]] = {
-                "start_date": session["start_date"],
-                "end_date": session["end_date"],
-            }
+            # Get sessions for this term
+            sessions = (
+                cycles_query.filter(term=term_id)
+                .values("session")
+                .annotate(
+                    start_date=Min("start_date"),
+                    end_date=Max("end_date"),
+                )
+            )
 
-        meetings = cycles.values("term", "session", "meeting").annotate(
-            start_date=Min("start_date"),
-            end_date=Max("end_date"),
-        )
-        for meeting in meetings:
-            cycle_map[meeting["term"]][meeting["session"]][meeting["meeting"]] = {
-                "start_date": meeting["start_date"],
-                "end_date": meeting["end_date"],
-            }
-            if not is_dropdown:
-                cycle_map[meeting["term"]][meeting["session"]][meeting["meeting"]][
-                    "sitting_list"
-                ] = []
+            for session in sessions:
+                session_id = session["session"]
+                cycle_map[term_id][session_id] = {
+                    "start_date": session["start_date"],
+                    "end_date": session["end_date"],
+                }
+
+                # Get meetings for this session
+                meetings = (
+                    cycles_query.filter(term=term_id, session=session_id)
+                    .values("meeting")
+                    .annotate(
+                        start_date=Min("start_date"),
+                        end_date=Max("end_date"),
+                    )
+                )
+
+                for meeting in meetings:
+                    meeting_id = meeting["meeting"]
+                    cycle_map[term_id][session_id][meeting_id] = {
+                        "start_date": meeting["start_date"],
+                        "end_date": meeting["end_date"],
+                    }
+                    if not is_dropdown:
+                        cycle_map[term_id][session_id][meeting_id]["sitting_list"] = []
+
+        # Get all sittings for this house
+        sittings_query = Sitting.objects.filter(cycle__house=house_int)
 
-        data = {"catalogue_list": cycle_map, "total_count": sittings.count()}
+        # Apply term filter to sittings if provided
+        if term_filter:
+            sittings_query = sittings_query.filter(cycle__term=term_filter)
+
+        total_count = sittings_query.count()
+
+        data = {
+            "catalogue_list": cycle_map,
+            "total_count": total_count,
+        }
 
         if not is_dropdown:
-            for sitting in sittings:
+            # Use select_related to efficiently fetch related cycle data
+            for sitting in sittings_query.order_by("date").select_related("cycle"):
                 term = sitting.cycle.term
                 session = sitting.cycle.session
                 meeting = sitting.cycle.meeting
 
-                data["catalogue_list"][term][session][meeting]["sitting_list"].append(
-                    SittingSerializer(sitting).data
-                )
+                # Use a minimal serializer or manually create dict to avoid serialization overhead
+                sitting_data = {
+                    "sitting_id": sitting.sitting_id,
+                    "date": sitting.date,
+                    "filename": sitting.filename,
+                    "has_dataset": sitting.has_dataset,
+                    # "is_final": sitting.is_final,
+                }
+
+                cycle_map[term][session][meeting]["sitting_list"].append(sitting_data)
 
         return JsonResponse(data)