Hot fix PR to add expenditure and income to the aggregating API without changing the API parameter and field names for the demo, see #HEA-659

Author: chrispreee

apps/baseline/serializers.py

@@ -1509,11 +1509,17 @@ class Meta:
             "livelihood_activity_pk",
             "wealth_group_category_code",
             "population_estimate",
+            "product_cpc",
+            "product_common_name",
             "slice_sum_kcals_consumed",
             "sum_kcals_consumed",
             "kcals_consumed_percent",
-            "product_cpc",
-            "product_common_name",
+            "sum_income",
+            "slice_sum_income",
+            "income_percent",
+            "sum_expenditure",
+            "slice_sum_expenditure",
+            "expenditure_percent",
         )
 
     # For each of these aggregates the following calculation columns are added:
@@ -1524,6 +1530,8 @@ class Meta:
     # If no ordering is specified by the FilterSet, the results are ordered by percent descending in the order here.
     aggregates = {
         "kcals_consumed": Sum,
+        "income": Sum,
+        "expenditure": Sum,
     }
 
     # For each of these pairs, a URL parameter is created "slice_{field}", eg, ?slice_product=
@@ -1532,7 +1540,12 @@ class Meta:
     # For example: (product=R0 OR product=L0) AND (strategy_type=MilkProd OR strategy_type=CropProd)
     slice_fields = {
         "product": "livelihood_strategies__product__cpc__istartswith",
+        # this parameter must be set to one of values (not labels) from LivelihoodStrategyType, eg, MilkProduction
         "strategy_type": "livelihood_strategies__strategy_type__iexact",
+        # TODO: Support filter expressions on the right here, so we can slice on, for example, a
+        #  WealthGroupCharacteristicValue where WealthGroupCharacteristic is some hard-coded value,
+        #  eg, the slice on WGCV where WGC=PhoneOwnership, or on WGCV > 3 where WGC=HouseholdSize, eg:
+        #  {"phone_ownership": lambda val: Q(wgcv__path=val, wgc__path__code="PhoneOwnership")}
     }
 
     livelihood_zone_name = DictQuerySetField("livelihood_zone_name")
@@ -1565,6 +1578,14 @@ class Meta:
     sum_kcals_consumed = DictQuerySetField("sum_kcals_consumed")
     kcals_consumed_percent = DictQuerySetField("kcals_consumed_percent")
 
+    slice_sum_income = DictQuerySetField("slice_sum_income")
+    sum_income = DictQuerySetField("sum_income")
+    income_percent = DictQuerySetField("income_percent")
+
+    slice_sum_expenditure = DictQuerySetField("slice_sum_expenditure")
+    sum_expenditure = DictQuerySetField("sum_expenditure")
+    expenditure_percent = DictQuerySetField("expenditure_percent")
+
     def get_fields(self):
         """
         User can specify fields= parameter to specify a field list, comma-delimited.
@@ -1621,12 +1642,15 @@ def field_to_database_path(field_name):
             "livelihood_activity_pk": "livelihood_strategies__livelihoodactivity__pk",
             "wealth_group_category_code": "livelihood_strategies__livelihoodactivity__wealth_group__wealth_group_category__code",  # NOQA: E501
             "kcals_consumed": "livelihood_strategies__livelihoodactivity__kcals_consumed",
+            "income": "livelihood_strategies__livelihoodactivity__income",
+            "expenditure": "livelihood_strategies__livelihoodactivity__expenditure",
+            "percentage_kcals": "livelihood_strategies__livelihoodactivity__percentage_kcals",
             "livelihood_zone_name": f"livelihood_zone__name_{language_code}",
             "source_organization_pk": "source_organization__pk",
             "source_organization_name": "source_organization__name",
             "country_pk": "livelihood_zone__country__pk",
             "country_iso_en_name": "livelihood_zone__country__iso_en_name",
-            "product_cpc": "livelihood_strategies__product",
+            "product_cpc": "livelihood_strategies__product__cpc",
             "strategy_type": "livelihood_strategies__strategy_type",
             "product_common_name": f"livelihood_strategies__product__common_name_{language_code}",
         }.get(field_name, field_name)

apps/baseline/viewsets.py

@@ -1,7 +1,7 @@
 from django.apps import apps
 from django.conf import settings
 from django.db import models
-from django.db.models import F, OuterRef, Q, Subquery
+from django.db.models import F, FloatField, Q, Subquery
 from django.db.models.functions import Coalesce, NullIf
 from django.utils.translation import override
 from django_filters import rest_framework as filters
@@ -1628,6 +1628,115 @@ class CopingStrategyViewSet(BaseModelViewSet):
     ]
 
 
+MODELS_TO_SEARCH = [
+    {
+        "app_name": "common",
+        "model_name": "ClassifiedProduct",
+        "filter": {"key": "product", "label": "Product", "category": "products"},
+    },
+    {
+        "app_name": "metadata",
+        "model_name": "LivelihoodCategory",
+        "filter": {"key": "main_livelihood_category", "label": "Main Livelihood Category", "category": "zone_types"},
+    },
+    {
+        "app_name": "baseline",
+        "model_name": "LivelihoodZone",
+        "filter": {"key": "livelihood_zone", "label": "Livelihood zone", "category": "zones"},
+    },
+    {
+        "app_name": "metadata",
+        "model_name": "WealthCharacteristic",
+        "filter": {"key": "wealth_characteristic", "label": "Items", "category": "items"},
+    },
+    {
+        "app_name": "common",
+        "model_name": "Country",
+        "filter": {"key": "country", "label": "Country", "category": "countries"},
+    },
+]
+
+
+class LivelihoodZoneBaselineFacetedSearchView(APIView):
+    """
+    Performs a faceted search to find Livelihood Zone Baselines using a specified search term.
+
+    The search applies to multiple related models, filtering results based on the configured
+    criteria for each model. For each matching result, it calculates the number of unique
+    livelihood zones associated with the filter and includes relevant metadata in the response.
+    """
+
+    renderer_classes = [JSONRenderer]
+    permission_classes = [AllowAny]
+
+    def get(self, request, format=None):
+        """
+        Return a faceted set of matching filters
+        """
+        results = {}
+        search_term = request.query_params.get(settings.REST_FRAMEWORK["SEARCH_PARAM"], "")
+        language = request.query_params.get("language", "en")
+
+        if search_term:
+            for model_entry in MODELS_TO_SEARCH:
+                app_name = model_entry["app_name"]
+                model_name = model_entry["model_name"]
+                filter, filter_label, filter_category = (
+                    model_entry["filter"]["key"],
+                    model_entry["filter"]["label"],
+                    model_entry["filter"]["category"],
+                )
+                ModelClass = apps.get_model(app_name, model_name)
+                search_per_model = ModelClass.objects.search(search_term)
+                results[filter_category] = []
+                # for activating language
+                with override(language):
+                    for search_result in search_per_model:
+                        if model_name == "ClassifiedProduct":
+                            unique_zones = (
+                                LivelihoodStrategy.objects.filter(product=search_result)
+                                .values("livelihood_zone_baseline")
+                                .distinct()
+                                .count()
+                            )
+                            value_label, value = search_result.description, search_result.pk
+                        elif model_name == "LivelihoodCategory":
+                            unique_zones = LivelihoodZoneBaseline.objects.filter(
+                                main_livelihood_category=search_result
+                            ).count()
+                            value_label, value = search_result.description, search_result.pk
+                        elif model_name == "LivelihoodZone":
+                            unique_zones = LivelihoodZoneBaseline.objects.filter(livelihood_zone=search_result).count()
+                            value_label, value = search_result.name, search_result.pk
+                        elif model_name == "WealthCharacteristic":
+                            unique_zones = (
+                                WealthGroupCharacteristicValue.objects.filter(wealth_characteristic=search_result)
+                                .values("wealth_group__livelihood_zone_baseline")
+                                .distinct()
+                                .count()
+                            )
+                            value_label, value = search_result.description, search_result.pk
+                        elif model_name == "Country":
+                            unique_zones = (
+                                LivelihoodZoneBaseline.objects.filter(livelihood_zone__country=search_result)
+                                .distinct()
+                                .count()
+                            )
+                            value_label, value = search_result.iso_en_name, search_result.pk
+                        if unique_zones > 0:
+                            results[filter_category].append(
+                                {
+                                    "filter": filter,
+                                    "filter_label": filter_label,
+                                    "value_label": value_label,
+                                    "value": value,
+                                    "count": unique_zones,
+                                }
+                            )
+
+        return Response(results)
+
+
 class LivelihoodZoneBaselineReportViewSet(ModelViewSet):
     """
     There are two ‘levels’ of filter needed on this endpoint. The standard ones which are already on the LZB endpoint
@@ -1770,24 +1879,9 @@ def global_aggregates(self):
         """
         global_aggregates = {}
         for field_name, aggregate in self.serializer_class.aggregates.items():
-            subquery = LivelihoodZoneBaseline.objects.all()
-
-            # The FilterSet applies the global filters, such as Wealth Group Category.
-            # We also need to apply these to the subquery that gets the kcal totals per LZB (eg, the kcal_percent
-            # denominator), to restrict the 100% value by, for example, wealth group.
-            subquery = self.filter_queryset(subquery)
-
-            # Join to outer query
-            subquery = subquery.filter(pk=OuterRef("pk"))
-
-            # Annotate with the aggregate expression, eg, sum_kcals_consumed
             aggregate_field_name = self.serializer_class.aggregate_field_name(field_name, aggregate)
-            subquery = subquery.annotate(
-                **{aggregate_field_name: aggregate(self.serializer_class.field_to_database_path(field_name))}
-            ).values(aggregate_field_name)[:1]
-
-            global_aggregates[aggregate_field_name] = Subquery(subquery)
-
+            field_path = self.serializer_class.field_to_database_path(field_name)
+            global_aggregates[aggregate_field_name] = aggregate(field_path, default=0, output_field=FloatField())
         return global_aggregates
 
     def get_slice_aggregates(self):
@@ -1803,7 +1897,9 @@ def get_slice_aggregates(self):
                 # Annotate the queryset with the aggregate, eg, slice_sum_kcals_consumed, applying the slice filters.
                 # This is then divided by, eg, sum_kcals_consumed for the percentage of the slice.
                 field_path = self.serializer_class.field_to_database_path(field_name)
-                slice_aggregates[aggregate_field_name] = aggregate(field_path, filter=slice_filter, default=0)
+                slice_aggregates[aggregate_field_name] = aggregate(
+                    field_path, filter=slice_filter, default=0, output_field=FloatField()
+                )
         return slice_aggregates
 
     def get_slice_filters(self):
@@ -1825,117 +1921,10 @@ def get_calculations_on_aggregates(self):
         for field_name, aggregate in self.serializer_class.aggregates.items():
             slice_total = F(self.serializer_class.slice_aggregate_field_name(field_name, aggregate))
             overall_total = F(self.serializer_class.aggregate_field_name(field_name, aggregate))
-            expr = slice_total * 100 / NullIf(overall_total, 0)  # Protects against divide by zero
-            expr = Coalesce(expr, 0)  # Zero if no LivActivities found for prod/strategy slice
+            # Protect against divide by zero (divide by null returns null without error)
+            expr = slice_total * 100 / NullIf(overall_total, 0)
+            # Zero if no LivActivities found for prod/strategy slice, rather than null:
+            expr = Coalesce(expr, 0, output_field=FloatField())
             slice_percent_field_name = self.serializer_class.slice_percent_field_name(field_name, aggregate)
             calcs_on_aggregates[slice_percent_field_name] = expr
         return calcs_on_aggregates
-
-
-MODELS_TO_SEARCH = [
-    {
-        "app_name": "common",
-        "model_name": "ClassifiedProduct",
-        "filter": {"key": "product", "label": "Product", "category": "products"},
-    },
-    {
-        "app_name": "metadata",
-        "model_name": "LivelihoodCategory",
-        "filter": {"key": "main_livelihood_category", "label": "Main Livelihood Category", "category": "zone_types"},
-    },
-    {
-        "app_name": "baseline",
-        "model_name": "LivelihoodZone",
-        "filter": {"key": "livelihood_zone", "label": "Livelihood zone", "category": "zones"},
-    },
-    {
-        "app_name": "metadata",
-        "model_name": "WealthCharacteristic",
-        "filter": {"key": "wealth_characteristic", "label": "Items", "category": "items"},
-    },
-    {
-        "app_name": "common",
-        "model_name": "Country",
-        "filter": {"key": "country", "label": "Country", "category": "countries"},
-    },
-]
-
-
-class LivelihoodZoneBaselineFacetedSearchView(APIView):
-    """
-    Performs a faceted search to find Livelihood Zone Baselines using a specified search term.
-
-    The search applies to multiple related models, filtering results based on the configured
-    criteria for each model. For each matching result, it calculates the number of unique
-    livelihood zones associated with the filter and includes relevant metadata in the response.
-    """
-
-    renderer_classes = [JSONRenderer]
-    permission_classes = [AllowAny]
-
-    def get(self, request, format=None):
-        """
-        Return a faceted set of matching filters
-        """
-        results = {}
-        search_term = request.query_params.get(settings.REST_FRAMEWORK["SEARCH_PARAM"], "")
-        language = request.query_params.get("language", "en")
-
-        if search_term:
-            for model_entry in MODELS_TO_SEARCH:
-                app_name = model_entry["app_name"]
-                model_name = model_entry["model_name"]
-                filter, filter_label, filter_category = (
-                    model_entry["filter"]["key"],
-                    model_entry["filter"]["label"],
-                    model_entry["filter"]["category"],
-                )
-                ModelClass = apps.get_model(app_name, model_name)
-                search_per_model = ModelClass.objects.search(search_term)
-                results[filter_category] = []
-                # for activating language
-                with override(language):
-                    for search_result in search_per_model:
-                        if model_name == "ClassifiedProduct":
-                            unique_zones = (
-                                LivelihoodStrategy.objects.filter(product=search_result)
-                                .values("livelihood_zone_baseline")
-                                .distinct()
-                                .count()
-                            )
-                            value_label, value = search_result.description, search_result.pk
-                        elif model_name == "LivelihoodCategory":
-                            unique_zones = LivelihoodZoneBaseline.objects.filter(
-                                main_livelihood_category=search_result
-                            ).count()
-                            value_label, value = search_result.description, search_result.pk
-                        elif model_name == "LivelihoodZone":
-                            unique_zones = LivelihoodZoneBaseline.objects.filter(livelihood_zone=search_result).count()
-                            value_label, value = search_result.name, search_result.pk
-                        elif model_name == "WealthCharacteristic":
-                            unique_zones = (
-                                WealthGroupCharacteristicValue.objects.filter(wealth_characteristic=search_result)
-                                .values("wealth_group__livelihood_zone_baseline")
-                                .distinct()
-                                .count()
-                            )
-                            value_label, value = search_result.description, search_result.pk
-                        elif model_name == "Country":
-                            unique_zones = (
-                                LivelihoodZoneBaseline.objects.filter(livelihood_zone__country=search_result)
-                                .distinct()
-                                .count()
-                            )
-                            value_label, value = search_result.iso_en_name, search_result.pk
-                        if unique_zones > 0:
-                            results[filter_category].append(
-                                {
-                                    "filter": filter,
-                                    "filter_label": filter_label,
-                                    "value_label": value_label,
-                                    "value": value,
-                                    "count": unique_zones,
-                                }
-                            )
-
-        return Response(results)