From 03ed7e2e7b886ec7765cacbd7ea07898eb76119d Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Mon, 13 Jan 2025 16:50:40 +0300 Subject: [PATCH 01/11] Add SearchQueryMixin enabled Model managers for search to work see HEA-651 --- apps/baseline/models.py | 26 +++++++++++++++++++++++++- apps/metadata/models.py | 28 +++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/apps/baseline/models.py b/apps/baseline/models.py index a9dc8a0..2b5cf83 100644 --- a/apps/baseline/models.py +++ b/apps/baseline/models.py @@ -8,7 +8,7 @@ from django.contrib.gis.db import models from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator -from django.db.models import F +from django.db.models import F, Q from django.utils.translation import gettext_lazy as _ from model_utils.managers import InheritanceManager @@ -18,6 +18,7 @@ ClassifiedProduct, Country, Currency, + SearchQueryMixin, UnitOfMeasure, UnitOfMeasureConversion, ) @@ -62,6 +63,28 @@ class ExtraMeta: identifier = ["name"] +class LivelihoodZoneQuerySet(SearchQueryMixin, models.QuerySet): + """ + Searchable LivelihoodZones + """ + + def get_search_filter(self, search_term): + return ( + Q(code__iexact=search_term) + | Q(alternate_code__iexact=search_term) + | Q(name_en__iexact=search_term) + | Q(name_pt__iexact=search_term) + | Q(name_es__iexact=search_term) + | Q(name_fr__iexact=search_term) + | Q(name_ar__iexact=search_term) + | Q(description_en__iexact=search_term) + | Q(description_pt__iexact=search_term) + | Q(description_es__iexact=search_term) + | Q(description_fr__iexact=search_term) + | Q(description_ar__iexact=search_term) + ) + + class LivelihoodZone(common_models.Model): """ A geographical area within a Country in which people share broadly the same @@ -98,6 +121,7 @@ class LivelihoodZone(common_models.Model): name = TranslatedField(common_models.NameField(max_length=200, unique=True)) description = TranslatedField(common_models.DescriptionField()) country = models.ForeignKey(Country, verbose_name=_("Country"), db_column="country_code", on_delete=models.PROTECT) + objects = common_models.IdentifierManager.from_queryset(LivelihoodZoneQuerySet)() class Meta: verbose_name = _("Livelihood Zone") diff --git a/apps/metadata/models.py b/apps/metadata/models.py index 854b571..6a4052c 100644 --- a/apps/metadata/models.py +++ b/apps/metadata/models.py @@ -2,17 +2,38 @@ from django.contrib.gis.db import models from django.core.validators import MaxValueValidator, MinValueValidator +from django.db.models import Q from django.db.models.functions import Lower from django.utils.translation import gettext_lazy as _ from django.utils.translation import pgettext_lazy import common.models as common_models from common.fields import TranslatedField -from common.models import ClassifiedProduct, Country, Currency, UnitOfMeasure +from common.models import ClassifiedProduct, Country, Currency, UnitOfMeasure, SearchQueryMixin, IdentifierManager logger = logging.getLogger(__name__) +class ReferenceDataQuerySet(SearchQueryMixin, models.QuerySet): + """ + Extends ReferenceData QuerySet with custom search method + """ + def get_search_filter(self, search_term): + return ( + Q(code__iexact=search_term) + | Q(name_en__iexact=search_term) + | Q(name_pt__iexact=search_term) + | Q(name_es__iexact=search_term) + | Q(name_fr__iexact=search_term) + | Q(name_ar__iexact=search_term) + | Q(description_en__iexact=search_term) + | Q(description_pt__iexact=search_term) + | Q(description_es__iexact=search_term) + | Q(description_fr__iexact=search_term) + | Q(description_ar__iexact=search_term) + | Q(aliases__contains=[search_term.lower()]) + ) + class ReferenceData(common_models.Model): """ Reference data for a model. @@ -38,7 +59,7 @@ class ReferenceData(common_models.Model): verbose_name=_("aliases"), help_text=_("A list of alternate names for the object."), ) - + objects = IdentifierManager.from_queryset(ReferenceDataQuerySet)() def calculate_fields(self): # Ensure that aliases are lowercase and don't contain duplicates if self.aliases: @@ -76,6 +97,7 @@ class LivelihoodCategory(ReferenceData): ), help_text=_("Color hex value code for the Livelihood Category."), ) + objects = IdentifierManager.from_queryset(ReferenceDataQuerySet)() class Meta: verbose_name = _("Livelihood Category") @@ -121,7 +143,7 @@ class VariableType(models.TextChoices): default=VariableType.STR, help_text=_("Whether the field is numeric, character, boolean, etc."), ) - + objects = IdentifierManager.from_queryset(ReferenceDataQuerySet)() class Meta: verbose_name = _("Wealth Characteristic") verbose_name_plural = _("Wealth Characteristics") From c61850108f88324f2696d2b5d9add54ee1bac88a Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Mon, 13 Jan 2025 16:52:16 +0300 Subject: [PATCH 02/11] Create LivelihoodBaselineFacetedSearch and update LivelihoodBaselineViewset to support product and wealth_characteristics filters see HEA-651 --- apps/baseline/viewsets.py | 164 ++++++++++++++++++++++++++++++++++++++ hea/urls.py | 8 ++ 2 files changed, 172 insertions(+) diff --git a/apps/baseline/viewsets.py b/apps/baseline/viewsets.py index 9ba00d8..b7f625b 100644 --- a/apps/baseline/viewsets.py +++ b/apps/baseline/viewsets.py @@ -1,7 +1,17 @@ +import logging + +from django.apps import apps +from django.core.cache import cache from django.db import models from django.db.models import F, OuterRef, Q, Subquery from django.db.models.functions import Coalesce, NullIf +from django.utils import translation from django_filters import rest_framework as filters +from django_filters.filters import CharFilter +from rest_framework.permissions import AllowAny, DjangoModelPermissionsOrAnonReadOnly +from rest_framework.renderers import JSONRenderer +from rest_framework.response import Response +from rest_framework.views import APIView from rest_framework.viewsets import ModelViewSet from common.fields import translation_fields @@ -87,6 +97,8 @@ WildFoodGatheringSerializer, ) +logger = logging.getLogger(__name__) + class SourceOrganizationFilterSet(filters.FilterSet): class Meta: @@ -188,6 +200,50 @@ class Meta: label="Country", ) population_estimate = filters.RangeFilter(label="Population estimate range") + product = CharFilter(method="filter_by_product", label="Filter by Product") + wealth_characteristic = CharFilter( + method="filter_by_wealth_characteristic", label="Filter by Wealth Characteristic" + ) + + def filter_by_product(self, queryset, name, value): + """ + Filter the baseline by matching products + """ + field_lookups = [ + *[(field, "icontains") for field in translation_fields("product__common_name")], + ("product__cpc", "istartswith"), + *[(field, "icontains") for field in translation_fields("product__description")], + ("product__aliases", "icontains"), + ] + + q_object = Q() + for field, lookup in field_lookups: + q_object |= Q(**{f"{field}__{lookup}": value}) + + matching_baselines = LivelihoodStrategy.objects.filter(q_object).values("livelihood_zone_baseline") + + return queryset.filter(id__in=Subquery(matching_baselines)) + + def filter_by_wealth_characteristic(self, queryset, name, value): + """ + Filter the baseline by matching wealth_characteristic + """ + field_lookups = [ + *[(field, "icontains") for field in translation_fields("wealth_characteristic__name")], + ("wealth_characteristic__code", "istartswith"), + *[(field, "icontains") for field in translation_fields("wealth_characteristic__description")], + ("wealth_characteristic__aliases", "icontains"), + ] + + q_object = Q() + for field, lookup in field_lookups: + q_object |= Q(**{f"{field}__{lookup}": value}) + + matching_baselines = WealthGroupCharacteristicValue.objects.filter(q_object).values( + "wealth_group__livelihood_zone_baseline" + ) + + return queryset.filter(id__in=Subquery(matching_baselines)) class LivelihoodZoneBaselineViewSet(BaseModelViewSet): @@ -1778,3 +1834,111 @@ def get_calculations_on_aggregates(self): slice_percent_field_name = self.serializer_class.slice_percent_field_name(field_name, aggregate) calcs_on_aggregates[slice_percent_field_name] = expr return calcs_on_aggregates + + +MODELS_TO_SEARCH = [ + {"app_name": "common", "model_name": "ClassifiedProduct", "filter": ("product", "Product", "products")}, + { + "app_name": "metadata", + "model_name": "LivelihoodCategory", + "filter": ("main_livelihood_category", "main_livelihood_category", "zone_types"), + }, + { + "app_name": "metadata", + "model_name": "WealthCharacteristic", + "filter": ("wealth_characteristic", "Items", "items"), + }, + {"app_name": "common", "model_name": "Country", "filter": ("country", "Country", "countries")}, +] + + +class LivelihoodBaselineFacetedSearch(APIView): + """ + Use a search term to find Livelihood Zone Baselines through various filters. + + Returns faceted search results + """ + + renderer_classes = [JSONRenderer] + permission_classes = [DjangoModelPermissionsOrAnonReadOnly] + + def get_permissions(self): + # Bypass the queryset check for permission classes + if DjangoModelPermissionsOrAnonReadOnly in self.permission_classes: + return [AllowAny()] + return super().get_permissions() + + def get(self, request, format=None): + """ + Return a faceted set of matching filters + """ + results = {} + search_term = request.query_params.get("search", "") + language = request.query_params.get("language", "en") + + # Construct a cache key based on search and language parameters + cache_key = f"filters_{search_term}_{language}".lower() + cached_results = cache.get(cache_key) + logger.debug(f"Cached result: {cached_results}") + if cached_results: + logger.info(f"Cache hit for key: {cache_key}") + return Response(cached_results) + logger.info(f"Cache miss for key: {cache_key}") + if search_term: + for model_entry in MODELS_TO_SEARCH: + app_name = model_entry["app_name"] + model_name = model_entry["model_name"] + filter, filter_label, filter_category = model_entry["filter"] + ModelClass = apps.get_model(app_name, model_name) + search_per_model = ModelClass.objects.search(search_term) + results[filter_category] = [] + + translation.activate(language) + + for search_result in search_per_model: + if model_name == "ClassifiedProduct": + unique_zones = ( + LivelihoodStrategy.objects.filter(product=search_result) + .values("livelihood_zone_baseline") + .distinct() + .count() + ) + value_label, value = search_result.description, search_result.pk + elif model_name == "LivelihoodCategory": + unique_zones = LivelihoodZoneBaseline.objects.filter( + main_livelihood_category=search_result + ).count() + value_label, value = search_result.description, search_result.pk + elif model_name == "LivelihoodZone": + unique_zones = LivelihoodZoneBaseline.objects.filter(livelihood_zone=search_result).count() + value_label, value = search_result.name, search_result.pk + elif model_name == "WealthCharacteristic": + unique_zones = ( + WealthGroupCharacteristicValue.objects.filter(wealth_characteristic=search_result) + .values("wealth_group__livelihood_zone_baseline") + .distinct() + .count() + ) + value_label, value = search_result.description, search_result.pk + elif model_name == "Country": + unique_zones = ( + LivelihoodZoneBaseline.objects.filter(livelihood_zone__country=search_result) + .distinct() + .count() + ) + value_label, value = search_result.iso_en_name, search_result.pk + if unique_zones > 0: + results[filter_category].append( + { + "filter": filter, + "filter_label": filter_label, + "value_label": value_label, + "value": value, + "count": unique_zones, + } + ) + + # Cache results for a week maybe + cache.set(cache_key, results, timeout=60 * 60 * 24 * 7) + logger.info(f"Cache set for key: {cache_key}") + return Response(results) diff --git a/hea/urls.py b/hea/urls.py index 3ec63ba..3092dc6 100644 --- a/hea/urls.py +++ b/hea/urls.py @@ -24,6 +24,7 @@ HazardViewSet, HuntingViewSet, LivelihoodActivityViewSet, + LivelihoodBaselineFacetedSearch, LivelihoodProductCategoryViewSet, LivelihoodStrategyViewSet, LivelihoodZoneBaselineReportViewSet, @@ -130,6 +131,13 @@ # Provides il8n/set_language to change Django language: path("i18n/", include("django.conf.urls.i18n")), ] +urlpatterns += [ + path( + "api/livelihoodbaselinefacetedsearch/", + LivelihoodBaselineFacetedSearch.as_view(), + name="livelihood-baseline-faceted-search", + ), +] # Django's solution for translating JavaScript apps # Provides gettext translation functionality for Javascript clients (and ngettext, pgettext, iterpolate, etc.) From 6d3e2ac4e9352a192b9d5b3c0d7acafb95cc4ca3 Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Mon, 13 Jan 2025 16:52:35 +0300 Subject: [PATCH 03/11] Add unit test LivelihoodBaselineFacetedSearchTestCase see HEA-651 --- apps/baseline/tests/test_viewsets.py | 134 +++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/apps/baseline/tests/test_viewsets.py b/apps/baseline/tests/test_viewsets.py index c39ea0c..9377d32 100644 --- a/apps/baseline/tests/test_viewsets.py +++ b/apps/baseline/tests/test_viewsets.py @@ -12,6 +12,10 @@ from baseline.models import LivelihoodZoneBaseline from common.fields import translation_fields from common.tests.factories import ClassifiedProductFactory, CountryFactory +from metadata.tests.factories import ( + LivelihoodCategoryFactory, + WealthCharacteristicFactory, +) from .factories import ( BaselineLivelihoodActivityFactory, @@ -536,6 +540,136 @@ def test_population_estimate_range_filter(self): self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) + def test_filter_by_product(self): + product = ClassifiedProductFactory( + cpc="K0111", + description_en="my product", + common_name_en="common", + kcals_per_unit=550, + aliases=["test alias"], + ) + ClassifiedProductFactory(cpc="K01111") + baseline = LivelihoodZoneBaselineFactory() + LivelihoodStrategyFactory(product=product, livelihood_zone_baseline=baseline) + response = self.client.get(self.url, {"product": "K011"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content)), 1) + # filter by cpc + response = self.client.get(self.url, {"product": "K0111"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content)), 1) + # filter by cpc startswith + response = self.client.get(self.url, {"product": "K01111"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content)), 0) + # filter by description icontains + response = self.client.get(self.url, {"product": "my"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content.decode("utf-8"))), 1) + # filter by description + response = self.client.get(self.url, {"product": "my product"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content.decode("utf-8"))), 1) + # filter by alias + response = self.client.get(self.url, {"product": "test"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content.decode("utf-8"))), 1) + + def test_filter_by_wealth_characteristic(self): + baseline = LivelihoodZoneBaselineFactory() + wealth_characteristic = WealthCharacteristicFactory() + WealthGroupCharacteristicValueFactory( + wealth_group__livelihood_zone_baseline=baseline, wealth_characteristic=wealth_characteristic + ) + response = self.client.get(self.url, {"wealth_characteristic": wealth_characteristic.code}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content.decode("utf-8"))), 1) + response = self.client.get(self.url, {"wealth_characteristic": wealth_characteristic.name}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 1) + + +class LivelihoodBaselineFacetedSearchTestCase(APITestCase): + def setUp(self): + self.category1 = LivelihoodCategoryFactory() + self.baseline1 = LivelihoodZoneBaselineFactory(main_livelihood_category=self.category1) + self.baseline2 = LivelihoodZoneBaselineFactory(main_livelihood_category=self.category1) + self.baseline3 = LivelihoodZoneBaselineFactory() + self.product1 = ClassifiedProductFactory( + cpc="K0111", + description_en="my test", + common_name_en="common", + kcals_per_unit=550, + aliases=["test alias"], + ) + self.product2 = ClassifiedProductFactory( + cpc="L0111", + description_en="my mukera", + common_name_en="common mukera", + kcals_per_unit=550, + ) + LivelihoodStrategyFactory(product=self.product1, livelihood_zone_baseline=self.baseline1) + self.characteristic1 = WealthCharacteristicFactory(description_en="my test") + self.characteristic2 = WealthCharacteristicFactory(description_en="my mukera", description_fr="my test") + WealthGroupCharacteristicValueFactory( + wealth_group__livelihood_zone_baseline=self.baseline1, wealth_characteristic=self.characteristic1 + ) + WealthGroupCharacteristicValueFactory( + wealth_group__livelihood_zone_baseline=self.baseline2, wealth_characteristic=self.characteristic2 + ) + self.characteristic3 = WealthCharacteristicFactory() + self.strategy = LivelihoodStrategyFactory(product=self.product1) + self.baseline = LivelihoodZoneBaselineFactory(main_livelihood_category=self.category1) + self.url = reverse("livelihood-baseline-faceted-search") + + def test_search_with_product(self): + # Test when search matches entries + response = self.client.get(self.url, {"search": self.product1.description_en, "language": "en"}) + self.assertEqual(response.status_code, 200) + data = response.data + self.assertEqual(len(data["products"]), 1) + self.assertEqual(data["products"][0]["count"], 2) # 2 zones have this proudct + # Search by the second product + response = self.client.get( + self.url, + { + "search": self.product2.description_en, + }, + ) + self.assertEqual(response.status_code, 200) + data = response.data + self.assertEqual(len(data["products"]), 0) + + def test_search_with_wealth_characterstics(self): + # Test when search matches entries + response = self.client.get(self.url, {"search": self.characteristic1.description_en}) + self.assertEqual(response.status_code, 200) + data = response.data + self.assertEqual(len(data["items"]), 2) + self.assertEqual(data["items"][0]["count"], 1) # 1 zone for this characteristic + self.assertEqual(data["items"][1]["count"], 1) # 1 zone for this characteristic + # Search by the second characteristic + response = self.client.get( + self.url, + { + "search": self.characteristic2.description_en, + }, + ) + self.assertEqual(response.status_code, 200) + data = response.data + self.assertEqual(len(data["items"]), 1) + self.assertEqual(data["items"][0]["count"], 1) # 1 zone for this characteristic + # Search by the third characteristic + response = self.client.get( + self.url, + { + "search": self.characteristic3.description_en, + }, + ) + data = response.data + self.assertEqual(response.status_code, 200) + self.assertEqual(len(data["items"]), 0) + class LivelihoodProductCategoryViewSetTestCase(APITestCase): @classmethod From 7dc562a26ef360cc62ff8a10abc25a0ed4bd25b2 Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Mon, 13 Jan 2025 17:09:16 +0300 Subject: [PATCH 04/11] Add missing linting see HEA-651 --- apps/metadata/models.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/metadata/models.py b/apps/metadata/models.py index 6a4052c..1b15d88 100644 --- a/apps/metadata/models.py +++ b/apps/metadata/models.py @@ -9,7 +9,14 @@ import common.models as common_models from common.fields import TranslatedField -from common.models import ClassifiedProduct, Country, Currency, UnitOfMeasure, SearchQueryMixin, IdentifierManager +from common.models import ( + ClassifiedProduct, + Country, + Currency, + IdentifierManager, + SearchQueryMixin, + UnitOfMeasure, +) logger = logging.getLogger(__name__) @@ -18,6 +25,7 @@ class ReferenceDataQuerySet(SearchQueryMixin, models.QuerySet): """ Extends ReferenceData QuerySet with custom search method """ + def get_search_filter(self, search_term): return ( Q(code__iexact=search_term) @@ -34,6 +42,7 @@ def get_search_filter(self, search_term): | Q(aliases__contains=[search_term.lower()]) ) + class ReferenceData(common_models.Model): """ Reference data for a model. @@ -60,6 +69,7 @@ class ReferenceData(common_models.Model): help_text=_("A list of alternate names for the object."), ) objects = IdentifierManager.from_queryset(ReferenceDataQuerySet)() + def calculate_fields(self): # Ensure that aliases are lowercase and don't contain duplicates if self.aliases: @@ -144,6 +154,7 @@ class VariableType(models.TextChoices): help_text=_("Whether the field is numeric, character, boolean, etc."), ) objects = IdentifierManager.from_queryset(ReferenceDataQuerySet)() + class Meta: verbose_name = _("Wealth Characteristic") verbose_name_plural = _("Wealth Characteristics") From 9e392833167497465ef1083e3a92748a1f35902f Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Tue, 14 Jan 2025 08:23:30 +0300 Subject: [PATCH 05/11] Add missed CountryQuerySet see HEA-651 --- apps/common/models.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/apps/common/models.py b/apps/common/models.py index 30f4ceb..1afb6fc 100644 --- a/apps/common/models.py +++ b/apps/common/models.py @@ -490,6 +490,35 @@ def date_label(self): return date_text +class CountryQuerySet(SearchQueryMixin, models.QuerySet): + """ + Makes country searchable + """ + def get_search_filter(self, search_term): + return ( + Q(iso3166a2__iexact=search_term) + | Q(iso3166a3__iexact=search_term) + | Q(name__iexact=search_term) + | Q(iso_en_name__iexact=search_term) + | Q(iso_en_proper__iexact=search_term) + | Q(iso_en_ro_name__iexact=search_term) + | Q(iso_en_ro_proper__iexact=search_term) + | Q(iso_fr_name__iexact=search_term) + | Q(iso_fr_proper__iexact=search_term) + | Q(iso_es_name__iexact=search_term) + ) + +class CountryManager(models.Manager): + """ + Custom manager for Country model using CountryQuerySet. + """ + def get_queryset(self): + qs = CountryQuerySet(self.model, using=self._db) + return qs + + def search(self, search_term): + return self.get_queryset().search(search_term) + class Country(models.Model): """ A Country (or dependent territory or special area of geographical interest) included in ISO 3166. @@ -546,6 +575,7 @@ class Country(models.Model): verbose_name=_("ISO Spanish name"), help_text=_("The name in Spanish of the Country approved by the ISO 3166 Maintenance Agency"), ) + objects = CountryManager() def __str__(self): return self.iso_en_ro_name if self.iso_en_ro_name else "" From 398068db99d3e28629d6ec51fcd141cdc9071a4a Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Tue, 14 Jan 2025 08:28:30 +0300 Subject: [PATCH 06/11] Add missed CountryQuerySet see HEA-651 --- apps/common/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/common/models.py b/apps/common/models.py index 1afb6fc..3bd17b4 100644 --- a/apps/common/models.py +++ b/apps/common/models.py @@ -494,6 +494,7 @@ class CountryQuerySet(SearchQueryMixin, models.QuerySet): """ Makes country searchable """ + def get_search_filter(self, search_term): return ( Q(iso3166a2__iexact=search_term) @@ -508,10 +509,12 @@ def get_search_filter(self, search_term): | Q(iso_es_name__iexact=search_term) ) + class CountryManager(models.Manager): """ Custom manager for Country model using CountryQuerySet. """ + def get_queryset(self): qs = CountryQuerySet(self.model, using=self._db) return qs @@ -519,6 +522,7 @@ def get_queryset(self): def search(self, search_term): return self.get_queryset().search(search_term) + class Country(models.Model): """ A Country (or dependent territory or special area of geographical interest) included in ISO 3166. From 7e36b95c240b4b11e6636e5e948050037d74be8f Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Tue, 14 Jan 2025 10:28:58 +0300 Subject: [PATCH 07/11] Remove inconsistent cache which was an attempt to cache using LocMemCache see HEA-651 --- apps/baseline/viewsets.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/apps/baseline/viewsets.py b/apps/baseline/viewsets.py index b7f625b..95ef42c 100644 --- a/apps/baseline/viewsets.py +++ b/apps/baseline/viewsets.py @@ -1,7 +1,6 @@ import logging from django.apps import apps -from django.core.cache import cache from django.db import models from django.db.models import F, OuterRef, Q, Subquery from django.db.models.functions import Coalesce, NullIf @@ -1876,14 +1875,6 @@ def get(self, request, format=None): search_term = request.query_params.get("search", "") language = request.query_params.get("language", "en") - # Construct a cache key based on search and language parameters - cache_key = f"filters_{search_term}_{language}".lower() - cached_results = cache.get(cache_key) - logger.debug(f"Cached result: {cached_results}") - if cached_results: - logger.info(f"Cache hit for key: {cache_key}") - return Response(cached_results) - logger.info(f"Cache miss for key: {cache_key}") if search_term: for model_entry in MODELS_TO_SEARCH: app_name = model_entry["app_name"] @@ -1938,7 +1929,4 @@ def get(self, request, format=None): } ) - # Cache results for a week maybe - cache.set(cache_key, results, timeout=60 * 60 * 24 * 7) - logger.info(f"Cache set for key: {cache_key}") return Response(results) From d6c89b4fa7ff9b1e5ab35fc461b85312c4959cc3 Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Wed, 15 Jan 2025 12:58:44 +0300 Subject: [PATCH 08/11] Add missed LivelihoodZone to the model list for search see HEA-651 --- apps/baseline/viewsets.py | 128 +++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 57 deletions(-) diff --git a/apps/baseline/viewsets.py b/apps/baseline/viewsets.py index 95ef42c..1d4aaab 100644 --- a/apps/baseline/viewsets.py +++ b/apps/baseline/viewsets.py @@ -1,10 +1,8 @@ -import logging - from django.apps import apps from django.db import models from django.db.models import F, OuterRef, Q, Subquery from django.db.models.functions import Coalesce, NullIf -from django.utils import translation +from django.utils.translation import override from django_filters import rest_framework as filters from django_filters.filters import CharFilter from rest_framework.permissions import AllowAny, DjangoModelPermissionsOrAnonReadOnly @@ -96,8 +94,6 @@ WildFoodGatheringSerializer, ) -logger = logging.getLogger(__name__) - class SourceOrganizationFilterSet(filters.FilterSet): class Meta: @@ -1836,26 +1832,41 @@ def get_calculations_on_aggregates(self): MODELS_TO_SEARCH = [ - {"app_name": "common", "model_name": "ClassifiedProduct", "filter": ("product", "Product", "products")}, + { + "app_name": "common", + "model_name": "ClassifiedProduct", + "filter": {"key": "product", "label": "Product", "category": "products"}, + }, { "app_name": "metadata", "model_name": "LivelihoodCategory", - "filter": ("main_livelihood_category", "main_livelihood_category", "zone_types"), + "filter": {"key": "main_livelihood_category", "label": "Main Livelihood Category", "category": "zone_types"}, + }, + { + "app_name": "baseline", + "model_name": "LivelihoodZone", + "filter": {"key": "livelihood_zone", "label": "Livelihood zone", "category": "zones"}, }, { "app_name": "metadata", "model_name": "WealthCharacteristic", - "filter": ("wealth_characteristic", "Items", "items"), + "filter": {"key": "wealth_characteristic", "label": "Items", "category": "items"}, + }, + { + "app_name": "common", + "model_name": "Country", + "filter": {"key": "country", "label": "Country", "category": "countries"}, }, - {"app_name": "common", "model_name": "Country", "filter": ("country", "Country", "countries")}, ] class LivelihoodBaselineFacetedSearch(APIView): """ - Use a search term to find Livelihood Zone Baselines through various filters. + Performs a faceted search to find Livelihood Zone Baselines using a specified search term. - Returns faceted search results + The search applies to multiple related models, filtering results based on the configured + criteria for each model. For each matching result, it calculates the number of unique + livelihodd zones associated with the filter and includes relevant metadata in the response. """ renderer_classes = [JSONRenderer] @@ -1879,54 +1890,57 @@ def get(self, request, format=None): for model_entry in MODELS_TO_SEARCH: app_name = model_entry["app_name"] model_name = model_entry["model_name"] - filter, filter_label, filter_category = model_entry["filter"] + filter, filter_label, filter_category = ( + model_entry["filter"]["key"], + model_entry["filter"]["label"], + model_entry["filter"]["category"], + ) ModelClass = apps.get_model(app_name, model_name) search_per_model = ModelClass.objects.search(search_term) results[filter_category] = [] - - translation.activate(language) - - for search_result in search_per_model: - if model_name == "ClassifiedProduct": - unique_zones = ( - LivelihoodStrategy.objects.filter(product=search_result) - .values("livelihood_zone_baseline") - .distinct() - .count() - ) - value_label, value = search_result.description, search_result.pk - elif model_name == "LivelihoodCategory": - unique_zones = LivelihoodZoneBaseline.objects.filter( - main_livelihood_category=search_result - ).count() - value_label, value = search_result.description, search_result.pk - elif model_name == "LivelihoodZone": - unique_zones = LivelihoodZoneBaseline.objects.filter(livelihood_zone=search_result).count() - value_label, value = search_result.name, search_result.pk - elif model_name == "WealthCharacteristic": - unique_zones = ( - WealthGroupCharacteristicValue.objects.filter(wealth_characteristic=search_result) - .values("wealth_group__livelihood_zone_baseline") - .distinct() - .count() - ) - value_label, value = search_result.description, search_result.pk - elif model_name == "Country": - unique_zones = ( - LivelihoodZoneBaseline.objects.filter(livelihood_zone__country=search_result) - .distinct() - .count() - ) - value_label, value = search_result.iso_en_name, search_result.pk - if unique_zones > 0: - results[filter_category].append( - { - "filter": filter, - "filter_label": filter_label, - "value_label": value_label, - "value": value, - "count": unique_zones, - } - ) + # for activating language + with override(language): + for search_result in search_per_model: + if model_name == "ClassifiedProduct": + unique_zones = ( + LivelihoodStrategy.objects.filter(product=search_result) + .values("livelihood_zone_baseline") + .distinct() + .count() + ) + value_label, value = search_result.description, search_result.pk + elif model_name == "LivelihoodCategory": + unique_zones = LivelihoodZoneBaseline.objects.filter( + main_livelihood_category=search_result + ).count() + value_label, value = search_result.description, search_result.pk + elif model_name == "LivelihoodZone": + unique_zones = LivelihoodZoneBaseline.objects.filter(livelihood_zone=search_result).count() + value_label, value = search_result.name, search_result.pk + elif model_name == "WealthCharacteristic": + unique_zones = ( + WealthGroupCharacteristicValue.objects.filter(wealth_characteristic=search_result) + .values("wealth_group__livelihood_zone_baseline") + .distinct() + .count() + ) + value_label, value = search_result.description, search_result.pk + elif model_name == "Country": + unique_zones = ( + LivelihoodZoneBaseline.objects.filter(livelihood_zone__country=search_result) + .distinct() + .count() + ) + value_label, value = search_result.iso_en_name, search_result.pk + if unique_zones > 0: + results[filter_category].append( + { + "filter": filter, + "filter_label": filter_label, + "value_label": value_label, + "value": value, + "count": unique_zones, + } + ) return Response(results) From a435ccaa65f2d4d7e0917e70e5e384accb7673b6 Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Tue, 21 Jan 2025 12:18:05 +0300 Subject: [PATCH 09/11] Address PR feedback see HEA-651 --- apps/baseline/models.py | 20 ++++++++-------- apps/baseline/tests/test_viewsets.py | 2 +- apps/baseline/viewsets.py | 15 +++++------- apps/common/models.py | 34 ++++++++++------------------ apps/metadata/models.py | 20 ++++++++-------- hea/urls.py | 4 ++-- 6 files changed, 41 insertions(+), 54 deletions(-) diff --git a/apps/baseline/models.py b/apps/baseline/models.py index 2b5cf83..6467c7f 100644 --- a/apps/baseline/models.py +++ b/apps/baseline/models.py @@ -72,16 +72,16 @@ def get_search_filter(self, search_term): return ( Q(code__iexact=search_term) | Q(alternate_code__iexact=search_term) - | Q(name_en__iexact=search_term) - | Q(name_pt__iexact=search_term) - | Q(name_es__iexact=search_term) - | Q(name_fr__iexact=search_term) - | Q(name_ar__iexact=search_term) - | Q(description_en__iexact=search_term) - | Q(description_pt__iexact=search_term) - | Q(description_es__iexact=search_term) - | Q(description_fr__iexact=search_term) - | Q(description_ar__iexact=search_term) + | Q(name_en__icontains=search_term) + | Q(name_pt__icontains=search_term) + | Q(name_es__icontains=search_term) + | Q(name_fr__icontains=search_term) + | Q(name_ar__icontains=search_term) + | Q(description_en__icontains=search_term) + | Q(description_pt__icontains=search_term) + | Q(description_es__icontains=search_term) + | Q(description_fr__icontains=search_term) + | Q(description_ar__icontains=search_term) ) diff --git a/apps/baseline/tests/test_viewsets.py b/apps/baseline/tests/test_viewsets.py index 9377d32..130ccaa 100644 --- a/apps/baseline/tests/test_viewsets.py +++ b/apps/baseline/tests/test_viewsets.py @@ -589,7 +589,7 @@ def test_filter_by_wealth_characteristic(self): self.assertEqual(len(response.json()), 1) -class LivelihoodBaselineFacetedSearchTestCase(APITestCase): +class LivelihoodBaselineFacetedSearchViewTestCase(APITestCase): def setUp(self): self.category1 = LivelihoodCategoryFactory() self.baseline1 = LivelihoodZoneBaselineFactory(main_livelihood_category=self.category1) diff --git a/apps/baseline/viewsets.py b/apps/baseline/viewsets.py index 1d4aaab..a56db66 100644 --- a/apps/baseline/viewsets.py +++ b/apps/baseline/viewsets.py @@ -5,7 +5,7 @@ from django.utils.translation import override from django_filters import rest_framework as filters from django_filters.filters import CharFilter -from rest_framework.permissions import AllowAny, DjangoModelPermissionsOrAnonReadOnly +from rest_framework.permissions import AllowAny from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from rest_framework.views import APIView @@ -225,7 +225,7 @@ def filter_by_wealth_characteristic(self, queryset, name, value): """ field_lookups = [ *[(field, "icontains") for field in translation_fields("wealth_characteristic__name")], - ("wealth_characteristic__code", "istartswith"), + ("wealth_characteristic__code", "iexact"), *[(field, "icontains") for field in translation_fields("wealth_characteristic__description")], ("wealth_characteristic__aliases", "icontains"), ] @@ -1860,23 +1860,20 @@ def get_calculations_on_aggregates(self): ] -class LivelihoodBaselineFacetedSearch(APIView): +class LivelihoodBaselineFacetedSearchView(APIView): """ Performs a faceted search to find Livelihood Zone Baselines using a specified search term. The search applies to multiple related models, filtering results based on the configured criteria for each model. For each matching result, it calculates the number of unique - livelihodd zones associated with the filter and includes relevant metadata in the response. + livelihood zones associated with the filter and includes relevant metadata in the response. """ renderer_classes = [JSONRenderer] - permission_classes = [DjangoModelPermissionsOrAnonReadOnly] + permission_classes = [] def get_permissions(self): - # Bypass the queryset check for permission classes - if DjangoModelPermissionsOrAnonReadOnly in self.permission_classes: - return [AllowAny()] - return super().get_permissions() + return [AllowAny()] def get(self, request, format=None): """ diff --git a/apps/common/models.py b/apps/common/models.py index 3bd17b4..dfa8eb6 100644 --- a/apps/common/models.py +++ b/apps/common/models.py @@ -499,30 +499,17 @@ def get_search_filter(self, search_term): return ( Q(iso3166a2__iexact=search_term) | Q(iso3166a3__iexact=search_term) - | Q(name__iexact=search_term) - | Q(iso_en_name__iexact=search_term) - | Q(iso_en_proper__iexact=search_term) - | Q(iso_en_ro_name__iexact=search_term) - | Q(iso_en_ro_proper__iexact=search_term) - | Q(iso_fr_name__iexact=search_term) - | Q(iso_fr_proper__iexact=search_term) - | Q(iso_es_name__iexact=search_term) + | Q(name__icontains=search_term) + | Q(iso_en_name__icontains=search_term) + | Q(iso_en_proper__icontains=search_term) + | Q(iso_en_ro_name__icontains=search_term) + | Q(iso_en_ro_proper__icontains=search_term) + | Q(iso_fr_name__icontains=search_term) + | Q(iso_fr_proper__icontains=search_term) + | Q(iso_es_name__icontains=search_term) ) -class CountryManager(models.Manager): - """ - Custom manager for Country model using CountryQuerySet. - """ - - def get_queryset(self): - qs = CountryQuerySet(self.model, using=self._db) - return qs - - def search(self, search_term): - return self.get_queryset().search(search_term) - - class Country(models.Model): """ A Country (or dependent territory or special area of geographical interest) included in ISO 3166. @@ -579,7 +566,7 @@ class Country(models.Model): verbose_name=_("ISO Spanish name"), help_text=_("The name in Spanish of the Country approved by the ISO 3166 Maintenance Agency"), ) - objects = CountryManager() + objects = IdentifierManager.from_queryset(CountryQuerySet)() def __str__(self): return self.iso_en_ro_name if self.iso_en_ro_name else "" @@ -588,6 +575,9 @@ class Meta: verbose_name = _("Country") verbose_name_plural = _("Countries") + class ExtraMeta: + identifier = ["iso_en_ro_name"] + class Currency(models.Model): """ diff --git a/apps/metadata/models.py b/apps/metadata/models.py index 1b15d88..01c2c97 100644 --- a/apps/metadata/models.py +++ b/apps/metadata/models.py @@ -29,16 +29,16 @@ class ReferenceDataQuerySet(SearchQueryMixin, models.QuerySet): def get_search_filter(self, search_term): return ( Q(code__iexact=search_term) - | Q(name_en__iexact=search_term) - | Q(name_pt__iexact=search_term) - | Q(name_es__iexact=search_term) - | Q(name_fr__iexact=search_term) - | Q(name_ar__iexact=search_term) - | Q(description_en__iexact=search_term) - | Q(description_pt__iexact=search_term) - | Q(description_es__iexact=search_term) - | Q(description_fr__iexact=search_term) - | Q(description_ar__iexact=search_term) + | Q(name_en__icontains=search_term) + | Q(name_pt__icontains=search_term) + | Q(name_es__icontains=search_term) + | Q(name_fr__icontains=search_term) + | Q(name_ar__icontains=search_term) + | Q(description_en__icontains=search_term) + | Q(description_pt__icontains=search_term) + | Q(description_es__icontains=search_term) + | Q(description_fr__icontains=search_term) + | Q(description_ar__icontains=search_term) | Q(aliases__contains=[search_term.lower()]) ) diff --git a/hea/urls.py b/hea/urls.py index 3092dc6..cf69a5c 100644 --- a/hea/urls.py +++ b/hea/urls.py @@ -24,7 +24,7 @@ HazardViewSet, HuntingViewSet, LivelihoodActivityViewSet, - LivelihoodBaselineFacetedSearch, + LivelihoodBaselineFacetedSearchView, LivelihoodProductCategoryViewSet, LivelihoodStrategyViewSet, LivelihoodZoneBaselineReportViewSet, @@ -134,7 +134,7 @@ urlpatterns += [ path( "api/livelihoodbaselinefacetedsearch/", - LivelihoodBaselineFacetedSearch.as_view(), + LivelihoodBaselineFacetedSearchView.as_view(), name="livelihood-baseline-faceted-search", ), ] From c529806254bec2f184ee36c85b1c8af620a7a154 Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Wed, 22 Jan 2025 08:12:04 +0300 Subject: [PATCH 10/11] Address additional PR feedback see HEA-651 --- apps/baseline/tests/test_viewsets.py | 37 ++++++++++++++++++++++------ apps/baseline/viewsets.py | 10 +++----- hea/settings/base.py | 1 + hea/urls.py | 8 +++--- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/apps/baseline/tests/test_viewsets.py b/apps/baseline/tests/test_viewsets.py index 130ccaa..7e28bdd 100644 --- a/apps/baseline/tests/test_viewsets.py +++ b/apps/baseline/tests/test_viewsets.py @@ -589,7 +589,7 @@ def test_filter_by_wealth_characteristic(self): self.assertEqual(len(response.json()), 1) -class LivelihoodBaselineFacetedSearchViewTestCase(APITestCase): +class LivelihoodZoneBaselineFacetedSearchViewTestCase(APITestCase): def setUp(self): self.category1 = LivelihoodCategoryFactory() self.baseline1 = LivelihoodZoneBaselineFactory(main_livelihood_category=self.category1) @@ -618,17 +618,38 @@ def setUp(self): wealth_group__livelihood_zone_baseline=self.baseline2, wealth_characteristic=self.characteristic2 ) self.characteristic3 = WealthCharacteristicFactory() - self.strategy = LivelihoodStrategyFactory(product=self.product1) + self.strategy = LivelihoodStrategyFactory(product=self.product1, livelihood_zone_baseline=self.baseline3) self.baseline = LivelihoodZoneBaselineFactory(main_livelihood_category=self.category1) - self.url = reverse("livelihood-baseline-faceted-search") + self.url = reverse("livelihood-zone-baseline-faceted-search") def test_search_with_product(self): # Test when search matches entries response = self.client.get(self.url, {"search": self.product1.description_en, "language": "en"}) self.assertEqual(response.status_code, 200) - data = response.data - self.assertEqual(len(data["products"]), 1) - self.assertEqual(data["products"][0]["count"], 2) # 2 zones have this proudct + search_data = response.data + self.assertEqual(len(search_data["products"]), 1) + self.assertEqual(search_data["products"][0]["count"], 2) # 2 zones have this product + # confirm the product value is correct + self.assertEqual(search_data["products"][0]["value"], self.product1.cpc) + # Apply the filters to the baseline + baseline_url = reverse("livelihoodzonebaseline-list") + response = self.client.get( + baseline_url, {search_data["products"][0]["filter"]: search_data["products"][0]["value"]} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content)), 2) + data = json.loads(response.content) + self.assertTrue(any(d["name"] == self.baseline1.name for d in data)) + self.assertTrue(any(d["name"] == self.baseline3.name for d in data)) + self.assertFalse(any(d["name"] == self.baseline2.name for d in data)) + + response = self.client.get(baseline_url, {search_data["items"][0]["filter"]: search_data["items"][0]["value"]}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(json.loads(response.content)), 1) + data = json.loads(response.content) + self.assertTrue(any(d["name"] == self.baseline1.name for d in data)) + self.assertFalse(any(d["name"] == self.baseline2.name for d in data)) + self.assertFalse(any(d["name"] == self.baseline3.name for d in data)) # Search by the second product response = self.client.get( self.url, @@ -637,8 +658,8 @@ def test_search_with_product(self): }, ) self.assertEqual(response.status_code, 200) - data = response.data - self.assertEqual(len(data["products"]), 0) + search_data = response.data + self.assertEqual(len(search_data["products"]), 0) def test_search_with_wealth_characterstics(self): # Test when search matches entries diff --git a/apps/baseline/viewsets.py b/apps/baseline/viewsets.py index a56db66..20e9d13 100644 --- a/apps/baseline/viewsets.py +++ b/apps/baseline/viewsets.py @@ -1,4 +1,5 @@ from django.apps import apps +from django.conf import settings from django.db import models from django.db.models import F, OuterRef, Q, Subquery from django.db.models.functions import Coalesce, NullIf @@ -1860,7 +1861,7 @@ def get_calculations_on_aggregates(self): ] -class LivelihoodBaselineFacetedSearchView(APIView): +class LivelihoodZoneBaselineFacetedSearchView(APIView): """ Performs a faceted search to find Livelihood Zone Baselines using a specified search term. @@ -1870,17 +1871,14 @@ class LivelihoodBaselineFacetedSearchView(APIView): """ renderer_classes = [JSONRenderer] - permission_classes = [] - - def get_permissions(self): - return [AllowAny()] + permission_classes = [AllowAny] def get(self, request, format=None): """ Return a faceted set of matching filters """ results = {} - search_term = request.query_params.get("search", "") + search_term = request.query_params.get(settings.REST_FRAMEWORK["SEARCH_PARAM"], "") language = request.query_params.get("language", "en") if search_term: diff --git a/hea/settings/base.py b/hea/settings/base.py index 4c1092c..9c5430e 100644 --- a/hea/settings/base.py +++ b/hea/settings/base.py @@ -150,6 +150,7 @@ "TEST_REQUEST_DEFAULT_FORMAT": "json", "EXCEPTION_HANDLER": "apps.common.exception_handlers.drf_exception_handler", "STRICT_JSON": True, + "SEARCH_PARAM": "search", } ROOT_URLCONF = "hea.urls" diff --git a/hea/urls.py b/hea/urls.py index cf69a5c..82c59ba 100644 --- a/hea/urls.py +++ b/hea/urls.py @@ -24,7 +24,7 @@ HazardViewSet, HuntingViewSet, LivelihoodActivityViewSet, - LivelihoodBaselineFacetedSearchView, + LivelihoodZoneBaselineFacetedSearchView, LivelihoodProductCategoryViewSet, LivelihoodStrategyViewSet, LivelihoodZoneBaselineReportViewSet, @@ -133,9 +133,9 @@ ] urlpatterns += [ path( - "api/livelihoodbaselinefacetedsearch/", - LivelihoodBaselineFacetedSearchView.as_view(), - name="livelihood-baseline-faceted-search", + "api/livelihoodzonebaselinefacetedsearch/", + LivelihoodZoneBaselineFacetedSearchView.as_view(), + name="livelihood-zone-baseline-faceted-search", ), ] From 1489e7a25494062e1cd0677e02c5f96ba807588e Mon Sep 17 00:00:00 2001 From: Girum Bizuayehu Date: Wed, 22 Jan 2025 08:13:54 +0300 Subject: [PATCH 11/11] Address additional PR feedback see HEA-651 --- hea/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hea/urls.py b/hea/urls.py index 82c59ba..1ba8a0b 100644 --- a/hea/urls.py +++ b/hea/urls.py @@ -24,9 +24,9 @@ HazardViewSet, HuntingViewSet, LivelihoodActivityViewSet, - LivelihoodZoneBaselineFacetedSearchView, LivelihoodProductCategoryViewSet, LivelihoodStrategyViewSet, + LivelihoodZoneBaselineFacetedSearchView, LivelihoodZoneBaselineReportViewSet, LivelihoodZoneBaselineViewSet, LivelihoodZoneViewSet,