Merge pull request #17 from CivicDataLab/dev

Collaboration and Geography Enhancements

Author: Deepthi-Chand

api/management/commands/populate_geographies.py

@@ -1,6 +1,6 @@
 """
 Django management command to populate Asia Pacific geography data.
-Covers: India, Indonesia, Thailand at state/province level.
+Covers: India, Indonesia, Thailand, Philippines at state/province level.
 
 Usage:
     python manage.py populate_geographies
@@ -17,7 +17,9 @@
 
 
 class Command(BaseCommand):
-    help = "Populate Asia Pacific geography data (India, Indonesia, Thailand)"
+    help = (
+        "Populate Asia Pacific geography data (India, Indonesia, Thailand, Philippines)"
+    )
 
     def add_arguments(self, parser: CommandParser) -> None:
         parser.add_argument(
@@ -64,6 +66,9 @@ def _populate_geographies(self) -> None:
         # Thailand and its provinces
         self._populate_thailand(asia_pacific)
 
+        # Philippines and its provinces
+        self._populate_philippines(asia_pacific)
+
     def _populate_india(self, parent: Geography) -> None:
         """Populate India and its states/UTs."""
         india, created = Geography.objects.get_or_create(
@@ -302,3 +307,133 @@ def _populate_thailand(self, parent: Geography) -> None:
         self.stdout.write(
             f"Thai provinces: {created_count} created, {len(thai_provinces) - created_count} already existed"
         )
+
+    def _populate_philippines(self, parent: Geography) -> None:
+        """Populate Philippines and its provinces."""
+        philippines, created = Geography.objects.get_or_create(
+            name="Philippines",
+            defaults={"code": "PH", "type": GeoTypes.COUNTRY, "parent_id": parent},
+        )
+        if created:
+            self.stdout.write(f"Created country: {philippines.name}")
+        else:
+            self.stdout.write(f"Country already exists: {philippines.name}")
+
+        philippine_provinces = [
+            # Luzon - NCR
+            ("Metro Manila", "NCR"),
+            # Luzon - CAR
+            ("Abra", "ABR"),
+            ("Apayao", "APA"),
+            ("Benguet", "BEN"),
+            ("Ifugao", "IFU"),
+            ("Kalinga", "KAL"),
+            ("Mountain Province", "MOU"),
+            # Luzon - Region I (Ilocos)
+            ("Ilocos Norte", "ILN"),
+            ("Ilocos Sur", "ILS"),
+            ("La Union", "LUN"),
+            ("Pangasinan", "PAN"),
+            # Luzon - Region II (Cagayan Valley)
+            ("Batanes", "BTN"),
+            ("Cagayan", "CAG"),
+            ("Isabela", "ISA"),
+            ("Nueva Vizcaya", "NUV"),
+            ("Quirino", "QUI"),
+            # Luzon - Region III (Central Luzon)
+            ("Aurora", "AUR"),
+            ("Bataan", "BAN"),
+            ("Bulacan", "BUL"),
+            ("Nueva Ecija", "NUE"),
+            ("Pampanga", "PAM"),
+            ("Tarlac", "TAR"),
+            ("Zambales", "ZMB"),
+            # Luzon - Region IV-A (CALABARZON)
+            ("Batangas", "BTG"),
+            ("Cavite", "CAV"),
+            ("Laguna", "LAG"),
+            ("Quezon", "QUE"),
+            ("Rizal", "RIZ"),
+            # Luzon - Region IV-B (MIMAROPA)
+            ("Marinduque", "MAD"),
+            ("Occidental Mindoro", "MDC"),
+            ("Oriental Mindoro", "MDR"),
+            ("Palawan", "PLW"),
+            ("Romblon", "ROM"),
+            # Luzon - Region V (Bicol)
+            ("Albay", "ALB"),
+            ("Camarines Norte", "CAN"),
+            ("Camarines Sur", "CAS"),
+            ("Catanduanes", "CAT"),
+            ("Masbate", "MAS"),
+            ("Sorsogon", "SOR"),
+            # Visayas - Region VI (Western Visayas)
+            ("Aklan", "AKL"),
+            ("Antique", "ANT"),
+            ("Capiz", "CAP"),
+            ("Guimaras", "GUI"),
+            ("Iloilo", "ILI"),
+            ("Negros Occidental", "NEC"),
+            # Visayas - Region VII (Central Visayas)
+            ("Bohol", "BOH"),
+            ("Cebu", "CEB"),
+            ("Negros Oriental", "NER"),
+            ("Siquijor", "SIG"),
+            # Visayas - Region VIII (Eastern Visayas)
+            ("Biliran", "BIL"),
+            ("Eastern Samar", "EAS"),
+            ("Leyte", "LEY"),
+            ("Northern Samar", "NSA"),
+            ("Samar", "WSA"),
+            ("Southern Leyte", "SLE"),
+            # Mindanao - Region IX (Zamboanga Peninsula)
+            ("Zamboanga del Norte", "ZAN"),
+            ("Zamboanga del Sur", "ZAS"),
+            ("Zamboanga Sibugay", "ZSI"),
+            # Mindanao - Region X (Northern Mindanao)
+            ("Bukidnon", "BUK"),
+            ("Camiguin", "CAM"),
+            ("Lanao del Norte", "LAN"),
+            ("Misamis Occidental", "MSC"),
+            ("Misamis Oriental", "MSR"),
+            # Mindanao - Region XI (Davao)
+            ("Davao de Oro", "COM"),
+            ("Davao del Norte", "DAV"),
+            ("Davao del Sur", "DAS"),
+            ("Davao Occidental", "DAO"),
+            ("Davao Oriental", "DAO"),
+            # Mindanao - Region XII (SOCCSKSARGEN)
+            ("Cotabato", "NCO"),
+            ("Sarangani", "SAR"),
+            ("South Cotabato", "SCO"),
+            ("Sultan Kudarat", "SUK"),
+            # Mindanao - Region XIII (Caraga)
+            ("Agusan del Norte", "AGN"),
+            ("Agusan del Sur", "AGS"),
+            ("Dinagat Islands", "DIN"),
+            ("Surigao del Norte", "SUN"),
+            ("Surigao del Sur", "SUR"),
+            # Mindanao - BARMM
+            ("Basilan", "BAS"),
+            ("Lanao del Sur", "LAS"),
+            ("Maguindanao", "MAG"),
+            ("Sulu", "SLU"),
+            ("Tawi-Tawi", "TAW"),
+        ]
+
+        created_count = 0
+        for province_name, province_code in philippine_provinces:
+            _, created = Geography.objects.get_or_create(
+                name=province_name,
+                defaults={
+                    "code": province_code,
+                    "type": GeoTypes.STATE,
+                    "parent_id": philippines,
+                },
+            )
+            if created:
+                created_count += 1
+
+        self.stdout.write(
+            f"Philippine provinces: {created_count} created, {len(philippine_provinces) - created_count} already existed"
+        )

api/models/Geography.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import List, Optional
 
 from django.db import models
 
@@ -19,6 +19,49 @@ class Geography(models.Model):
     def __str__(self) -> str:
         return f"{self.name} ({self.type})"
 
+    def get_all_descendant_names(self) -> List[str]:
+        """
+        Get all descendant geography names including self.
+        This is used for hierarchical filtering - when a parent geography is selected,
+        all child geographies should also be included in the filter.
+
+        Returns:
+            List of geography names including self and all descendants
+        """
+        descendants = [self.name]
+        children = Geography.objects.filter(parent_id=self.id)
+
+        for child in children:
+            descendants.extend(child.get_all_descendant_names())  # type: ignore[attr-defined]
+
+        return descendants
+
+    @classmethod
+    def get_geography_names_with_descendants(
+        cls, geography_names: List[str]
+    ) -> List[str]:
+        """
+        Given a list of geography names, return all names including their descendants.
+        This is a helper method for filtering that expands parent geographies to include children.
+
+        Args:
+            geography_names: List of geography names to expand
+
+        Returns:
+            List of geography names including all descendants
+        """
+        all_names = set()
+
+        for name in geography_names:
+            try:
+                geography = cls.objects.get(name=name)
+                all_names.update(geography.get_all_descendant_names())
+            except cls.DoesNotExist:
+                # If geography doesn't exist, just add the name as-is
+                all_names.add(name)
+
+        return list(all_names)
+
     class Meta:
         db_table = "geography"
         verbose_name_plural = "geographies"

api/models/SDG.py

@@ -13,6 +13,9 @@ class SDG(models.Model):
     code = models.CharField(
         max_length=10, unique=True, null=False, blank=False
     )  # e.g., "SDG1", "SDG2"
+    number = models.IntegerField(
+        null=True, blank=True
+    )  # Numeric value for proper ordering (1, 2, 3... 17)
     description = models.TextField(null=True, blank=True)
     slug = models.SlugField(max_length=100, null=True, blank=False, unique=True)
 
@@ -28,4 +31,4 @@ class Meta:
         db_table = "sdg"
         verbose_name = "SDG"
         verbose_name_plural = "SDGs"
-        ordering = ["code"]
+        ordering = ["number"]

api/types/type_collaborative.py

@@ -187,7 +187,7 @@ def metadata(self) -> Optional[List["TypeCollaborativeMetadata"]]:
     def contributors(self) -> Optional[List["TypeUser"]]:
         """Get contributors associated with this collaborative."""
         try:
-            queryset = self.contributors.all()  # type: ignore
+            queryset = self.contributors.all().order_by("first_name")  # type: ignore
             if not queryset.exists():
                 return []
             return TypeUser.from_django_list(queryset)
@@ -215,10 +215,14 @@ def organization_relationships(
     def supporting_organizations(self) -> Optional[List["TypeOrganization"]]:
         """Get supporting organizations for this collaborative."""
         try:
-            relationships = CollaborativeOrganizationRelationship.objects.filter(
-                collaborative=self,  # type: ignore
-                relationship_type=OrganizationRelationshipType.SUPPORTER,
-            ).select_related("organization")
+            relationships = (
+                CollaborativeOrganizationRelationship.objects.filter(
+                    collaborative=self,  # type: ignore
+                    relationship_type=OrganizationRelationshipType.SUPPORTER,
+                )
+                .select_related("organization")
+                .order_by("organization__name")
+            )
 
             if not relationships.exists():
                 return []
@@ -232,10 +236,14 @@ def supporting_organizations(self) -> Optional[List["TypeOrganization"]]:
     def partner_organizations(self) -> Optional[List["TypeOrganization"]]:
         """Get partner organizations for this collaborative."""
         try:
-            relationships = CollaborativeOrganizationRelationship.objects.filter(
-                collaborative=self,  # type: ignore
-                relationship_type=OrganizationRelationshipType.PARTNER,
-            ).select_related("organization")
+            relationships = (
+                CollaborativeOrganizationRelationship.objects.filter(
+                    collaborative=self,  # type: ignore
+                    relationship_type=OrganizationRelationshipType.PARTNER,
+                )
+                .select_related("organization")
+                .order_by("organization__name")
+            )
 
             if not relationships.exists():
                 return []

api/types/type_sdg.py

@@ -21,6 +21,7 @@ class SDGFilter:
 class SDGOrder:
     """Order class for SDG model."""
 
+    number: auto
     code: auto
     name: auto
 

api/views/paginated_elastic_view.py

@@ -91,19 +91,21 @@ def get(self, request: HttpRequest) -> Response:
             serializer = self.serializer_class(response, many=True)
             aggregations: Dict[str, Any] = response.aggregations.to_dict()
 
-            metadata_aggregations = aggregations["metadata"]["filtered_metadata"][
-                "composite_agg"
-            ]["buckets"]
-            aggregations.pop("metadata")
-            if "catalogs" in aggregations:
-                aggregations.pop("catalogs")
-            for agg in metadata_aggregations:
-                label: str = agg["key"]["metadata_label"]
+            if "metadata" in aggregations:
+
+                metadata_aggregations = aggregations["metadata"]["filtered_metadata"][
+                    "composite_agg"
+                ]["buckets"]
+                aggregations.pop("metadata")
+                for agg in metadata_aggregations:
+                    label: str = agg["key"]["metadata_label"]
                 value: str = agg["key"].get("metadata_value", "")
                 if label not in aggregations:
                     aggregations[label] = {}
                 aggregations[label][value] = agg["doc_count"]
 
+            if "catalogs" in aggregations:
+                aggregations.pop("catalogs")
             # Handle sectors aggregation (now comes as "sectors.raw")
             if "sectors.raw" in aggregations:
                 sectors_agg = aggregations["sectors.raw"]["buckets"]
@@ -139,6 +141,20 @@ def get(self, request: HttpRequest) -> Response:
                 for agg in formats_agg:
                     aggregations["formats"][agg["key"]] = agg["doc_count"]
 
+            # Handle geographies aggregation (now comes as "geographies.raw")
+            if "geographies.raw" in aggregations:
+                geographies_agg = aggregations["geographies.raw"]["buckets"]
+                aggregations.pop("geographies.raw")
+                aggregations["geographies"] = {}
+                for agg in geographies_agg:
+                    aggregations["geographies"][agg["key"]] = agg["doc_count"]
+            elif "geographies" in aggregations:
+                geographies_agg = aggregations["geographies"]["buckets"]
+                aggregations.pop("geographies")
+                aggregations["geographies"] = {}
+                for agg in geographies_agg:
+                    aggregations["geographies"][agg["key"]] = agg["doc_count"]
+
             if "status" in aggregations:
                 status_agg = aggregations["status"]["buckets"]
                 aggregations.pop("status")

api/views/search_dataset.py

@@ -9,7 +9,7 @@
 from rest_framework import serializers
 from rest_framework.permissions import AllowAny
 
-from api.models import Dataset, DatasetMetadata, Metadata
+from api.models import Dataset, DatasetMetadata, Geography, Metadata
 from api.utils.telemetry_utils import trace_method, track_metrics
 from api.views.paginated_elastic_view import PaginatedElasticSearchAPIView
 from search.documents import DatasetDocument
@@ -260,6 +260,13 @@ def add_filters(self, filters: Dict[str, str], search: Search) -> Search:
                 raw_filter = filter + ".raw"
                 if raw_filter in self.aggregations:
                     filter_values = filters[filter].split(",")
+
+                    # For geographies, expand to include all descendant geographies
+                    if filter == "geographies":
+                        filter_values = Geography.get_geography_names_with_descendants(
+                            filter_values
+                        )
+
                     search = search.filter("terms", **{raw_filter: filter_values})
                 else:
                     search = search.filter("term", **{filter: filters[filter]})

api/views/search_usecase.py

@@ -9,7 +9,7 @@
 from rest_framework import serializers
 from rest_framework.permissions import AllowAny
 
-from api.models import Metadata, UseCase, UseCaseMetadata
+from api.models import Geography, Metadata, UseCase, UseCaseMetadata
 from api.utils.telemetry_utils import trace_method, track_metrics
 from api.views.paginated_elastic_view import PaginatedElasticSearchAPIView
 from search.documents import UseCaseDocument
@@ -311,6 +311,13 @@ def add_filters(self, filters: Dict[str, str], search: Search) -> Search:
                 raw_filter = filter + ".raw"
                 if raw_filter in self.aggregations:
                     filter_values = filters[filter].split(",")
+
+                    # For geographies, expand to include all descendant geographies
+                    if filter == "geographies":
+                        filter_values = Geography.get_geography_names_with_descendants(
+                            filter_values
+                        )
+
                     search = search.filter("terms", **{raw_filter: filter_values})
                 else:
                     search = search.filter("term", **{filter: filters[filter]})