Add clean redundant tabular items function + add migration to fix AC

names

Author: willemarcel

vbos/datasets/admin.py

@@ -8,7 +8,7 @@
 from django.shortcuts import redirect, render, reverse
 from django.urls import path
 
-from vbos.datasets.utils import GeoJSONProperties
+from vbos.datasets.utils import GeoJSONProperties, clean_redundant_tabular_items
 
 from .forms import CSVUploadForm, GeoJSONUploadForm
 from .models import (
@@ -137,6 +137,21 @@ def import_file(self, request):
 class TabularDatasetAdmin(admin.ModelAdmin):
     list_display = ["id", "name", "cluster", "type", "updated"]
     list_filter = ["cluster", "type"]
+    actions = ["clean_redundant_items"]
+
+    @admin.action(description="Clean redundant TabularItems for dataset")
+    def clean_redundant_items(self, request, queryset):
+        for dataset in queryset:
+            clean_redundant_tabular_items(dataset)
+
+        dataset_names = list(queryset.values_list("name", flat=True))
+        if len(dataset_names) == 1:
+            message = f"Cleaned redundant values for: {dataset_names[0]}."
+        else:
+            # Join all but last with commas, then add "and" before last item
+            message = f"Cleaned redundant values for: {', '.join(dataset_names[:-1])} and {dataset_names[-1]}."
+
+        messages.success(request, message)
 
 
 @admin.register(TabularItem)

vbos/datasets/fixtures/area-councils.geojson

@@ -54,11 +54,3 @@ Year,Month,Day,Type,Cluster,Indicator,Attribute,Hazard Type,Hazard Response & Re
 2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,1
 2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,12
 2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,35
-2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Torba,East Vanualava,,number,Ministry of Health,2024,Annually,1
-2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Torba,West Vanualava,,number,Ministry of Health,2024,Annually,1
-2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,East Gaua,,number,Ministry of Health,2024,Annually,4
-2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,East Vanualava,,number,Ministry of Health,2024,Annually,3
-2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,Merelava,,number,Ministry of Health,2024,Annually,1
-2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Ministry of Health,2024,Annually,2
-2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,Torres,,number,Ministry of Health,2024,Annually,2
-2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Ministry of Health,2024,Annually,1

vbos/datasets/fixtures/master-import.csv

@@ -0,0 +1,16 @@
+from django.core.management.base import BaseCommand
+
+from ...models import TabularDataset
+from ...utils import clean_redundant_tabular_items
+
+
+class Command(BaseCommand):
+    help = """Clean TabularItem data, removing entries that are redundant."""
+
+    def handle(self, *args, **options):
+        datasets = TabularDataset.objects.all()
+
+        for d in datasets:
+            clean_redundant_tabular_items(d)
+
+            self.stdout.write(f"Removed redundant {d.name} tabular items.")

vbos/datasets/management/commands/clean_tabular_data.py

@@ -2,9 +2,8 @@
 
 from django.core.management.base import BaseCommand
 
-from ...models import TYPE_CHOICES, Cluster, TabularDataset
-
-REVERSE_TYPE_MAPPING = {str(label): value for (value, label) in TYPE_CHOICES.items()}
+from ...models import Cluster, TabularDataset
+from ...utils import REVERSE_TYPE_MAPPING
 
 
 class Command(BaseCommand):

vbos/datasets/management/commands/import_datasets.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.2.5 on 2025-11-14 00:18
+
+from django.db import migrations
+
+
+def update_area_councils_names(apps, schema_editor):
+    AreaCouncil = apps.get_model("datasets", "AreaCouncil")
+    updates = [
+        ["Bigbay Coast", "Big Bay Coast"],
+        ["Bigbay Inland", "Big Bay Inland"],
+        ["East Vanua Lava", "East Vanualava"],
+        ["West Vanua Lava", "West Vanualava"],
+        ["West Vanua Lava", "West Vanualava"],
+        ["Canal - Fanafo", "Canal Fanafo"],
+    ]
+    for entry in updates:
+        try:
+            e = AreaCouncil.objects.get(name=entry[0])
+            e.name = entry[1]
+            e.save()
+        except AreaCouncil.DoesNotExist:
+            print("AreaCouncil: {entry[0]} not found")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("datasets", "0016_alter_rasterdataset_unique_together_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            update_area_councils_names, reverse_code=migrations.RunPython.noop
+        ),
+    ]

vbos/datasets/migrations/0017_auto_20251114_0018.py

@@ -32,14 +32,25 @@ def test_import(self):
             TabularDataset.objects.get(name="Health Facility").source,
             "Ministry of Health",
         )
-        self.assertEqual(TabularItem.objects.count(), 63)
+        # Test TabularItem import
+        self.assertEqual(TabularItem.objects.count(), 55)
         self.assertEqual(
             TabularItem.objects.filter(dataset__name="Number Schools").count(), 31
         )
         self.assertEqual(
-            TabularItem.objects.filter(dataset__name="Health Facility").count(), 32
+            TabularItem.objects.filter(dataset__name="Health Facility").count(), 24
         )
         self.assertIn(
-            "63 tabular items created",
+            "55 tabular items created",
             self.out.getvalue(),
         )
+
+        # Clean redundant entries
+        call_command("clean_tabular_data", stdout=self.out)
+        self.assertEqual(TabularItem.objects.count(), 30)
+        self.assertEqual(
+            TabularItem.objects.filter(dataset__name="Number Schools").count(), 10
+        )
+        self.assertEqual(
+            TabularItem.objects.filter(dataset__name="Health Facility").count(), 20
+        )

vbos/datasets/test/test_management_commands.py

@@ -137,3 +137,16 @@ def create_tabular_item(csv_row: CSVRow, dataset: TabularDataset):
             name__iexact=csv_row.area_council
         ).first(),
     )
+
+
+def clean_redundant_tabular_items(dataset: TabularDataset):
+    items = TabularItem.objects.filter(dataset=dataset)
+    # If a dataset has items with an Area Council value,
+    # remove all items that don't have the Area Council set
+    if items.filter(area_council__isnull=False).count() > 0:
+        items.filter(area_council__isnull=True).delete()
+
+    # If a dataset has items with a Province value,
+    # remove all items that don't have the Province set
+    if items.filter(province__isnull=False).count() > 0:
+        items.filter(province__isnull=True).delete()