Add commands to import Tabular datasets and items from a CSV file

vbos/datasets/admin.py

@@ -2,14 +2,15 @@
 import json
 from io import TextIOWrapper
 
-from django.contrib.gis import admin
 from django.contrib import messages
+from django.contrib.gis import admin
 from django.contrib.gis.geos.geometry import GEOSGeometry
-from django.shortcuts import render, redirect, reverse
+from django.shortcuts import redirect, render, reverse
 from django.urls import path
 
-from vbos.datasets.utils import CSVRow, GeoJSONProperties
+from vbos.datasets.utils import GeoJSONProperties, clean_redundant_tabular_items
 
+from .forms import CSVUploadForm, GeoJSONUploadForm
 from .models import (
     AreaCouncil,
     Cluster,
@@ -21,7 +22,7 @@
     VectorDataset,
     VectorItem,
 )
-from .forms import CSVUploadForm, GeoJSONUploadForm
+from .utils import CSVRow, create_tabular_item
 
 
 @admin.register(Cluster)
@@ -136,6 +137,21 @@ def import_file(self, request):
 class TabularDatasetAdmin(admin.ModelAdmin):
     list_display = ["id", "name", "cluster", "type", "updated"]
     list_filter = ["cluster", "type"]
+    actions = ["clean_redundant_items"]
+
+    @admin.action(description="Clean redundant TabularItems for dataset")
+    def clean_redundant_items(self, request, queryset):
+        for dataset in queryset:
+            clean_redundant_tabular_items(dataset)
+
+        dataset_names = list(queryset.values_list("name", flat=True))
+        if len(dataset_names) == 1:
+            message = f"Cleaned redundant values for: {dataset_names[0]}."
+        else:
+            # Join all but last with commas, then add "and" before last item
+            message = f"Cleaned redundant values for: {', '.join(dataset_names[:-1])} and {dataset_names[-1]}."
+
+        messages.success(request, message)
 
 
 @admin.register(TabularItem)
@@ -178,22 +194,10 @@ def import_file(self, request):
                     created_count = 0
                     error_count = 0
 
-                    for row in reader:  # start=2 to account for header row
+                    for row in reader:
                         try:
                             csv_row = CSVRow(row)
-                            TabularItem.objects.create(
-                                dataset=form.cleaned_data["dataset"],
-                                metadata=csv_row.metadata,
-                                attribute=csv_row.attribute.strip(),
-                                value=csv_row.value,
-                                date=csv_row.date,
-                                province=Province.objects.filter(
-                                    name__iexact=csv_row.province
-                                ).first(),
-                                area_council=AreaCouncil.objects.filter(
-                                    name__iexact=csv_row.area_council
-                                ).first(),
-                            )
+                            create_tabular_item(csv_row, form.cleaned_data["dataset"])
                             created_count += 1
                         except Exception as e:
                             print(e)

vbos/datasets/fixtures/master-import.csv

@@ -0,0 +1,56 @@
+Year,Month,Day,Type,Cluster,Indicator,Attribute,Hazard Type,Hazard Response & Recovery,Climate Change,Past Hazard Events,Blue Economy,Agriculture,International Trade,Measure,Country,National,Province,Area Council,Village,Unit,Source,Year Collected,Frequency Collection,Value
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,,,,number,Ministry of Education and Training ,2024,Annually,1154
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,,,,number,Ministry of Education and Training ,2024,Annually,  870
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,,,,number,Ministry of Education and Training ,2024,Annually,  272
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Education and Training ,2024,Annually,  66
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Education and Training ,2024,Annually,  54
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Education and Training ,2024,Annually,  20
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Education and Training ,2024,Annually,  240
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Education and Training ,2024,Annually,  182
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Education and Training ,2024,Annually,  38
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Education and Training ,2024,Annually,  196
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Education and Training ,2024,Annually,  124
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Education and Training ,2024,Annually,  34
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Education and Training ,2024,Annually,  236
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Education and Training ,2024,Annually,  172
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Education and Training ,2024,Annually,  50
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Education and Training ,2024,Annually,  190
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Education and Training ,2024,Annually,  188
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Education and Training ,2024,Annually,  82
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Education and Training ,2024,Annually,  226
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Education and Training ,2024,Annually,  150
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Education and Training ,2024,Annually,  48
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,Torres,,number,Ministry of Education and Training ,2024,Annually,  6
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,Torres,,number,Ministry of Education and Training ,2024,Annually,  6
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,Torres,,number,Ministry of Education and Training ,2024,Annually,  2
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Ministry of Education and Training ,2024,Annually,  4
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Ministry of Education and Training ,2024,Annually,  4
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Ministry of Education and Training ,2024,Annually,  2
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Ministry of Education and Training ,2024,Annually,  8
+2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Ministry of Education and Training ,2024,Annually,  6
+2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Ministry of Education and Training ,2024,Annually,  2
+2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,East Vanualava,,number,Ministry of Education and Training ,2024,Annually,  8
+2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,6
+2024,,,Baseline,Health,Health Facility,health centre,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,4
+2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,47
+2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,169
+2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Health,2024,Annually,1
+2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Health,2024,Annually,1
+2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Health,2024,Annually,20
+2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Health,2024,Annually,1
+2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Health,2024,Annually,7
+2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Health,2024,Annually,33
+2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,1
+2024,,,Baseline,Health,Health Facility,health centre,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,2
+2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,6
+2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,30
+2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Health,2024,Annually,1
+2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Health,2024,Annually,12
+2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Health,2024,Annually,26
+2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,1
+2024,,,Baseline,Health,Health Facility,health centre,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,2
+2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,9
+2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,25
+2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,1
+2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,12
+2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,35

vbos/datasets/forms.py

@@ -1,11 +1,13 @@
 from django import forms
+
 from .models import TabularDataset, VectorDataset
 
 
 class CSVUploadForm(forms.Form):
     file = forms.FileField(label="File")
     dataset = forms.ModelChoiceField(
-        queryset=TabularDataset.objects.all(), empty_label="Select a dataset"
+        queryset=TabularDataset.objects.all(),
+        empty_label="Select a dataset",
     )
 
 

vbos/datasets/management/commands/clean_tabular_data.py

@@ -0,0 +1,16 @@
+from django.core.management.base import BaseCommand
+
+from ...models import TabularDataset
+from ...utils import clean_redundant_tabular_items
+
+
+class Command(BaseCommand):
+    help = """Clean TabularItem data, removing entries that are redundant."""
+
+    def handle(self, *args, **options):
+        datasets = TabularDataset.objects.all()
+
+        for d in datasets:
+            clean_redundant_tabular_items(d)
+
+            self.stdout.write(f"Removed redundant {d.name} tabular items.")

vbos/datasets/management/commands/import_datasets.py

@@ -0,0 +1,51 @@
+import csv
+
+from django.core.management.base import BaseCommand
+
+from ...models import Cluster, TabularDataset
+from ...utils import REVERSE_TYPE_MAPPING
+
+
+class Command(BaseCommand):
+    help = """Read a local CSV file and import datasets from it."""
+
+    def add_arguments(self, parser):
+        parser.add_argument("filename", nargs=1, type=str)
+
+    def handle(self, *args, **options):
+        filename = options["filename"][0]
+
+        with open(filename) as file:
+            reader = csv.DictReader(file)
+            created_count = 0
+
+            datasets = [
+                (i.name, i.cluster.name, i.type) for i in TabularDataset.objects.all()
+            ]
+
+            for row in reader:
+                type = REVERSE_TYPE_MAPPING[row["Type"]]
+                if (
+                    row["Indicator"],
+                    row["Cluster"],
+                    type,
+                ) not in datasets:
+                    TabularDataset.objects.create(
+                        name=row["Indicator"].strip(),
+                        cluster=Cluster.objects.get_or_create(
+                            name=row["Cluster"].strip()
+                        )[0],
+                        type=type,
+                        unit=row["Unit"].strip(),
+                        source=row["Source"].strip(),
+                    )
+                    datasets.append(
+                        (
+                            row["Indicator"],
+                            row["Cluster"],
+                            type,
+                        )
+                    )
+                    created_count += 1
+
+            self.stdout.write(f"{created_count} datasets created from {filename}.")

vbos/datasets/management/commands/import_tabular_data.py

@@ -0,0 +1,33 @@
+import csv
+
+from django.core.management.base import BaseCommand
+
+from ...utils import CSVRow, create_tabular_item, get_dataset
+
+
+class Command(BaseCommand):
+    help = """Read a local CSV file and import TabularItems from it."""
+
+    def add_arguments(self, parser):
+        parser.add_argument("filename", nargs=1, type=str)
+
+    def handle(self, *args, **options):
+        filename = options["filename"][0]
+
+        with open(filename) as file:
+            reader = csv.DictReader(file)
+            created_count = 0
+            error_count = 0
+
+            for row in reader:
+                try:
+                    dataset = get_dataset(row)
+                    csv_row = CSVRow(row)
+                    create_tabular_item(csv_row, dataset)
+
+                    created_count += 1
+                except Exception as e:
+                    print(e)
+                    error_count += 1
+
+            self.stdout.write(f"{created_count} tabular items created from {filename}.")

vbos/datasets/migrations/0017_auto_20251114_0018.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.2.5 on 2025-11-14 00:18
+
+from django.db import migrations
+
+
+def update_area_councils_names(apps, schema_editor):
+    AreaCouncil = apps.get_model("datasets", "AreaCouncil")
+    updates = [
+        ["Bigbay Coast", "Big Bay Coast"],
+        ["Bigbay Inland", "Big Bay Inland"],
+        ["East Vanua Lava", "East Vanualava"],
+        ["West Vanua Lava", "West Vanualava"],
+        ["West Vanua Lava", "West Vanualava"],
+        ["Canal - Fanafo", "Canal Fanafo"],
+    ]
+    for entry in updates:
+        try:
+            e = AreaCouncil.objects.get(name=entry[0])
+            e.name = entry[1]
+            e.save()
+        except AreaCouncil.DoesNotExist:
+            print("AreaCouncil: {entry[0]} not found")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("datasets", "0016_alter_rasterdataset_unique_together_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            update_area_councils_names, reverse_code=migrations.RunPython.noop
+        ),
+    ]

vbos/datasets/test/test_management_commands.py

@@ -0,0 +1,56 @@
+from io import StringIO
+
+from django.core.management import call_command
+from django.test import TestCase
+
+from ..models import TabularDataset, TabularItem
+
+
+class TestImportDatasets(TestCase):
+    def setUp(self):
+        self.filename = "./vbos/datasets/fixtures/master-import.csv"
+        self.out = StringIO()
+        call_command("import_datasets", self.filename, stdout=self.out)
+        call_command("import_tabular_data", self.filename, stdout=self.out)
+
+    def test_import(self):
+        self.assertEqual(TabularDataset.objects.count(), 2)
+        self.assertIn(
+            "2 datasets created from {}.".format(self.filename), self.out.getvalue()
+        )
+        self.assertEqual(
+            TabularDataset.objects.get(name="Number Schools").unit, "number"
+        )
+        self.assertEqual(
+            TabularDataset.objects.get(name="Health Facility").unit, "number"
+        )
+        self.assertEqual(
+            TabularDataset.objects.get(name="Number Schools").source,
+            "Ministry of Education and Training",
+        )
+        self.assertEqual(
+            TabularDataset.objects.get(name="Health Facility").source,
+            "Ministry of Health",
+        )
+        # Test TabularItem import
+        self.assertEqual(TabularItem.objects.count(), 55)
+        self.assertEqual(
+            TabularItem.objects.filter(dataset__name="Number Schools").count(), 31
+        )
+        self.assertEqual(
+            TabularItem.objects.filter(dataset__name="Health Facility").count(), 24
+        )
+        self.assertIn(
+            "55 tabular items created",
+            self.out.getvalue(),
+        )
+
+        # Clean redundant entries
+        call_command("clean_tabular_data", stdout=self.out)
+        self.assertEqual(TabularItem.objects.count(), 30)
+        self.assertEqual(
+            TabularItem.objects.filter(dataset__name="Number Schools").count(), 10
+        )
+        self.assertEqual(
+            TabularItem.objects.filter(dataset__name="Health Facility").count(), 20
+        )