Skip to content

Commit 7041c7f

Browse files
willemarcelwillemarcel
authored
Merge pull request #44 from developmentseed/feat/import-commands
Add commands to import Tabular datasets and items from a CSV file
2 parents d520ce2 + 066b76e commit 7041c7f

File tree

13 files changed

+429
-26
lines changed

13 files changed

+429
-26
lines changed

‎vbos/datasets/admin.py‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,15 @@
22
import json
33
from io import TextIOWrapper
44

5-
from django.contrib.gis import admin
65
from django.contrib import messages
6+
from django.contrib.gis import admin
77
from django.contrib.gis.geos.geometry import GEOSGeometry
8-
from django.shortcuts import render, redirect, reverse
8+
from django.shortcuts import redirect, render, reverse
99
from django.urls import path
1010

11-
from vbos.datasets.utils import CSVRow, GeoJSONProperties
11+
from vbos.datasets.utils import GeoJSONProperties, clean_redundant_tabular_items
1212

13+
from .forms import CSVUploadForm, GeoJSONUploadForm
1314
from .models import (
1415
AreaCouncil,
1516
Cluster,
@@ -21,7 +22,7 @@
2122
VectorDataset,
2223
VectorItem,
2324
)
24-
from .forms import CSVUploadForm, GeoJSONUploadForm
25+
from .utils import CSVRow, create_tabular_item
2526

2627

2728
@admin.register(Cluster)
@@ -136,6 +137,21 @@ def import_file(self, request):
136137
class TabularDatasetAdmin(admin.ModelAdmin):
137138
list_display = ["id", "name", "cluster", "type", "updated"]
138139
list_filter = ["cluster", "type"]
140+
actions = ["clean_redundant_items"]
141+
142+
@admin.action(description="Clean redundant TabularItems for dataset")
143+
def clean_redundant_items(self, request, queryset):
144+
for dataset in queryset:
145+
clean_redundant_tabular_items(dataset)
146+
147+
dataset_names = list(queryset.values_list("name", flat=True))
148+
if len(dataset_names) == 1:
149+
message = f"Cleaned redundant values for: {dataset_names[0]}."
150+
else:
151+
# Join all but last with commas, then add "and" before last item
152+
message = f"Cleaned redundant values for: {', '.join(dataset_names[:-1])} and {dataset_names[-1]}."
153+
154+
messages.success(request, message)
139155

140156

141157
@admin.register(TabularItem)
@@ -178,22 +194,10 @@ def import_file(self, request):
178194
created_count = 0
179195
error_count = 0
180196

181-
for row in reader: # start=2 to account for header row
197+
for row in reader:
182198
try:
183199
csv_row = CSVRow(row)
184-
TabularItem.objects.create(
185-
dataset=form.cleaned_data["dataset"],
186-
metadata=csv_row.metadata,
187-
attribute=csv_row.attribute.strip(),
188-
value=csv_row.value,
189-
date=csv_row.date,
190-
province=Province.objects.filter(
191-
name__iexact=csv_row.province
192-
).first(),
193-
area_council=AreaCouncil.objects.filter(
194-
name__iexact=csv_row.area_council
195-
).first(),
196-
)
200+
create_tabular_item(csv_row, form.cleaned_data["dataset"])
197201
created_count += 1
198202
except Exception as e:
199203
print(e)

‎vbos/datasets/fixtures/area-councils.geojson‎

Lines changed: 5 additions & 5 deletions
Large diffs are not rendered by default.

‎vbos/datasets/fixtures/master-import.csv‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
Year,Month,Day,Type,Cluster,Indicator,Attribute,Hazard Type,Hazard Response & Recovery,Climate Change,Past Hazard Events,Blue Economy,Agriculture,International Trade,Measure,Country,National,Province,Area Council,Village,Unit,Source,Year Collected,Frequency Collection,Value
2+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,,,,number,Ministry of Education and Training ,2024,Annually,1154
3+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,,,,number,Ministry of Education and Training ,2024,Annually, 870
4+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,,,,number,Ministry of Education and Training ,2024,Annually, 272
5+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Education and Training ,2024,Annually, 66
6+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Education and Training ,2024,Annually, 54
7+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Education and Training ,2024,Annually, 20
8+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Education and Training ,2024,Annually, 240
9+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Education and Training ,2024,Annually, 182
10+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Education and Training ,2024,Annually, 38
11+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Education and Training ,2024,Annually, 196
12+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Education and Training ,2024,Annually, 124
13+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Education and Training ,2024,Annually, 34
14+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Education and Training ,2024,Annually, 236
15+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Education and Training ,2024,Annually, 172
16+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Education and Training ,2024,Annually, 50
17+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Education and Training ,2024,Annually, 190
18+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Education and Training ,2024,Annually, 188
19+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Education and Training ,2024,Annually, 82
20+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Education and Training ,2024,Annually, 226
21+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Education and Training ,2024,Annually, 150
22+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Education and Training ,2024,Annually, 48
23+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,Torres,,number,Ministry of Education and Training ,2024,Annually, 6
24+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,Torres,,number,Ministry of Education and Training ,2024,Annually, 6
25+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,Torres,,number,Ministry of Education and Training ,2024,Annually, 2
26+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Ministry of Education and Training ,2024,Annually, 4
27+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Ministry of Education and Training ,2024,Annually, 4
28+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Ministry of Education and Training ,2024,Annually, 2
29+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Ministry of Education and Training ,2024,Annually, 8
30+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Ministry of Education and Training ,2024,Annually, 6
31+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Ministry of Education and Training ,2024,Annually, 2
32+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,East Vanualava,,number,Ministry of Education and Training ,2024,Annually, 8
33+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,6
34+
2024,,,Baseline,Health,Health Facility,health centre,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,4
35+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,47
36+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,169
37+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Health,2024,Annually,1
38+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Health,2024,Annually,1
39+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Health,2024,Annually,20
40+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Health,2024,Annually,1
41+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Health,2024,Annually,7
42+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Health,2024,Annually,33
43+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,1
44+
2024,,,Baseline,Health,Health Facility,health centre,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,2
45+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,6
46+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,30
47+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Health,2024,Annually,1
48+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Health,2024,Annually,12
49+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Health,2024,Annually,26
50+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,1
51+
2024,,,Baseline,Health,Health Facility,health centre,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,2
52+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,9
53+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,25
54+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,1
55+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,12
56+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,35

‎vbos/datasets/forms.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
from django import forms
2+
23
from .models import TabularDataset, VectorDataset
34

45

56
class CSVUploadForm(forms.Form):
67
file = forms.FileField(label="File")
78
dataset = forms.ModelChoiceField(
8-
queryset=TabularDataset.objects.all(), empty_label="Select a dataset"
9+
queryset=TabularDataset.objects.all(),
10+
empty_label="Select a dataset",
911
)
1012

1113

‎vbos/datasets/management/__init__.py‎

Whitespace-only changes.

‎vbos/datasets/management/commands/__init__.py‎

Whitespace-only changes.

‎vbos/datasets/management/commands/clean_tabular_data.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from django.core.management.base import BaseCommand
2+
3+
from ...models import TabularDataset
4+
from ...utils import clean_redundant_tabular_items
5+
6+
7+
class Command(BaseCommand):
8+
help = """Clean TabularItem data, removing entries that are redundant."""
9+
10+
def handle(self, *args, **options):
11+
datasets = TabularDataset.objects.all()
12+
13+
for d in datasets:
14+
clean_redundant_tabular_items(d)
15+
16+
self.stdout.write(f"Removed redundant {d.name} tabular items.")

‎vbos/datasets/management/commands/import_datasets.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import csv
2+
3+
from django.core.management.base import BaseCommand
4+
5+
from ...models import Cluster, TabularDataset
6+
from ...utils import REVERSE_TYPE_MAPPING
7+
8+
9+
class Command(BaseCommand):
10+
help = """Read a local CSV file and import datasets from it."""
11+
12+
def add_arguments(self, parser):
13+
parser.add_argument("filename", nargs=1, type=str)
14+
15+
def handle(self, *args, **options):
16+
filename = options["filename"][0]
17+
18+
with open(filename) as file:
19+
reader = csv.DictReader(file)
20+
created_count = 0
21+
22+
datasets = [
23+
(i.name, i.cluster.name, i.type) for i in TabularDataset.objects.all()
24+
]
25+
26+
for row in reader:
27+
type = REVERSE_TYPE_MAPPING[row["Type"]]
28+
if (
29+
row["Indicator"],
30+
row["Cluster"],
31+
type,
32+
) not in datasets:
33+
TabularDataset.objects.create(
34+
name=row["Indicator"].strip(),
35+
cluster=Cluster.objects.get_or_create(
36+
name=row["Cluster"].strip()
37+
)[0],
38+
type=type,
39+
unit=row["Unit"].strip(),
40+
source=row["Source"].strip(),
41+
)
42+
datasets.append(
43+
(
44+
row["Indicator"],
45+
row["Cluster"],
46+
type,
47+
)
48+
)
49+
created_count += 1
50+
51+
self.stdout.write(f"{created_count} datasets created from {filename}.")

‎vbos/datasets/management/commands/import_tabular_data.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import csv
2+
3+
from django.core.management.base import BaseCommand
4+
5+
from ...utils import CSVRow, create_tabular_item, get_dataset
6+
7+
8+
class Command(BaseCommand):
9+
help = """Read a local CSV file and import TabularItems from it."""
10+
11+
def add_arguments(self, parser):
12+
parser.add_argument("filename", nargs=1, type=str)
13+
14+
def handle(self, *args, **options):
15+
filename = options["filename"][0]
16+
17+
with open(filename) as file:
18+
reader = csv.DictReader(file)
19+
created_count = 0
20+
error_count = 0
21+
22+
for row in reader:
23+
try:
24+
dataset = get_dataset(row)
25+
csv_row = CSVRow(row)
26+
create_tabular_item(csv_row, dataset)
27+
28+
created_count += 1
29+
except Exception as e:
30+
print(e)
31+
error_count += 1
32+
33+
self.stdout.write(f"{created_count} tabular items created from {filename}.")

‎vbos/datasets/migrations/0017_auto_20251114_0018.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Generated by Django 5.2.5 on 2025-11-14 00:18
2+
3+
from django.db import migrations
4+
5+
6+
def update_area_councils_names(apps, schema_editor):
7+
AreaCouncil = apps.get_model("datasets", "AreaCouncil")
8+
updates = [
9+
["Bigbay Coast", "Big Bay Coast"],
10+
["Bigbay Inland", "Big Bay Inland"],
11+
["East Vanua Lava", "East Vanualava"],
12+
["West Vanua Lava", "West Vanualava"],
13+
["West Vanua Lava", "West Vanualava"],
14+
["Canal - Fanafo", "Canal Fanafo"],
15+
]
16+
for entry in updates:
17+
try:
18+
e = AreaCouncil.objects.get(name=entry[0])
19+
e.name = entry[1]
20+
e.save()
21+
except AreaCouncil.DoesNotExist:
22+
print("AreaCouncil: {entry[0]} not found")
23+
24+
25+
class Migration(migrations.Migration):
26+
dependencies = [
27+
("datasets", "0016_alter_rasterdataset_unique_together_and_more"),
28+
]
29+
30+
operations = [
31+
migrations.RunPython(
32+
update_area_councils_names, reverse_code=migrations.RunPython.noop
33+
),
34+
]

0 commit comments

Comments
 (0)