Skip to content

Commit 2ba4f6d

Browse files
willemarcelwillemarcel
committed
Add commands to import Tabular datasets and items from a CSV file
1 parent 8538217 commit 2ba4f6d

File tree

10 files changed

+322
-21
lines changed

10 files changed

+322
-21
lines changed

‎vbos/datasets/admin.py‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,15 @@
22
import json
33
from io import TextIOWrapper
44

5-
from django.contrib.gis import admin
65
from django.contrib import messages
6+
from django.contrib.gis import admin
77
from django.contrib.gis.geos.geometry import GEOSGeometry
8-
from django.shortcuts import render, redirect, reverse
8+
from django.shortcuts import redirect, render, reverse
99
from django.urls import path
1010

11-
from vbos.datasets.utils import CSVRow, GeoJSONProperties
11+
from vbos.datasets.utils import GeoJSONProperties
1212

13+
from .forms import CSVUploadForm, GeoJSONUploadForm
1314
from .models import (
1415
AreaCouncil,
1516
Cluster,
@@ -21,7 +22,7 @@
2122
VectorDataset,
2223
VectorItem,
2324
)
24-
from .forms import CSVUploadForm, GeoJSONUploadForm
25+
from .utils import CSVRow, create_tabular_item
2526

2627

2728
@admin.register(Cluster)
@@ -178,22 +179,10 @@ def import_file(self, request):
178179
created_count = 0
179180
error_count = 0
180181

181-
for row in reader: # start=2 to account for header row
182+
for row in reader:
182183
try:
183184
csv_row = CSVRow(row)
184-
TabularItem.objects.create(
185-
dataset=form.cleaned_data["dataset"],
186-
metadata=csv_row.metadata,
187-
attribute=csv_row.attribute.strip(),
188-
value=csv_row.value,
189-
date=csv_row.date,
190-
province=Province.objects.filter(
191-
name__iexact=csv_row.province
192-
).first(),
193-
area_council=AreaCouncil.objects.filter(
194-
name__iexact=csv_row.area_council
195-
).first(),
196-
)
185+
create_tabular_item(csv_row, form.cleaned_data["dataset"])
197186
created_count += 1
198187
except Exception as e:
199188
print(e)

‎vbos/datasets/fixtures/master-import.csv‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
Year,Month,Day,Type,Cluster,Indicator,Attribute,Hazard Type,Hazard Response & Recovery,Climate Change,Past Hazard Events,Blue Economy,Agriculture,International Trade,Measure,Country,National,Province,Area Council,Village,Unit,Source,Year Collected,Frequency Collection,Value
2+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,,,,number,Minsitry of Education and Training ,2024,Annually,1154
3+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,,,,number,Minsitry of Education and Training ,2024,Annually, 870
4+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,,,,number,Minsitry of Education and Training ,2024,Annually, 272
5+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,,,number,Minsitry of Education and Training ,2024,Annually, 66
6+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,,,number,Minsitry of Education and Training ,2024,Annually, 54
7+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,,,number,Minsitry of Education and Training ,2024,Annually, 20
8+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Sanma,,,number,Minsitry of Education and Training ,2024,Annually, 240
9+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Sanma,,,number,Minsitry of Education and Training ,2024,Annually, 182
10+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Sanma,,,number,Minsitry of Education and Training ,2024,Annually, 38
11+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Penama,,,number,Minsitry of Education and Training ,2024,Annually, 196
12+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Penama,,,number,Minsitry of Education and Training ,2024,Annually, 124
13+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Penama,,,number,Minsitry of Education and Training ,2024,Annually, 34
14+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Malampa,,,number,Minsitry of Education and Training ,2024,Annually, 236
15+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Malampa,,,number,Minsitry of Education and Training ,2024,Annually, 172
16+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Malampa,,,number,Minsitry of Education and Training ,2024,Annually, 50
17+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Shefa,,,number,Minsitry of Education and Training ,2024,Annually, 190
18+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Shefa,,,number,Minsitry of Education and Training ,2024,Annually, 188
19+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Shefa,,,number,Minsitry of Education and Training ,2024,Annually, 82
20+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Tafea,,,number,Minsitry of Education and Training ,2024,Annually, 226
21+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Tafea,,,number,Minsitry of Education and Training ,2024,Annually, 150
22+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Tafea,,,number,Minsitry of Education and Training ,2024,Annually, 48
23+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,Torres,,number,Minsitry of Education and Training ,2024,Annually, 6
24+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,Torres,,number,Minsitry of Education and Training ,2024,Annually, 6
25+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,Torres,,number,Minsitry of Education and Training ,2024,Annually, 2
26+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Minsitry of Education and Training ,2024,Annually, 4
27+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Minsitry of Education and Training ,2024,Annually, 4
28+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Minsitry of Education and Training ,2024,Annually, 2
29+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Minsitry of Education and Training ,2024,Annually, 8
30+
2024,,,Baseline,Education,Number Schools,primary ,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Minsitry of Education and Training ,2024,Annually, 6
31+
2024,,,Baseline,Education,Number Schools,secondary,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Minsitry of Education and Training ,2024,Annually, 2
32+
2024,,,Baseline,Education,Number Schools,ecce,,,,,,,,,,Vanuatu,Torba,East Vanualava,,number,Minsitry of Education and Training ,2024,Annually, 8
33+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,6
34+
2024,,,Baseline,Health,Health Facility,health centre,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,4
35+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,47
36+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,,,,number,Ministry of Health,2024,Annually,169
37+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Health,2024,Annually,1
38+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Health,2024,Annually,1
39+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,,,number,Ministry of Health,2024,Annually,20
40+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Health,2024,Annually,1
41+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Health,2024,Annually,7
42+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Sanma,,,number,Ministry of Health,2024,Annually,33
43+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,1
44+
2024,,,Baseline,Health,Health Facility,health centre,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,2
45+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,6
46+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Penama,,,number,Ministry of Health,2024,Annually,30
47+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Health,2024,Annually,1
48+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Health,2024,Annually,12
49+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Malampa,,,number,Ministry of Health,2024,Annually,26
50+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,1
51+
2024,,,Baseline,Health,Health Facility,health centre,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,2
52+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,9
53+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Shefa,,,number,Ministry of Health,2024,Annually,25
54+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,1
55+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,12
56+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Tafea,,,number,Ministry of Health,2024,Annually,35
57+
2024,,,Baseline,Health,Health Facility,hospital,,,,,,,,,,Vanuatu,Torba,East Vanualava,,number,Ministry of Health,2024,Annually,1
58+
2024,,,Baseline,Health,Health Facility,dispensary,,,,,,,,,,Vanuatu,Torba,West Vanualava,,number,Ministry of Health,2024,Annually,1
59+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,East Gaua,,number,Ministry of Health,2024,Annually,4
60+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,East Vanualava,,number,Ministry of Health,2024,Annually,3
61+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,Merelava,,number,Ministry of Health,2024,Annually,1
62+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,Motalava,,number,Ministry of Health,2024,Annually,2
63+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,Torres,,number,Ministry of Health,2024,Annually,2
64+
2024,,,Baseline,Health,Health Facility,aidpost,,,,,,,,,,Vanuatu,Torba,Ureparapara,,number,Ministry of Health,2024,Annually,1

‎vbos/datasets/forms.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
from django import forms
2+
23
from .models import TabularDataset, VectorDataset
34

45

56
class CSVUploadForm(forms.Form):
67
file = forms.FileField(label="File")
78
dataset = forms.ModelChoiceField(
8-
queryset=TabularDataset.objects.all(), empty_label="Select a dataset"
9+
queryset=TabularDataset.objects.all(),
10+
empty_label="Select a dataset",
911
)
1012

1113

‎vbos/datasets/management/__init__.py‎

Whitespace-only changes.

‎vbos/datasets/management/commands/__init__.py‎

Whitespace-only changes.

‎vbos/datasets/management/commands/import_datasets.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import csv
2+
3+
from django.core.management.base import BaseCommand
4+
5+
from ...models import TYPE_CHOICES, Cluster, TabularDataset
6+
7+
REVERSE_TYPE_MAPPING = {str(label): value for (value, label) in TYPE_CHOICES.items()}
8+
9+
10+
class Command(BaseCommand):
11+
help = """Read a local CSV file and import datasets from it."""
12+
13+
def add_arguments(self, parser):
14+
parser.add_argument("filename", nargs=1, type=str)
15+
16+
def handle(self, *args, **options):
17+
filename = options["filename"][0]
18+
19+
with open(filename) as file:
20+
reader = csv.DictReader(file)
21+
created_count = 0
22+
23+
datasets = [
24+
(i.name, i.cluster.name, i.type) for i in TabularDataset.objects.all()
25+
]
26+
27+
for row in reader:
28+
type = REVERSE_TYPE_MAPPING[row["Type"]]
29+
if (
30+
row["Indicator"],
31+
row["Cluster"],
32+
type,
33+
) not in datasets:
34+
TabularDataset.objects.create(
35+
name=row["Indicator"],
36+
cluster=Cluster.objects.get_or_create(name=row["Cluster"])[0],
37+
type=type,
38+
unit=row["Unit"],
39+
source=row["Source"],
40+
)
41+
datasets.append(
42+
(
43+
row["Indicator"],
44+
row["Cluster"],
45+
type,
46+
)
47+
)
48+
created_count += 1
49+
50+
self.stdout.write(f"{created_count} datasets created from {filename}.")

‎vbos/datasets/management/commands/import_tabular_data.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import csv
2+
3+
from django.core.management.base import BaseCommand
4+
5+
from ...utils import CSVRow, create_tabular_item, get_dataset
6+
7+
8+
class Command(BaseCommand):
9+
help = """Read a local CSV file and import TabularItems from it."""
10+
11+
def add_arguments(self, parser):
12+
parser.add_argument("filename", nargs=1, type=str)
13+
14+
def handle(self, *args, **options):
15+
filename = options["filename"][0]
16+
17+
with open(filename) as file:
18+
reader = csv.DictReader(file)
19+
created_count = 0
20+
error_count = 0
21+
22+
for row in reader:
23+
try:
24+
dataset = get_dataset(row)
25+
csv_row = CSVRow(row)
26+
create_tabular_item(csv_row, dataset)
27+
28+
created_count += 1
29+
except Exception as e:
30+
print(e)
31+
error_count += 1
32+
33+
self.stdout.write(f"{created_count} tabular items created from {filename}.")

‎vbos/datasets/test/test_management_commands.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
from io import StringIO
2+
3+
from django.core.management import call_command
4+
from django.test import TestCase
5+
6+
from ..models import TabularDataset, TabularItem
7+
8+
9+
class TestImportDatasets(TestCase):
10+
def setUp(self):
11+
self.filename = "./vbos/datasets/fixtures/master-import.csv"
12+
self.out = StringIO()
13+
call_command("import_datasets", self.filename, stdout=self.out)
14+
call_command("import_tabular_data", self.filename, stdout=self.out)
15+
16+
def test_import(self):
17+
self.assertEqual(TabularDataset.objects.count(), 2)
18+
self.assertIn(
19+
"2 datasets created from {}.".format(self.filename), self.out.getvalue()
20+
)
21+
self.assertEqual(TabularItem.objects.count(), 63)
22+
self.assertIn(
23+
"63 tabular items created",
24+
self.out.getvalue(),
25+
)

‎vbos/datasets/test/test_utils.py‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
from datetime import date
2+
23
from django.test import TestCase
34

4-
from vbos.datasets.utils import CSVRow, GeoJSONProperties
5+
from vbos.datasets.utils import CSVRow, GeoJSONProperties, group_by_dataset
56

67

78
class TestGeoJSONProperties(TestCase):
@@ -65,3 +66,61 @@ def test_class_attributes(self):
6566
assert self.r2.province == "TAFEA"
6667
assert self.r2.area_council == "Futuna"
6768
assert self.r2.value == "154.2"
69+
70+
71+
class TestGroupByDataset(TestCase):
72+
def setUp(self):
73+
self.data = [
74+
{
75+
"Type": "Baseline",
76+
"Cluster": "Education",
77+
"Indicator": "Number Schools",
78+
"Attribute": "ecce",
79+
"Value": 32,
80+
"Province": "TORBA",
81+
"Area Council": "East Gaua",
82+
"Year": "2023",
83+
"Month": "January",
84+
},
85+
{
86+
"Type": "Baseline",
87+
"Cluster": "Education",
88+
"Indicator": "Number Teachers",
89+
"Attribute": "primary",
90+
"Value": 334,
91+
"Province": "TORBA",
92+
"Area Council": "East Gaua",
93+
"Year": "2023",
94+
"Month": "January",
95+
},
96+
{
97+
"Type": "Baseline",
98+
"Cluster": "Education",
99+
"Indicator": "Number Schools",
100+
"Attribute": "tertiary",
101+
"Value": 32,
102+
"Province": "TORBA",
103+
"Area Council": "East Gaua",
104+
"Year": "2023",
105+
"Month": "January",
106+
},
107+
{
108+
"Type": "Baseline",
109+
"Cluster": "Education",
110+
"Indicator": "Number Teachers",
111+
"Attribute": "secondary",
112+
"Value": 534,
113+
"Province": "TORBA",
114+
"Area Council": "East Gaua",
115+
"Year": "2023",
116+
"Month": "January",
117+
},
118+
]
119+
120+
def test_group_by_dataset_function(self):
121+
result = group_by_dataset(self.data)
122+
self.assertEqual(len(result), 2)
123+
self.assertEqual(len(result[0]["items"]), 2)
124+
self.assertEqual(len(result[1]["items"]), 2)
125+
self.assertEqual(result[0]["items"][0]["Attribute"], "ecce")
126+
self.assertEqual(result[0]["items"][1]["Attribute"], "tertiary")

0 commit comments

Comments
 (0)