Include new columns to VectorItem and TabularItem tables (#19)

Author: willemarcel

vbos/datasets/admin.py

@@ -8,8 +8,12 @@
 from django.shortcuts import render, redirect, reverse
 from django.urls import path
 
+from vbos.datasets.utils import CSVRow, GeoJSONProperties
+
 from .models import (
+    AreaCouncil,
     Cluster,
+    Province,
     RasterDataset,
     RasterFile,
     TabularDataset,
@@ -44,8 +48,8 @@ class VectorDatasetAdmin(admin.ModelAdmin):
 
 @admin.register(VectorItem)
 class VectorItemAdmin(admin.GISModelAdmin):
-    list_display = ["id", "dataset", "metadata"]
-    list_filter = ["dataset"]
+    list_display = ["id", "dataset", "name", "attribute", "province", "area_council"]
+    list_filter = ["dataset", "province", "area_council"]
 
     def get_urls(self):
         urls = super().get_urls()
@@ -82,10 +86,20 @@ def import_file(self, request):
                     error_count = 0
 
                     for item in geojson_content["features"]:
+                        metadata = GeoJSONProperties(item["properties"])
                         try:
                             VectorItem.objects.create(
                                 dataset=form.cleaned_data["dataset"],
-                                metadata=item["properties"],
+                                metadata=metadata.properties,
+                                name=metadata.name,
+                                ref=metadata.ref,
+                                attribute=metadata.attribute,
+                                province=Province.objects.filter(
+                                    name__iexact=metadata.province
+                                ).first(),
+                                area_council=AreaCouncil.objects.filter(
+                                    name__iexact=metadata.area_council
+                                ).first(),
                                 geometry=GEOSGeometry(json.dumps(item["geometry"])),
                             )
                             created_count += 1
@@ -126,8 +140,8 @@ class TabularDatasetAdmin(admin.ModelAdmin):
 
 @admin.register(TabularItem)
 class TabularItemAdmin(admin.GISModelAdmin):
-    list_display = ["id", "dataset", "data"]
-    list_filter = ["dataset"]
+    list_display = ["id", "dataset", "province", "area_council", "attribute", "value"]
+    list_filter = ["dataset", "province", "area_council"]
 
     def get_urls(self):
         urls = super().get_urls()
@@ -166,8 +180,19 @@ def import_file(self, request):
 
                     for row in reader:  # start=2 to account for header row
                         try:
+                            csv_row = CSVRow(row)
                             TabularItem.objects.create(
-                                dataset=form.cleaned_data["dataset"], data=row
+                                dataset=form.cleaned_data["dataset"],
+                                metadata=csv_row.metadata,
+                                attribute=csv_row.attribute,
+                                value=csv_row.value,
+                                date=csv_row.date,
+                                province=Province.objects.filter(
+                                    name__iexact=csv_row.province
+                                ).first(),
+                                area_council=AreaCouncil.objects.filter(
+                                    name__iexact=csv_row.area_council
+                                ).first(),
                             )
                             created_count += 1
                         except Exception as e:

vbos/datasets/filters.py

@@ -7,6 +7,8 @@
 )
 
 from .models import (
+    AreaCouncil,
+    Province,
     RasterDataset,
     TabularItem,
     VectorDataset,
@@ -50,9 +52,20 @@ class Meta:
         fields = ["name", "source", "cluster", "created", "updated"]
 
 
-class TabularItemFilter(FilterSet):
-    filter = CharFilter(
-        field_name="data",
+class DataItemsBaseFilter(FilterSet):
+    attribute = CharFilter(lookup_expr="icontains")
+    province = ModelChoiceFilter(
+        field_name="province__name",
+        to_field_name="name__iexact",
+        queryset=Province.objects.all(),
+    )
+    area_council = ModelChoiceFilter(
+        field_name="area_council__name",
+        to_field_name="name__iexact",
+        queryset=AreaCouncil.objects.all(),
+    )
+    metadata = CharFilter(
+        field_name="metadata",
         method="filter_metadata",
         help_text="""Filter by the content of the data JSONField.""",
     )
@@ -91,18 +104,27 @@ def filter_metadata(self, queryset, name, value):
 
         return queryset
 
+
+class TabularItemFilter(DataItemsBaseFilter):
+    date = DateFromToRangeFilter()
+
     class Meta:
         model = TabularItem
-        fields = ["filter", "id"]
+        fields = ["metadata", "attribute", "province", "area_council", "id", "date"]
 
 
-class VectorItemFilter(TabularItemFilter):
-    filter = CharFilter(
-        field_name="metadata",
-        method="filter_metadata",
-        help_text="""Filter by the content of the metadata JSONField.""",
-    )
+class VectorItemFilter(DataItemsBaseFilter):
+    name = CharFilter(lookup_expr="icontains")
+    ref = CharFilter(lookup_expr="icontains")
 
     class Meta:
         model = VectorItem
-        fields = ["filter", "id"]
+        fields = [
+            "metadata",
+            "attribute",
+            "province",
+            "area_council",
+            "id",
+            "name",
+            "ref",
+        ]

vbos/datasets/fixtures/test.csv

@@ -0,0 +1,4 @@
+Year,Month,Attribute,Province,Area Council,Value,Other
+2024,January,ecce,TAFEA,Futuna,1154,yes
+2022,may,secondary,TAFEA,Futuna,1154,no
+2025,,primary,,,1154

vbos/datasets/fixtures/test.geojson

@@ -4,7 +4,12 @@
     {
       "type": "Feature",
       "properties": {
-        "name": "Area 1"
+        "name": "Area 1",
+        "ref": "12NC",
+        "Province": "Torba",
+        "Area Council": "East Gaua",
+        "Attribute": "Schools",
+        "extra": "value"
       },
       "geometry": {
         "coordinates": [
@@ -36,7 +41,13 @@
     },
     {
       "type": "Feature",
-      "properties": {"name": "Line 1", "col": "val"},
+      "properties": {
+        "name": "Line 1",
+        "ref": "13NC",
+        "Province": "TAFEA",
+        "Area Council": "Futuna",
+        "Attribute": "Roads"
+    },
       "geometry": {
         "coordinates": [
           [
@@ -53,7 +64,14 @@
     },
     {
       "type": "Feature",
-      "properties": {"name": "Point 1", "source": "OpenStreetMap"},
+      "properties": {
+        "name": "Point 2",
+        "ref": "14NC",
+        "Province": "TAFEA",
+        "Area Council": "Futuna",
+        "Attribute": "Business",
+        "source": "OpenStreetMap"
+      },
       "geometry": {
         "coordinates": [
           167.72470,

vbos/datasets/migrations/0011_rename_data_tabularitem_metadata_and_more.py

@@ -0,0 +1,85 @@
+# Generated by Django 5.2.5 on 2025-10-03 12:44
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datasets", "0010_auto_20251001_1132"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="tabularitem",
+            old_name="data",
+            new_name="metadata",
+        ),
+        migrations.AddField(
+            model_name="tabularitem",
+            name="area_council",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="datasets.areacouncil",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tabularitem",
+            name="attribute",
+            field=models.CharField(blank=True, max_length=155, null=True),
+        ),
+        migrations.AddField(
+            model_name="tabularitem",
+            name="date",
+            field=models.DateField(null=True),
+        ),
+        migrations.AddField(
+            model_name="tabularitem",
+            name="province",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="datasets.province",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tabularitem",
+            name="value",
+            field=models.FloatField(default=0),
+        ),
+        migrations.AddField(
+            model_name="vectoritem",
+            name="area_council",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="datasets.areacouncil",
+            ),
+        ),
+        migrations.AddField(
+            model_name="vectoritem",
+            name="attribute",
+            field=models.CharField(blank=True, max_length=155, null=True),
+        ),
+        migrations.AddField(
+            model_name="vectoritem",
+            name="name",
+            field=models.CharField(blank=True, max_length=155, null=True),
+        ),
+        migrations.AddField(
+            model_name="vectoritem",
+            name="province",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="datasets.province",
+            ),
+        ),
+        migrations.AddField(
+            model_name="vectoritem",
+            name="ref",
+            field=models.CharField(blank=True, max_length=50, null=True),
+        ),
+    ]

vbos/datasets/models.py

@@ -119,11 +119,19 @@ class Meta:
 
 class VectorItem(models.Model):
     dataset = models.ForeignKey(VectorDataset, on_delete=models.CASCADE)
+    name = models.CharField(max_length=155, blank=True, null=True)
+    ref = models.CharField(max_length=50, blank=True, null=True)
+    attribute = models.CharField(max_length=155, blank=True, null=True)
+    province = models.ForeignKey(Province, null=True, on_delete=models.PROTECT)
+    area_council = models.ForeignKey(AreaCouncil, null=True, on_delete=models.PROTECT)
     geometry = models.GeometryField()
     metadata = models.JSONField(default=dict, blank=True, null=True)
 
     def __str__(self):
-        return f"{self.id}"
+        if self.name:
+            return f"{self.id} ({self.name})"
+        else:
+            return f"{self.id}"
 
     class Meta:
         ordering = ["id"]
@@ -149,7 +157,12 @@ class Meta:
 
 class TabularItem(models.Model):
     dataset = models.ForeignKey(TabularDataset, on_delete=models.CASCADE)
-    data = models.JSONField(default=dict)
+    date = models.DateField(null=True)
+    attribute = models.CharField(max_length=155, blank=True, null=True)
+    value = models.FloatField(default=0)
+    province = models.ForeignKey(Province, null=True, on_delete=models.PROTECT)
+    area_council = models.ForeignKey(AreaCouncil, null=True, on_delete=models.PROTECT)
+    metadata = models.JSONField(default=dict)
 
     def __str__(self):
         return f"{self.id}"

vbos/datasets/serializers.py

@@ -44,25 +44,21 @@ class Meta:
 
 
 class VectorItemSerializer(GeoFeatureModelSerializer):
+    province = serializers.ReadOnlyField(source="province.name")
+    area_council = serializers.ReadOnlyField(source="area_council.name")
+
     class Meta:
         model = VectorItem
         geo_field = "geometry"
-        fields = ["id", "metadata"]
-
-    def get_properties(self, instance, fields):
-        # This is a PostgreSQL HStore field, which django maps to a dict
-        return instance.metadata
-
-    def unformat_geojson(self, feature):
-        attrs = {
-            self.Meta.geo_field: feature["geometry"],
-            "metadata": feature["properties"],
-        }
-
-        if self.Meta.bbox_geo_field and "bbox" in feature:
-            attrs[self.Meta.bbox_geo_field] = Polygon.from_bbox(feature["bbox"])
-
-        return attrs
+        fields = [
+            "id",
+            "name",
+            "ref",
+            "attribute",
+            "province",
+            "area_council",
+            "metadata",
+        ]
 
 
 class TabularDatasetSerializer(serializers.ModelSerializer):
@@ -74,15 +70,26 @@ class Meta:
 
 
 class TabularItemSerializer(serializers.ModelSerializer):
+    province = serializers.ReadOnlyField(source="province.name")
+    area_council = serializers.ReadOnlyField(source="area_council.name")
+
     class Meta:
         model = TabularItem
-        fields = ["id", "data"]
+        fields = [
+            "id",
+            "attribute",
+            "date",
+            "value",
+            "province",
+            "area_council",
+            "metadata",
+        ]
 
     def to_representation(self, instance):
         representation = super().to_representation(instance)
 
         # Extract the data field and merge it with the top level fields
-        data_content = representation.pop("data", {})
+        data_content = representation.pop("metadata", {})
 
         return {**representation, **data_content}
 
@@ -103,7 +110,10 @@ def __init__(self, *args, **kwargs):
             # Create a field for each key
             for key in all_keys:
                 self.fields[key] = serializers.CharField(
-                    source=f"data.{key}", required=False, allow_blank=True, default=""
+                    source=f"metadata.{key}",
+                    required=False,
+                    allow_blank=True,
+                    default="",
                 )
 
     class Meta: