Add Raster and Tabular data models and views

Author: willemarcel

requirements.txt

@@ -18,6 +18,7 @@ djangorestframework-gis==1.2.0
 django-filter==24.3
 drf_spectacular==0.28.0
 django-cors-headers==4.7.0
+drf-excel==2.5.3
 
 # Developer Tools
 ipdb==0.13.13

vbos/config/common.py

@@ -182,12 +182,16 @@ class Common(Configuration):
     # Django Rest Framework
     REST_FRAMEWORK = {
         "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination",
-        "PAGE_SIZE": int(os.getenv("DJANGO_PAGINATION_LIMIT", 10)),
+        "PAGE_SIZE": int(os.getenv("DJANGO_PAGINATION_LIMIT", 20)),
         "DATETIME_FORMAT": "%Y-%m-%dT%H:%M:%S%z",
         "DEFAULT_RENDERER_CLASSES": (
             "rest_framework.renderers.JSONRenderer",
             "rest_framework.renderers.BrowsableAPIRenderer",
+            "drf_excel.renderers.XLSXRenderer",
         ),
+        "DEFAULT_FILTER_BACKENDS": [
+            "django_filters.rest_framework.DjangoFilterBackend"
+        ],
         "DEFAULT_PERMISSION_CLASSES": [
             "rest_framework.permissions.IsAuthenticated",
         ],

vbos/datasets/admin.py

@@ -1,5 +1,16 @@
 from django.contrib.gis import admin
-from .models import VectorDataset, VectorItem
+from .models import (
+    RasterDataset,
+    TabularDataset,
+    TabularItem,
+    VectorDataset,
+    VectorItem,
+)
+
+
+@admin.register(RasterDataset)
+class RasterDatasetAdmin(admin.ModelAdmin):
+    list_display = ["id", "name", "created", "updated", "file_path"]
 
 
 @admin.register(VectorDataset)
@@ -10,3 +21,13 @@ class VectorDatasetAdmin(admin.ModelAdmin):
 @admin.register(VectorItem)
 class VectorItemAdmin(admin.GISModelAdmin):
     list_display = ["id", "metadata"]
+
+
+@admin.register(TabularDataset)
+class TabularDatasetAdmin(admin.ModelAdmin):
+    list_display = ["id", "name", "created", "updated"]
+
+
+@admin.register(TabularItem)
+class TabularItemAdmin(admin.GISModelAdmin):
+    list_display = ["id", "data"]

vbos/datasets/filters.py

@@ -0,0 +1,37 @@
+from django_filters import (
+    FilterSet,
+    BooleanFilter,
+    CharFilter,
+    OrderingFilter,
+    DateFromToRangeFilter,
+    ModelChoiceFilter,
+)
+
+from .models import RasterDataset, VectorDataset, TabularDataset
+
+
+class DatasetFilter(FilterSet):
+    name = CharFilter(field_name="name", lookup_expr="icontains")
+    created = DateFromToRangeFilter()
+    updated = DateFromToRangeFilter()
+    order_by = OrderingFilter(
+        fields=("name", "id", "updated", "created"),
+    )
+
+
+class RasterDatasetFilter(DatasetFilter):
+    class Meta:
+        model = RasterDataset
+        fields = ["name", "created", "updated"]
+
+
+class VectorDatasetFilter(DatasetFilter):
+    class Meta:
+        model = VectorDataset
+        fields = ["name", "created", "updated"]
+
+
+class TabularDatasetFilter(DatasetFilter):
+    class Meta:
+        model = TabularDataset
+        fields = ["name", "created", "updated"]

vbos/datasets/migrations/0002_tabulardataset_tabularitem.py

@@ -0,0 +1,59 @@
+# Generated by Django 5.2.5 on 2025-08-29 20:49
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datasets", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="TabularDataset",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=155)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                "ordering": ["id"],
+            },
+        ),
+        migrations.CreateModel(
+            name="TabularItem",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("data", models.JSONField(default=dict)),
+                (
+                    "dataset",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="datasets.tabulardataset",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["id"],
+            },
+        ),
+    ]

vbos/datasets/migrations/0003_rasterdataset.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.2.5 on 2025-08-29 21:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datasets", "0002_tabulardataset_tabularitem"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RasterDataset",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=155)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+                ("file_path", models.CharField(max_length=2000)),
+            ],
+            options={
+                "ordering": ["id"],
+            },
+        ),
+    ]

vbos/datasets/models.py

@@ -1,6 +1,19 @@
 from django.contrib.gis.db import models
 
 
+class RasterDataset(models.Model):
+    name = models.CharField(max_length=155)
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+    file_path = models.CharField(max_length=2000, null=False, blank=False)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        ordering = ["id"]
+
+
 class VectorDataset(models.Model):
     name = models.CharField(max_length=155)
     created = models.DateTimeField(auto_now_add=True)
@@ -23,3 +36,26 @@ def __str__(self):
 
     class Meta:
         ordering = ["id"]
+
+
+class TabularDataset(models.Model):
+    name = models.CharField(max_length=155)
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        ordering = ["id"]
+
+
+class TabularItem(models.Model):
+    dataset = models.ForeignKey(TabularDataset, on_delete=models.CASCADE)
+    data = models.JSONField(default=dict)
+
+    def __str__(self):
+        return f"{self.id}"
+
+    class Meta:
+        ordering = ["id"]

vbos/datasets/serializers.py

@@ -1,7 +1,19 @@
 from rest_framework import serializers
 from rest_framework_gis.serializers import GeoFeatureModelSerializer
 
-from .models import VectorDataset, VectorItem
+from .models import (
+    RasterDataset,
+    TabularDataset,
+    TabularItem,
+    VectorDataset,
+    VectorItem,
+)
+
+
+class RasterDatasetSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = RasterDataset
+        fields = "__all__"
 
 
 class VectorDatasetSerializer(serializers.ModelSerializer):
@@ -30,3 +42,47 @@ def unformat_geojson(self, feature):
             attrs[self.Meta.bbox_geo_field] = Polygon.from_bbox(feature["bbox"])
 
         return attrs
+
+
+class TabularDatasetSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = TabularDataset
+        fields = "__all__"
+
+
+class TabularItemSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = TabularItem
+        fields = ["id", "data"]
+
+    def to_representation(self, instance):
+        representation = super().to_representation(instance)
+
+        # Extract the data field and merge it with the top level fields
+        data_content = representation.pop("data", {})
+
+        return {**representation, **data_content}
+
+
+class TabularItemExcelSerializer(serializers.ModelSerializer):
+    # Dynamically add fields based on all possible keys in the data
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # Get all possible keys from the queryset
+        if self.context.get("view"):
+            queryset = self.context["view"].get_queryset()
+            all_keys = set()
+            for item in queryset:
+                if item.data and isinstance(item.data, dict):
+                    all_keys.update(item.data.keys())
+
+            # Create a field for each key
+            for key in all_keys:
+                self.fields[key] = serializers.CharField(
+                    source=f"data.{key}", required=False, allow_blank=True, default=""
+                )
+
+    class Meta:
+        model = TabularItem
+        fields = ["id"]

vbos/datasets/test/test_raster_views.py

@@ -0,0 +1,32 @@
+from rest_framework import status
+from rest_framework.test import APITestCase
+from django.urls import reverse
+
+from ..models import RasterDataset
+
+
+class TestRasterDatasetListDetailViews(APITestCase):
+    def setUp(self):
+        self.dataset_1 = RasterDataset.objects.create(
+            name="Rainfall", file_path="cogs/rainfall.tiff"
+        )
+        self.dataset_2 = RasterDataset.objects.create(
+            name="Coastline changes", file_path="cogs/coastlines.tiff"
+        )
+        self.url = reverse("datasets:raster-list")
+
+    def test_raster_datasets_list(self):
+        req = self.client.get(self.url)
+        assert req.status_code == status.HTTP_200_OK
+        assert req.data.get("count") == 2
+        assert req.data.get("results")[0]["name"] == "Rainfall"
+        assert req.data.get("results")[1]["name"] == "Coastline changes"
+
+    def test_raster_datasets_detail(self):
+        url = reverse("datasets:raster-detail", args=[self.dataset_1.id])
+        req = self.client.get(url)
+        assert req.status_code == status.HTTP_200_OK
+        assert req.data.get("name") == "Rainfall"
+        assert req.data.get("file_path") == "cogs/rainfall.tiff"
+        assert req.data.get("created")
+        assert req.data.get("updated")