Merge branch 'main' into feature/add-custom-tiler

Author: weiji14

docs/data.md

@@ -0,0 +1,80 @@
+# Data Imports
+
+To add data into the system, the first step needed is to create a superuser account. Check the [index.md](/docs/index.md#initialize-the-project) for instructions on how to do it.
+
+With the superuser account, you can login on the Administration Interface. The URL will be: `{backend-url}/admin`.
+
+Once logged in, you will see a screen like this:
+
+<img width="2764" height="1492" alt="image" src="https://github.com/user-attachments/assets/109f4712-9577-4bef-8b4c-153620f3a2b1" />
+
+## Adding data
+
+We can add data by uploading raster files (TIFF or VRT format), vector files in the GeoJSON format or Tabular data in CSV format.
+However, before uploading data, you should create the Clusters, and the Dataset entries.
+
+### Creating clusters
+
+On the Administrative Interface, click on Clusters, then on `Add Cluster` button.
+
+You will see a form like this. Add a name and click on Save.
+
+<img width="50%" alt="image" src="https://github.com/user-attachments/assets/c7f62081-3d2f-4fa7-8b24-0a8c032a018c" />
+
+If you need to modify or delete a Cluster, you can do it by accessing the Cluster list page in the administrative interface.
+
+Clicking on the id of the cluster, you will have access to the form to modify it. If you need to delete clusters, select it and then click on the action dropdown and select `Delete selected clusters`. Finally, click on the `Go` button.
+
+<img width="50%" alt="image" src="https://github.com/user-attachments/assets/01f27226-566e-44c4-b279-6c818b53ff2e" />
+
+
+### Creating datasets
+
+The exact same pattern applies when creating Raster, Vector or Tabular datasets. Here you can see the Raster Dataset creation form:
+
+<img width="100%" alt="image" src="https://github.com/user-attachments/assets/01a838fb-8ab1-4cef-bab7-e1ebf624fa8e" />
+
+The forms to create Vector and Tabular datasets are very similar to the Raster one.
+
+### Uploading data
+
+The last and most important step of the data import is to upload the files containing each datasets data.
+
+### Tabular data
+
+Click on the `Tabular Items` link in the Administrative Interface homepage. Then, click on `Import File` in the right-top corner of the page.
+
+<img width="2210" height="1006" alt="image" src="https://github.com/user-attachments/assets/65dc540a-9bdf-4690-ab65-6d005aa22d96" />
+
+Once clicked, you will see a form like this, where you can upload a CSV file and select the dataset to which the data belongs to:
+
+<img width="80%" alt="image" src="https://github.com/user-attachments/assets/2702ae31-a4ec-4cfa-b782-7045876314c4" />
+
+The CSV file needs to be separated by commas and should have the following columns:
+
+- Year
+- Month `(optional)`
+- Province `(optional)`
+- Area Council `(optional)`
+- Attribute
+- Value
+
+The name of the columns can be in lower, UPPER, or Camel Case. You can upload a file with additional columns, and the additional information will be stored in the database.
+
+### Vector data
+
+Click on the `Vector Items` link in the Administrative Interface homepage. Then, click on `Import File` in the right-top corner of the page.
+
+<img width="2204" height="928" alt="image" src="https://github.com/user-attachments/assets/b6cd7cab-c9bd-4956-87f2-08268d848602" />
+
+Once clicked, you will see a form like this, where you can upload a GeoJSON file and select the dataset to which the data belongs to:
+
+<img width="50%" alt="image" src="https://github.com/user-attachments/assets/e65938e5-43b0-405d-9fdd-446b182a4e88" />
+
+If you have never worked with GeoJSON files, you can convert any geospatial data format to GeoJSON using QGIS or ArcGis. The items in the GeoJSON file should have the following columsn:
+
+- Name `(optional)`
+- Ref `(optional)`
+- Province `(optional)`
+- Area Council `(optional)`
+- Attribute `(optional)`

vbos/datasets/admin.py

@@ -91,14 +91,14 @@ def import_file(self, request):
                             VectorItem.objects.create(
                                 dataset=form.cleaned_data["dataset"],
                                 metadata=metadata.properties,
-                                name=metadata.name,
+                                name=metadata.name.strip(),
                                 ref=metadata.ref,
-                                attribute=metadata.attribute,
+                                attribute=metadata.attribute.strip(),
                                 province=Province.objects.filter(
-                                    name__iexact=metadata.province
+                                    name__iexact=metadata.province.strip()
                                 ).first(),
                                 area_council=AreaCouncil.objects.filter(
-                                    name__iexact=metadata.area_council
+                                    name__iexact=metadata.area_council.strip()
                                 ).first(),
                                 geometry=GEOSGeometry(json.dumps(item["geometry"])),
                             )
@@ -184,7 +184,7 @@ def import_file(self, request):
                             TabularItem.objects.create(
                                 dataset=form.cleaned_data["dataset"],
                                 metadata=csv_row.metadata,
-                                attribute=csv_row.attribute,
+                                attribute=csv_row.attribute.strip(),
                                 value=csv_row.value,
                                 date=csv_row.date,
                                 province=Province.objects.filter(

vbos/datasets/fixtures/test.csv

@@ -1,4 +1,4 @@
 Year,Month,Attribute,Province,Area Council,Value,Other
-2024,January,ecce,TAFEA,Futuna,1154,yes
+2024,January,ecce ,TAFEA,Futuna,1154,yes
 2022,may,secondary,TAFEA,Futuna,1154,no
 2025,,primary,,,1154

vbos/datasets/serializers.py

@@ -20,15 +20,13 @@ class Meta:
 
 
 class ProvinceSerializer(GeoFeatureModelSerializer):
-
     class Meta:
         model = Province
         geo_field = "geometry"
         fields = "__all__"
 
 
 class AreaCouncilSerializer(GeoFeatureModelSerializer):
-
     class Meta:
         model = AreaCouncil
         geo_field = "geometry"
@@ -137,6 +135,9 @@ def to_representation(self, instance):
 
 
 class TabularItemExcelSerializer(serializers.ModelSerializer):
+    province = serializers.ReadOnlyField(source="province.name")
+    area_council = serializers.ReadOnlyField(source="area_council.name")
+
     # Dynamically add fields based on all possible keys in the data
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -146,8 +147,8 @@ def __init__(self, *args, **kwargs):
             queryset = self.context["view"].get_queryset()
             all_keys = set()
             for item in queryset:
-                if item.data and isinstance(item.data, dict):
-                    all_keys.update(item.data.keys())
+                if item.metadata and isinstance(item.metadata, dict):
+                    all_keys.update(item.metadata.keys())
 
             # Create a field for each key
             for key in all_keys:
@@ -160,4 +161,11 @@ def __init__(self, *args, **kwargs):
 
     class Meta:
         model = TabularItem
-        fields = ["id"]
+        fields = [
+            "id",
+            "attribute",
+            "date",
+            "value",
+            "province",
+            "area_council",
+        ]

vbos/datasets/test/test_tabular_views.py

@@ -195,3 +195,15 @@ def test_filter_data(self):
         req = self.client.get(url, {"attribute": "population"})
         assert req.status_code == status.HTTP_200_OK
         assert req.data.get("count") == 4
+
+    def test_xlsx_format(self):
+        url = reverse("datasets:tabular-data-xlsx", args=[self.dataset_1.id])
+        req = self.client.get(url)
+        assert req.status_code == status.HTTP_200_OK
+        assert (
+            req.headers["Content-Type"]
+            == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet; charset=utf-8"
+        )
+        assert req.headers[
+            "content-disposition"
+        ] == "attachment; filename=vbos-mis-tabular-{}.xlsx".format(self.dataset_1.id)

vbos/datasets/urls.py

@@ -43,4 +43,9 @@
         views.TabularDatasetDataView.as_view(),
         name="tabular-data",
     ),
+    path(
+        "tabular/<int:pk>/data-xlsx/",
+        views.TabularDatasetXSLXDataView.as_view(),
+        name="tabular-data-xlsx",
+    ),
 ]

vbos/datasets/views.py

@@ -1,9 +1,11 @@
-from django.shortcuts import render
 import django_filters.rest_framework
 from rest_framework.generics import ListAPIView, RetrieveAPIView
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
 from rest_framework_gis.pagination import GeoJsonPagination
 from rest_framework_gis.filters import InBBoxFilter
+from drf_excel.mixins import XLSXFileMixin
+from drf_excel.renderers import XLSXRenderer
+
 
 from vbos.datasets.filters import (
     RasterDatasetFilter,
@@ -122,12 +124,16 @@ class TabularDatasetDetailView(RetrieveAPIView):
 class TabularDatasetDataView(ListAPIView):
     filterset_class = TabularItemFilter
     permission_classes = [IsAuthenticatedOrReadOnly]
+    serializer_class = TabularItemSerializer
 
     def get_queryset(self):
         return TabularItem.objects.filter(dataset=self.kwargs.get("pk"))
 
-    def get_serializer_class(self):
-        # Use different serializer for Excel format
-        if self.request.query_params.get("format") == "xlsx":
-            return TabularItemExcelSerializer
-        return TabularItemSerializer
+
+class TabularDatasetXSLXDataView(XLSXFileMixin, TabularDatasetDataView):
+    serializer_class = TabularItemExcelSerializer
+    renderer_classes = (XLSXRenderer,)
+    pagination_class = None
+
+    def get_filename(self, request, *args, **kwargs):
+        return f"vbos-mis-tabular-{kwargs.get('pk')}.xlsx"