Merge pull request #1894 from OpenEnergyPlatform/feature-1890-http-api-schenario-bundle-scenario-dataset

Feature-1890-http-api-schenario-bundle-scenario-dataset

Author: jh-RLI

.gitignore

@@ -86,7 +86,6 @@ venv*/
 apache*
 /oep-django-5
 
-
 .DS_Store
 
 # Deployment files

api/serializers.py

@@ -1,8 +1,13 @@
+from re import match
+from uuid import UUID
+
 from django.urls import reverse
 from rest_framework import serializers
 
+from dataedit.helper import get_readable_table_name
 from dataedit.models import Table
 from modelview.models import Energyframework, Energymodel
+from oeplatform.settings import URL
 
 
 class EnergyframeworkSerializer(serializers.ModelSerializer):
@@ -53,3 +58,104 @@ class Meta:
         model = Table
         # fields = ["id", "model_name", "acronym", "url"]
         fields = ["id", "name", "human_readable_name", "url"]
+
+
+class DatasetSerializer(serializers.Serializer):
+    name = serializers.CharField(max_length=255, required=True)
+    external_url = serializers.URLField(
+        max_length=1000, required=False, allow_null=True
+    )
+    type = serializers.ChoiceField(choices=["input", "output"], required=True)
+    # title = serializers.SerializerMethodField()
+
+    # ✅ Basic validation for 'name' (regex check only)
+    def validate_name(self, value):
+        if not match(r"^[\w]+$", value):
+            raise serializers.ValidationError(
+                "Dataset name should contain only alphanumeric characters "
+                "and underscores."
+            )
+        return value  # Don't check DB here, do it in validate()
+
+    # ✅ Main validation logic (includes db check for object existence)
+    def validate(self, data):
+        name = data.get("name")
+        external_url = data.get("external_url")
+
+        if external_url:
+            # ✅ External URL provided → Skip DB check for 'name'
+            if not external_url.startswith("https://databus.openenergyplatform.org"):
+                raise serializers.ValidationError(
+                    {
+                        "external_url": (
+                            "If you want to link distributions stored outside the OEP, "
+                            "please use the Databus: "
+                            "https://databus.openenergyplatform.org/app/publish-wizard "
+                            "to register your data and use the file or version URI as "
+                            "a persistent identifier."
+                        )
+                    }
+                )
+            data["name"] = f"{name} (external dataset)"
+        else:
+            # ✅ No external URL → Validate 'name' in the database
+            if not Table.objects.filter(name=name).exists():
+                raise serializers.ValidationError(
+                    {
+                        "name": f"Dataset '{name}' does not exist in the database."
+                        "If you want to add links to external distributions please "
+                        "add 'external_url' to the request body."
+                    }
+                )
+            full_label = self.get_title(data)
+            if full_label:
+                data["name"] = full_label
+
+            # ✅ Generate internal distribution URL
+            reversed_url = reverse(
+                "dataedit:view",
+                kwargs={"schema": "scenario", "table": name},
+            )
+            data["external_url"] = f"{URL}{reversed_url}"
+
+        return data  # Return updated data with 'distribution_url' if applicable
+
+    def get_title(self, data):
+        name = data.get("name")
+        # ✅ Generate internal distribution label
+        full_label = get_readable_table_name(table_obj=Table.objects.get(name=name))
+        if full_label:
+            return full_label
+        else:
+            return None
+
+
+class ScenarioBundleScenarioDatasetSerializer(serializers.Serializer):
+    scenario_bundle = serializers.UUIDField(
+        required=True
+    )  # Validate the scenario bundle UUID
+    scenario = serializers.UUIDField(required=True)  # Validate the scenario UUID
+    datasets = serializers.ListField(
+        child=DatasetSerializer(), required=True
+    )  # List of datasets with 'name' and 'type'
+
+    # Custom validation for 'scenario'
+    def validate_scenario(self, value):
+        try:
+            UUID(str(value))
+        except ValueError:
+            raise serializers.ValidationError("Invalid UUID format for scenario.")
+
+        return value
+
+    # Custom validation for the entire dataset list
+    def validate_dataset(self, value):
+        if not value:
+            raise serializers.ValidationError("The dataset list cannot be empty.")
+
+        # Check for duplicates in dataset names
+        dataset_names = [dataset["name"] for dataset in value]
+        if len(dataset_names) != len(set(dataset_names)):
+            raise serializers.ValidationError("Dataset names must be unique.")
+
+        return value

api/urls.py

@@ -203,4 +203,9 @@
         views.ScenarioDataTablesListAPIView.as_view(),
         name="list-scenario-datasets",
     ),
+    re_path(
+        r"^v0/scenario-bundle/scenario/manage-datasets/?$",
+        views.ManageOekgScenarioDatasets.as_view(),
+        name="add-scenario-datasets",
+    ),
 ]

api/utils.py

@@ -0,0 +1,14 @@
+"""
+Collection of utility functions for the API used to define various action
+like processing steps.
+"""
+
+from oekg.sparqlModels import DatasetConfig
+
+
+def get_dataset_configs(validated_data) -> list[DatasetConfig]:
+    """Converts validated serializer data into a list of DatasetConfig objects."""
+    return [
+        DatasetConfig.from_serializer_data(validated_data, dataset_entry)
+        for dataset_entry in validated_data["datasets"]
+    ]

api/views.py

@@ -31,6 +31,10 @@
 from omi.dialects.oep.compiler import JSONCompiler
 from omi.structure import OEPMetadata
 from rest_framework import generics, status
+from rest_framework.permissions import IsAuthenticated
+
+# views.py
+from rest_framework.response import Response
 from rest_framework.views import APIView
 
 import api.parser
@@ -43,13 +47,19 @@
 from api.serializers import (
     EnergyframeworkSerializer,
     EnergymodelSerializer,
+    ScenarioBundleScenarioDatasetSerializer,
     ScenarioDataTablesSerializer,
 )
+from api.utils import get_dataset_configs
 from dataedit.models import Embargo
 from dataedit.models import Schema as DBSchema
 from dataedit.models import Table as DBTable
 from dataedit.views import get_tag_keywords_synchronized_metadata, schema_whitelist
+from factsheet.permission_decorator import post_only_if_user_is_owner_of_scenario_bundle
 from modelview.models import Energyframework, Energymodel
+
+# from oekg.sparqlQuery import remove_datasets_from_scenario
+from oekg.utils import process_datasets_sparql_query
 from oeplatform.settings import PLAYGROUNDS, UNVERSIONED_SCHEMAS, USE_LOEP, USE_ONTOP
 
 if USE_LOEP:
@@ -244,11 +254,11 @@ class Sequence(APIView):
     @api_exception
     def put(self, request, schema, sequence):
         if schema not in PLAYGROUNDS and schema not in UNVERSIONED_SCHEMAS:
-            raise APIError('Schema is not in allowed set of schemes for upload')
+            raise APIError("Schema is not in allowed set of schemes for upload")
         if schema.startswith("_"):
-            raise APIError('Schema starts with _, which is not allowed')
+            raise APIError("Schema starts with _, which is not allowed")
         if request.user.is_anonymous:
-            raise APIError('User is anonymous', 401)
+            raise APIError("User is anonymous", 401)
         if actions.has_table(dict(schema=schema, sequence_name=sequence), {}):
             raise APIError("Sequence already exists", 409)
         return self.__create_sequence(request, schema, sequence, request.data)
@@ -257,11 +267,11 @@ def put(self, request, schema, sequence):
     @require_delete_permission
     def delete(self, request, schema, sequence):
         if schema not in PLAYGROUNDS and schema not in UNVERSIONED_SCHEMAS:
-            raise APIError('Schema is not in allowed set of schemes for upload')
+            raise APIError("Schema is not in allowed set of schemes for upload")
         if schema.startswith("_"):
-            raise APIError('Schema starts with _, which is not allowed')
+            raise APIError("Schema starts with _, which is not allowed")
         if request.user.is_anonymous:
-            raise APIError('User is anonymous', 401)
+            raise APIError("User is anonymous", 401)
         return self.__delete_sequence(request, schema, sequence, request.data)
 
     @load_cursor()
@@ -371,9 +381,9 @@ def post(self, request, schema, table):
         :return:
         """
         if schema not in PLAYGROUNDS and schema not in UNVERSIONED_SCHEMAS:
-            raise APIError('Schema is not in allowed set of schemes for upload')
+            raise APIError("Schema is not in allowed set of schemes for upload")
         if schema.startswith("_"):
-            raise APIError('Schema starts with _, which is not allowed')
+            raise APIError("Schema starts with _, which is not allowed")
         json_data = request.data
 
         if "column" in json_data["type"]:
@@ -423,11 +433,11 @@ def put(self, request, schema, table):
         :return:
         """
         if schema not in PLAYGROUNDS and schema not in UNVERSIONED_SCHEMAS:
-            raise APIError('Schema is not in allowed set of schemes for upload')
+            raise APIError("Schema is not in allowed set of schemes for upload")
         if schema.startswith("_"):
-            raise APIError('Schema starts with _, which is not allowed')
+            raise APIError("Schema starts with _, which is not allowed")
         if request.user.is_anonymous:
-            raise APIError('User is anonymous', 401)
+            raise APIError("User is anonymous", 401)
         if actions.has_table(dict(schema=schema, table=table), {}):
             raise APIError("Table already exists", 409)
         json_data = request.data.get("query", {})
@@ -967,10 +977,10 @@ def get(self, request, schema, table, row_id=None):
                 content_type="text/csv",
                 session=session,
             )
-            response["Content-Disposition"] = (
-                'attachment; filename="{schema}__{table}.csv"'.format(
-                    schema=schema, table=table
-                )
+            response[
+                "Content-Disposition"
+            ] = 'attachment; filename="{schema}__{table}.csv"'.format(
+                schema=schema, table=table
             )
             return response
         elif format == "datapackage":
@@ -998,10 +1008,10 @@ def get(self, request, schema, table, row_id=None):
                 content_type="application/zip",
                 session=session,
             )
-            response["Content-Disposition"] = (
-                'attachment; filename="{schema}__{table}.zip"'.format(
-                    schema=schema, table=table
-                )
+            response[
+                "Content-Disposition"
+            ] = 'attachment; filename="{schema}__{table}.zip"'.format(
+                schema=schema, table=table
             )
             return response
         else:
@@ -1574,3 +1584,56 @@ class ScenarioDataTablesListAPIView(generics.ListAPIView):
     topic = "scenario"
     queryset = DBTable.objects.filter(schema__name=topic)
     serializer_class = ScenarioDataTablesSerializer
+
+
+class ManageOekgScenarioDatasets(APIView):
+    permission_classes = [IsAuthenticated]  # Require authentication
+
+    @post_only_if_user_is_owner_of_scenario_bundle
+    def post(self, request):
+        serializer = ScenarioBundleScenarioDatasetSerializer(data=request.data)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        try:
+            dataset_configs = get_dataset_configs(serializer.validated_data)
+            response_data = process_datasets_sparql_query(dataset_configs)
+        except APIError as e:
+            return Response({"error": str(e)}, status=e.status)
+        except Exception:
+            return Response({"error": "An unexpected error occurred."}, status=500)
+
+        if "error" in response_data:
+            return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
+
+        return Response(response_data, status=status.HTTP_200_OK)
+
+    # @post_only_if_user_is_owner_of_scenario_bundle
+    # def delete(self, request):
+    #     serializer = ScenarioBundleScenarioDatasetSerializer(data=request.data)
+    #     if serializer.is_valid():
+    #         scenario_uuid = serializer.validated_data["scenario"]
+    #         datasets = serializer.validated_data["datasets"]
+
+    #         # Iterate over each dataset to process it properly
+    #         for dataset in datasets:
+    #             dataset_name = dataset["name"]
+    #             dataset_type = dataset["type"]
+
+    #             # Remove the dataset from the scenario in the bundle
+    #             success = remove_datasets_from_scenario(
+    #                 scenario_uuid, dataset_name, dataset_type
+    #             )
+
+    #             if not success:
+    #                 return Response(
+    #                     {"error": f"Failed to remove dataset {dataset_name}"},
+    #                     status=status.HTTP_400_BAD_REQUEST,
+    #                 )
+
+    #         return Response(
+    #             {"message": "Datasets removed successfully"},
+    #             status=status.HTTP_200_OK,
+    #         )
+
+    #     return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

dataedit/models.py

@@ -16,6 +16,7 @@
     IntegerField,
     JSONField,
 )
+from django.urls import reverse
 from django.utils import timezone
 
 # Create your models here.
@@ -77,6 +78,9 @@ class Table(Tagable):
     is_publish = BooleanField(null=False, default=False)
     human_readable_name = CharField(max_length=1000, null=True)
 
+    def get_absolute_url(self):
+        return reverse("dataedit:view", kwargs={"pk": self.pk})
+
     @classmethod
     def load(cls, schema, table):
         """
@@ -719,5 +723,6 @@ def filter_opr_by_table(schema, table):
         """
         return PeerReview.objects.filter(schema=schema, table=table)
 
+    @staticmethod
     def filter_opr_by_id(opr_id):
         return PeerReview.objects.filter(id=opr_id).first()

dataedit/views.py

@@ -2355,9 +2355,6 @@ def post(self, request, schema, table, review_id):
         Handle POST requests for contributor's review. Merges and updates
         the review data in the PeerReview table.
 
-        Missing parts:
-            - merge contributor field review and reviewer field review
-
         Args:
             request (HttpRequest): The incoming HTTP POST request.
             schema (str): The schema of the table.
@@ -2367,9 +2364,6 @@ def post(self, request, schema, table, review_id):
         Returns:
             HttpResponse: Rendered HTML response for contributor review.
 
-        Note:
-            This method has some missing parts regarding the merging of contributor
-            and reviewer field review.
         """
 
         context = {}