Merge pull request #104 from open-plan-tool/feature/sensitivity-analysis

Implement sensitivity analysis

Author: Bachibouzouk

app/dashboard/helpers.py

@@ -25,6 +25,7 @@
 EMPTY_SUBCAT = "none"
 
 KPI_PARAMETERS = {}
+KPI_PARAMETERS_ASSETS = {}
 
 if os.path.exists(staticfiles_storage.path("MVS_kpis_list.csv")) is True:
     with open(staticfiles_storage.path("MVS_kpis_list.csv")) as csvfile:
@@ -65,17 +66,24 @@
         for i, row in enumerate(csvreader):
             if i == 0:
                 hdr = [el.replace(" ", "_").replace(":", "").lower() for el in row]
-                print(hdr)
                 label_idx = hdr.index("label")
                 cat_idx = hdr.index("category")
+                scope_idx = hdr.index("scope")
             else:
                 label = row[label_idx]
                 category = row[cat_idx]
+                scope = row[scope_idx]
+
                 if category != "files":
                     KPI_PARAMETERS[label] = {
                         k: _(v) if k == "verbose" or k == "definition" else v
                         for k, v in zip(hdr, row)
                     }
+                    if "asset" in scope:
+                        KPI_PARAMETERS_ASSETS[label] = {
+                            k: _(v) if k == "verbose" or k == "definition" else v
+                            for k, v in zip(hdr, row)
+                        }
 
 
 #### FUNCTIONS ####

app/epa/settings.py

@@ -184,6 +184,8 @@
 MVS_POST_URL = f"{MVS_API_HOST}/sendjson/openplan"
 MVS_GET_URL = f"{MVS_API_HOST}/check/"
 MVS_LP_FILE_URL = f"{MVS_API_HOST}/get_lp_file/"
+MVS_SA_POST_URL = f"{MVS_API_HOST}/sendjson/openplan/sensitivity-analysis"
+MVS_SA_GET_URL = f"{MVS_API_HOST}/check-sensitivity-analysis/"
 
 # Allow iframes to show in page
 X_FRAME_OPTIONS = "SAMEORIGIN"

app/projects/dtos.py

@@ -4,7 +4,16 @@
 from numpy.core import long
 from datetime import date, datetime, time
 
-from projects.models import *
+from projects.models import (
+    ConnectionLink,
+    Scenario,
+    Project,
+    EconomicData,
+    Asset,
+    Bus,
+    Constraint,
+    ValueType,
+)
 
 
 class ProjectDataDto:

app/projects/forms.py

@@ -23,6 +23,8 @@
 from projects.models import *
 from projects.constants import MAP_EPA_MVS, RENEWABLE_ASSETS
 
+from dashboard.helpers import KPI_PARAMETERS_ASSETS
+
 from django.utils.translation import ugettext_lazy as _
 from django.conf import settings as django_settings
 
@@ -36,7 +38,11 @@
                 label_idx = hdr.index("label")
             else:
                 label = row[label_idx]
-                PARAMETERS[label] = {k: v for k, v in zip(hdr, row)}
+                PARAMETERS[label] = {}
+                for k, v in zip(hdr, row):
+                    if k == "sensitivity_analysis":
+                        v = bool(int(v))
+                    PARAMETERS[label][k] = v
 
 
 def gettext_variables(some_string, lang="de"):
@@ -71,7 +77,7 @@ def set_parameter_info(param_name, field, parameters=PARAMETERS):
     verbose = None
     default_value = None
     if param_name in PARAMETERS:
-        help_text = PARAMETERS[param_name][":Definition:"]
+        help_text = PARAMETERS[param_name][":Definition_Short:"]
         unit = PARAMETERS[param_name][":Unit:"]
         verbose = PARAMETERS[param_name]["verbose"]
         default_value = PARAMETERS[param_name][":Default:"]
@@ -481,6 +487,54 @@ class Meta:
         exclude = ["scenario", "value"]
 
 
+class SensitivityAnalysisForm(ModelForm):
+    output_parameters_names = forms.MultipleChoiceField(
+        choices=[
+            (v, _(KPI_PARAMETERS_ASSETS[v]["verbose"])) for v in KPI_PARAMETERS_ASSETS
+        ]
+    )
+
+    class Meta:
+        model = SensitivityAnalysis
+        fields = [
+            "variable_name",
+            "variable_min",
+            "variable_max",
+            "variable_step",
+            "variable_reference",
+            "output_parameters_names",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        scen_id = kwargs.pop("scen_id", None)
+        super().__init__(*args, **kwargs)
+
+        forbidden_parameters_for_sa = ("name", "input_timeseries")
+
+        if scen_id is not None:
+            scenario = Scenario.objects.get(id=scen_id)
+            asset_parameters = []
+            for asset in scenario.asset_set.all():
+                asset_parameters += [
+                    (f"{asset.name}.{p}", _(p) + f" ({asset.name})")
+                    for p in asset.visible_fields
+                    if p not in forbidden_parameters_for_sa
+                ]
+            self.fields["variable_name"] = forms.ChoiceField(choices=asset_parameters)
+            # self.fields["output_parameters_names"] = forms.MultipleChoiceField(choices = [(v, _(KPI_PARAMETERS_ASSETS[v]["verbose"])) for v in KPI_PARAMETERS_ASSETS])
+            # TODO restrict possible parameters here
+            self.fields["output_parameters_names"].choices = [
+                (v, _(KPI_PARAMETERS_ASSETS[v]["verbose"]))
+                for v in KPI_PARAMETERS_ASSETS
+            ]
+
+    def clean_output_parameters_names(self):
+        """method which gets called upon form validation"""
+        data = self.cleaned_data["output_parameters_names"]
+        data_js = json.dumps(data)
+        return data_js
+
+
 class BusForm(OpenPlanModelForm):
     def __init__(self, *args, **kwargs):
         bus_type_name = kwargs.pop("asset_type", None)  # always = bus

app/projects/helpers.py

@@ -0,0 +1,121 @@
+import json
+from projects.dtos import convert_to_dto
+
+
+# Helper method to clean dict data from empty values
+def remove_empty_elements(d):
+    def empty(x):
+        return x is None or x == {} or x == []
+
+    if not isinstance(d, (dict, list)):
+        return d
+    elif isinstance(d, list):
+        return [v for v in (remove_empty_elements(v) for v in d) if not empty(v)]
+    else:
+        return {
+            k: v
+            for k, v in ((k, remove_empty_elements(v)) for k, v in d.items())
+            if not empty(v)
+        }
+
+
+# Helper to convert Scenario data to MVS importable json
+def format_scenario_for_mvs(scenario_to_convert):
+    mvs_request_dto = convert_to_dto(scenario_to_convert)
+    dumped_data = json.loads(
+        json.dumps(mvs_request_dto.__dict__, default=lambda o: o.__dict__)
+    )
+
+    # format the constraints in MVS format directly, thus avoiding the need to maintain MVS-EPA
+    # parser in multi-vector-simulator package
+    constraint_dict = {}
+    for constraint in dumped_data["constraints"]:
+        constraint_dict[constraint["label"]] = constraint["value"]
+    dumped_data["constraints"] = constraint_dict
+
+    # Remove None values
+    return remove_empty_elements(dumped_data)
+
+
+def sensitivity_analysis_payload(
+    variable_parameter_name="",
+    variable_parameter_range="",
+    variable_parameter_ref_val="",
+    output_parameter_names=None,
+):
+    """format the parameters required to request a sensitivity analysis in a specific JSON"""
+    if output_parameter_names is None:
+        output_parameter_names = []
+    return {
+        "sensitivity_analysis_settings": {
+            "variable_parameter_name": variable_parameter_name,
+            "variable_parameter_range": variable_parameter_range,
+            "variable_parameter_ref_val": variable_parameter_ref_val,
+            "output_parameter_names": output_parameter_names,
+        }
+    }
+
+
+SA_RESPONSE_SCHEMA = {
+    "type": "object",
+    "required": ["server_info", "mvs_version", "id", "status", "results"],
+    "properties": {
+        "server_info": {"type": "string"},
+        "mvs_version": {"type": "string"},
+        "id": {"type": "string"},
+        "status": {"type": "string"},
+        "results": {
+            "type": "object",
+            "required": ["reference_simulation_id", "sensitivity_analysis_steps"],
+            "properties": {
+                "reference_simulation_id": {"type": "string"},
+                "sensitivity_analysis_steps": {
+                    "type": "array",
+                    "items": {"type": "object"},
+                },
+            },
+            "additionalProperties": False,
+        },
+        "ref_sim_id": {"type": "string"},
+        "sensitivity_analysis_ids": {"type": "array", "items": {"type": "string"}},
+    },
+    "additionalProperties": False,
+}
+
+
+# Used to proof the json objects stored as text in the db
+SA_OUPUT_NAMES_SCHEMA = {"type": "array", "items": {"type": "string"}}
+
+
+def sa_output_values_schema_generator(output_names):
+    return {
+        "type": "object",
+        "required": output_names,
+        "properties": {
+            output_name: {
+                "type": "object",
+                "required": ["value", "path"],
+                "properties": {
+                    "value": {
+                        "oneOf": [
+                            {"type": "null"},
+                            {
+                                "type": "array",
+                                "items": {
+                                    "anyOf": [{"type": "number"}, {"type": "null"}]
+                                },
+                            },
+                        ]
+                    },
+                    "path": {
+                        "oneOf": [
+                            {"type": "string"},
+                            {"type": "array", "items": {"type": "string"}},
+                        ]
+                    },
+                },
+            }
+            for output_name in output_names
+        },
+        "additionalProperties": False,
+    }

app/projects/models/__init__.py

@@ -0,0 +1,2 @@
+from .base_models import *
+from .simulation_models import Simulation, SensitivityAnalysis

app/projects/models.py app/projects/models/base_models.pyapp/projects/models.py renamed to app/projects/models/base_models.py

@@ -1,14 +1,13 @@
 import uuid
 import json
-from django.core.validators import MinValueValidator, MaxValueValidator
 from django.conf import settings
 from django.db import models
 from django.core.validators import MaxValueValidator, MinValueValidator
 from datetime import timedelta
 from django.forms.models import model_to_dict
 from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import ValidationError
-from .constants import (
+from projects.constants import (
     ASSET_CATEGORY,
     ASSET_TYPE,
     COUNTRY,
@@ -17,6 +16,7 @@
     FLOW_DIRECTION,
     MVS_TYPE,
     SIMULATION_STATUS,
+    PENDING,
     TRUE_FALSE_CHOICES,
     BOOL_CHOICES,
     USER_RATING,
@@ -174,6 +174,10 @@ def export(self):
         dm = model_to_dict(self, exclude=["id"])
         return dm
 
+    @property
+    def visible_fields(self):
+        return self.asset_fields.replace("[", "").replace("]", "").split(",")
+
 
 class TopologyNode(models.Model):
     name = models.CharField(max_length=60, null=False, blank=False)
@@ -279,6 +283,28 @@ def save(self, *args, **kwargs):
     def fields(self):
         return [f.name for f in self._meta.fields + self._meta.many_to_many]
 
+    @property
+    def visible_fields(self):
+        return self.asset_type.visible_fields
+
+    def has_parameter(self, param_name):
+        return param_name in self.visible_fields
+
+    def parameter_path(self, param_name):
+        # TODO for storage
+        if self.has_parameter(param_name):
+            # TODO if (unit, value) formatting, add "value" at the end
+            if self.asset_type.asset_category == "energy_provider":
+                asset_category = "energy_providers"
+            else:
+                asset_category = self.asset_type.asset_category
+            if param_name == "optimize_cap":
+                param_name = "optimize_capacity"
+            answer = (asset_category, self.name, param_name)
+        else:
+            answer = None
+        return answer
+
     @property
     def timestamps(self):
         return self.scenario.get_timestamps()
@@ -393,15 +419,17 @@ class ScenarioFile(models.Model):
     file = models.FileField(upload_to="tempFiles/", null=True, blank=True)
 
 
-class Simulation(models.Model):
+class AbstractSimulation(models.Model):
+
     start_date = models.DateTimeField(auto_now_add=True, null=False)
     end_date = models.DateTimeField(null=True)
     elapsed_seconds = models.FloatField(null=True)
     mvs_token = models.CharField(max_length=200, null=True)
-    status = models.CharField(max_length=20, choices=SIMULATION_STATUS, null=False)
-    scenario = models.OneToOneField(Scenario, on_delete=models.CASCADE, null=False)
-    user_rating = models.PositiveSmallIntegerField(
-        null=True, choices=USER_RATING, default=None
+    status = models.CharField(
+        max_length=20, choices=SIMULATION_STATUS, null=False, default=PENDING
     )
     results = models.TextField(null=True, max_length=30e6)
     errors = models.TextField(null=True)
+
+    class Meta:
+        abstract = True