Parse the sensitivity analysis results to the database

Make sure of the format of the json to avoid possible html injection

Author: Bachibouzouk

app/projects/helpers.py

@@ -56,10 +56,26 @@ def sensitivity_analysis_payload(
     }
 
 
-SA_MVS_TOKEN_SCHEMA = {
+SA_RESPONSE_SCHEMA = {
     "type": "object",
-    "required": ["ref_sim_id", "sensitivity_analysis_ids"],
+    "required": ["server_info", "mvs_version", "id", "status", "results"],
     "properties": {
+        "server_info": {"type": "string"},
+        "mvs_version": {"type": "string"},
+        "id": {"type": "string"},
+        "status": {"type": "string"},
+        "results": {
+            "type": "object",
+            "required": ["reference_simulation_id", "sensitivity_analysis_steps"],
+            "properties": {
+                "reference_simulation_id": {"type": "string"},
+                "sensitivity_analysis_steps": {
+                    "type": "array",
+                    "items": {"type": "object"},
+                },
+            },
+            "additionalProperties": False,
+        },
         "ref_sim_id": {"type": "string"},
         "sensitivity_analysis_ids": {"type": "array", "items": {"type": "string"}},
     },
@@ -83,7 +99,12 @@ def sa_output_values_schema_generator(output_names):
                     "value": {
                         "oneOf": [
                             {"type": "null"},
-                            {"type": "array", "items": {"type": "number"}},
+                            {
+                                "type": "array",
+                                "items": {
+                                    "anyOf": [{"type": "number"}, {"type": "null"}]
+                                },
+                            },
                         ]
                     },
                     "path": {

app/projects/models/simulation_models.py

@@ -5,17 +5,20 @@
 import jsonschema
 import numpy as np
 import logging
+
+logger = logging.getLogger(__name__)
 from django.db import models
 from django.core.validators import MaxValueValidator, MinValueValidator
 
 from django.utils.translation import gettext_lazy as _
+from datetime import datetime
 
-from projects.constants import USER_RATING
+from projects.constants import USER_RATING, DONE, PENDING, ERROR
 from projects.helpers import (
     sensitivity_analysis_payload,
     SA_OUPUT_NAMES_SCHEMA,
     sa_output_values_schema_generator,
-    SA_MVS_TOKEN_SCHEMA,
+    SA_RESPONSE_SCHEMA,
     format_scenario_for_mvs,
 )
 
@@ -65,12 +68,6 @@ def set_reference_scenario(self, scenario):
         self.scenario = scenario
         self.save()
 
-    def collect_simulations_tokens(self, server_response):
-        try:
-            jsonschema.validate(server_response, SA_MVS_TOKEN_SCHEMA)
-        except jsonschema.exceptions.ValidationError:
-            answer = {}
-
     @property
     def variable_range(self):
         return np.arange(
@@ -93,17 +90,59 @@ def output_names(self):
     @property
     def output_values(self):
         try:
-            answer = json.loads(self.output_parameters_values)
-            try:
-                jsonschema.validate(
-                    answer, sa_output_values_schema_generator(self.output_names)
-                )
-            except jsonschema.exceptions.ValidationError:
-                answer = {}
+            out_values = json.loads(self.output_parameters_values)
+            answer = {}
+            for in_value, sa_step in zip(self.variable_range, out_values):
+                try:
+                    jsonschema.validate(
+                        sa_step, sa_output_values_schema_generator(self.output_names)
+                    )
+                    answer[in_value] = sa_step
+                except jsonschema.exceptions.ValidationError:
+                    answer[in_value] = None
         except json.decoder.JSONDecodeError:
             answer = {}
         return answer
 
+    def parse_server_response(self, sa_results):
+        try:
+            # make sure the response is formatted as expected
+            jsonschema.validate(sa_results, SA_RESPONSE_SCHEMA)
+            self.status = sa_results["status"]
+            self.errors = (
+                json.dumps(sa_results["results"][ERROR])
+                if self.status == ERROR
+                else None
+            )
+            if self.status == DONE:
+                sa_steps = sa_results["results"]["sensitivity_analysis_steps"]
+                sa_steps_processed = []
+                # make sure that each step is formatted as expected
+                for step_idx, sa_step in enumerate(sa_steps):
+                    try:
+                        jsonschema.validate(
+                            sa_step,
+                            sa_output_values_schema_generator(self.output_names),
+                        )
+                        sa_steps_processed.append(sa_step)
+                    except jsonschema.exceptions.ValidationError as e:
+                        logger.error(
+                            f"Could not parse the results of the sensitivity analysis {self.id} for step {step_idx}"
+                        )
+                        sa_steps_processed.append(None)
+                self.output_parameters_values = json.dumps(sa_steps_processed)
+
+        except jsonschema.exceptions.ValidationError as e:
+            self.status = ERROR
+            self.output_parameters_values = ""
+            self.errors = str(e)
+
+        self.elapsed_seconds = (datetime.now() - self.start_date).seconds
+        self.end_date = (
+            datetime.now() if sa_results["status"] in [ERROR, DONE] else None
+        )
+        self.save()
+
     @property
     def payload(self):
         return sensitivity_analysis_payload(

app/projects/requests.py

@@ -108,28 +108,12 @@ def fetch_mvs_simulation_results(simulation):
 def fetch_mvs_sa_results(simulation):
     if simulation.status == PENDING:
         response = mvs_sa_check_status(token=simulation.mvs_token)
-        try:
-            simulation.status = response["status"]
-            simulation.errors = (
-                json.dumps(response["results"][ERROR])
-                if simulation.status == ERROR
-                else None
-            )
-            # TODO here fetch the results of the reference scenario in a separate Simulation instance
-            # and parse the
-            simulation.results = (
-                response["results"] if simulation.status == DONE else None
-            )
+
+        simulation.parse_server_response(response)
+
+        if simulation.status == DONE:
             logger.info(f"The simulation {simulation.id} is finished")
-        except:
-            simulation.status = ERROR
-            simulation.results = None
 
-        simulation.elapsed_seconds = (datetime.now() - simulation.start_date).seconds
-        simulation.end_date = (
-            datetime.now() if response["status"] in [ERROR, DONE] else None
-        )
-        simulation.save()
     return simulation.status != PENDING
 
 

app/projects/views.py

@@ -910,9 +910,7 @@ def sensitivity_analysis_create(request, scen_id, sa_id=None, step_id=5):
         else:
             sa_item = None
             sa_status = None
-            sa_form = SensitivityAnalysisForm(
-                scen_id=scen_id,
-            )
+            sa_form = SensitivityAnalysisForm(scen_id=scen_id)
 
         answer = render(
             request,