Merge branch 'master' into 380-support-figure-and-table-references

vincentchabannes · web-flow · commit 369cf6c85afb · 2026-03-18T11:50:35.000+01:00
diff --git a/examples/overview.json b/examples/overview.json
diff --git a/examples/website_config.json b/examples/website_config.json
@@ -10,11 +10,12 @@
     "template_defaults":{
         "repositories":{
             "template": "${TEMPLATE_DIR}/overview.adoc.j2",
-            "data":{"filepath":"./reports/overview.json","prefix":"plots"}
+            "data":{"filepath":"./examples/overview.json","prefix":"plots"}
         },
         "components":{
             "all": {
-                "template": "${TEMPLATE_DIR}/overview.adoc.j2"
+                "template": "${TEMPLATE_DIR}/overview.adoc.j2",
+                "data":{"filepath":"./examples/overview.json","prefix":"plots"}
             },
             "machines": {
                 "template": "${TEMPLATE_DIR}/machineOverview.adoc.j2",
@@ -50,37 +51,36 @@
             "parallelSum": {"title": "parallelSum", "description": "Compute the sum of an array using MPI","card_image":"fa-solid fa-plus" }
         },
         "use_cases":{
-            "multiplication": {"title":"multiplication", "description":"Multiplication between different objects", "card_image":"fa-solid fa-times"}
+            "Fibonacci": { "title": "Fibonacci", "description": "n-th Fibonacci number", "card_image": "fa-solid fa-fan" },
+            "Multiplication": { "title": "Multiplication", "description": "Multiplication between different objects", "card_image": "fa-solid fa-times" },
+            "parallel_sum": { "title": "Parallel_Sum", "description": "Compute the sum of an array using MPI", "card_image": "fa-solid fa-plus" },
+            "sorting": { "title": "Sorting", "description": "Sorting algorithms", "card_image": "fa-solid fa-arrow-up-wide-short" }
         }
-
     },
     "component_map": {
-        "component_order": [ "machines", "applications", "use_cases" ],
-        "mapping": {
-            "gaya": {
-                "sortingApp": {
-                    "path": "./reports/sortingApp/sorting/gaya",
-                    "template_info": { "application": "sortingApp", "machine": "gaya" }
-                },
-                "fibonacci": {
-                    "path": "./reports/fibonacci/Fibonacci/gaya",
-                    "template_info": { "application": "fibonacci", "machine": "gaya" }
-                },
-                "matrixvector": {
-                    "multiplication":{
-                        "path": "./reports/matrixvector/Multiplication/gaya",
-                        "template_info": { "application": "matrixvector", "machine": "gaya", "use_case":"multiplication"}
-                    }
-                },
-                "parallelSum": {
-                    "path": "./reports/parallelSum/parallel_sum/gaya",
-                    "template_info": { "application": "parallelSum", "machine": "gaya" }
+        "default": {
+            "sortingApp": {
+                "sorting":{
+                    "path": "./reports/sortingApp/sorting/default",
+                    "template_info": { "application": "sortingApp", "machine": "default" }
                 }
             },
-            "discoverer":{
-                "fibonacci": {
-                    "path": "./reports/fibonacci/Fibonacci/discoverer",
-                    "template_info": { "application": "fibonacci", "machine": "discoverer" }
+            "fibonacci": {
+                "Fibonacci":{
+                    "path": "./reports/fibonacci/Fibonacci/default",
+                    "template_info": { "application": "fibonacci", "machine": "default" }
+                }
+            },
+            "matrixvector": {
+                "Multiplication":{
+                    "path": "./reports/matrixvector/Multiplication/default",
+                    "template_info": { "application": "matrixvector", "machine": "default", "use_case":"multiplication"}
+                }
+            },
+            "parallelSum": {
+                "parallel_sum":{
+                    "path": "./reports/parallelSum/parallel_sum/default",
+                    "template_info": { "application": "parallelSum", "machine": "default" }
                 }
             }
         }
diff --git a/src/feelpp/benchmarking/dashboardRenderer/core/treeBuilder.py b/src/feelpp/benchmarking/dashboardRenderer/core/treeBuilder.py
@@ -134,7 +134,7 @@ def render( self, base_path:str ) -> None:
         super().render(base_path,None,renderLeaves=False)
         self.leaf_repository.render(base_path)
 
-    def patchTemplateInfo( self, patches:list[str], targets:str, prefix:str, save:bool = False ):
+    def patchTemplateInfo( self, patches:List[str], targets:str, prefix:str, save:bool = False ):
         """
         Applies template patches to specific leaf components based on the given target criteria.
 
diff --git a/src/feelpp/benchmarking/dashboardRenderer/repository/leafLoader.py b/src/feelpp/benchmarking/dashboardRenderer/repository/leafLoader.py
@@ -59,7 +59,7 @@ def __init__(self, location:str, template_info: TemplateInfo) -> None:
         if not os.path.exists( location ):
             warnings.warn(f"{location} does not contain any files")
 
-    def load(self,repository:Repository, parent_ids:list[str]) -> None:
+    def load(self,repository:Repository, parent_ids:List[str]) -> None:
         """
         Load local leaf components (in the filesystem) into the repository.
         It iterates over subdirectories in the `self.location`, treating each as a separate leaf component, and uses the `ViewFactory` to create a view for it.
diff --git a/src/feelpp/benchmarking/dashboardRenderer/schemas/dashboardSchema.py b/src/feelpp/benchmarking/dashboardRenderer/schemas/dashboardSchema.py
@@ -27,14 +27,27 @@ class TemplateDataFile(BaseModel):
     prefix: Optional[str] = ""
     format: str = None
     action: Optional[str] = "input"
+    platform: Optional[str] = "local"
     filepath: str
 
     @model_validator(mode="after")
     def checkFormat(self):
-        if self.format is None:
-            self.format = self.filepath.split(".")[-1]
+        if self.platform == "local":
+            if self.format is None:
+                self.format = self.filepath.split(".")[-1]
+        else:
+            if self.format is None:
+                raise ValueError("Format must be specified for remote data")
         return self
 
+    @field_validator("platform",mode="after")
+    @classmethod
+    def checkPlatform(cls,v):
+        if v not in ["local","girder"]:
+            raise NotImplementedError(f"Data management platform not supported : {v}")
+        return v
+
+
 class TemplateInfo(BaseModel):
     template:Optional[str] = None
     data: Union[
@@ -124,11 +137,11 @@ def castNodeTemplateInfo(cls,v):
 
 
 class DashboardSchema(BaseModel):
-    dashboard_metadata:Optional[Union[dict[str,str],TemplateInfo]] = TemplateInfo(data={})
+    dashboard_metadata:Optional[Union[Dict[str,str],TemplateInfo]] = TemplateInfo(data={})
     component_map: Union[ComponentMap,Dict]
-    components: Dict[str,Dict[str, Union[dict[str,str],TemplateInfo]]]
+    components: Dict[str,Dict[str, Union[Dict[str,str],TemplateInfo]]]
     views : Optional[Dict[str,Union[Dict,str]]] = None
-    repositories : Optional[Dict[str,Union[dict[str,str],TemplateInfo]]] = None
+    repositories : Optional[Dict[str,Union[Dict[str,str],TemplateInfo]]] = None
     template_defaults: Optional[TemplateDefaults] = TemplateDefaults()
 
     @model_validator(mode="after")
@@ -230,5 +243,5 @@ def setTemplateDefaults(self):
 
                 if not component_data.template:
                     component_data.template = template
-
+                component_data.data = component_data.data[::-1] #Put defaults at the start so actual values overwrite
         return self
diff --git a/src/feelpp/benchmarking/dashboardRenderer/views/templateDataHandler.py b/src/feelpp/benchmarking/dashboardRenderer/views/templateDataHandler.py
@@ -1,7 +1,7 @@
-
+from feelpp.benchmarking.dashboardRenderer.handlers.girder import GirderHandler
 from feelpp.benchmarking.dashboardRenderer.schemas.dashboardSchema import TemplateInfo,TemplateDataFile
 from feelpp.benchmarking.json_report.renderer import JsonReportController
-import json, os, warnings
+import json, os, warnings, tempfile
 from typing import Union, Dict,Any
 
 class DataHandler:
@@ -49,7 +49,15 @@ def extractData( self, data: TemplateDataFile, partials:Dict[str,Any] = {}, extr
             UserWarning: If a file specified by `filepath` does not exist and the action is "input".
             JSONDecodeError: If the file content is not valid JSON.
         """
-        filepath = os.path.join( self.template_data_dir, data.filepath ) if self.template_data_dir else data.filepath
+        if data.platform != "local":
+            tmpdir = tempfile.mkdtemp()
+            if data.platform == "girder":
+                download_handler = GirderHandler( tmpdir )
+                download_handler.downloadFile(data.filepath,name=data.filepath)
+                local_filepath = os.path.join(tmpdir,data.filepath)
+            filepath = local_filepath
+        else :
+            filepath = os.path.join( self.template_data_dir, data.filepath ) if self.template_data_dir else data.filepath
         if data.action == "input":
             if not os.path.exists( filepath ):
                 warnings.warn(f"{filepath} does not exist. Skipping")
@@ -60,6 +68,9 @@ def extractData( self, data: TemplateDataFile, partials:Dict[str,Any] = {}, extr
                     if not content:
                         content = "{}"
                     template_data = json.loads(content)
+
+            if data.platform != "local":
+                os.remove(local_filepath)
             if data.prefix:
                 return {data.prefix: template_data}
             return template_data
diff --git a/src/feelpp/benchmarking/json_report/figures/schemas/plot.py b/src/feelpp/benchmarking/json_report/figures/schemas/plot.py
@@ -4,7 +4,7 @@
 class PlotAxis(BaseModel):
     parameter: str
     label:Optional[str] = None
-    filter:Optional[Union[str,list[str],dict[str,str],list[dict[str,str]]]] = []
+    filter:Optional[Union[str,List[str],Dict[str,str],List[Dict[str,str]]]] = []
 
     @field_validator("filter",mode="after")
     @classmethod
diff --git a/src/feelpp/benchmarking/json_report/figures/templates/tikz/groupedBarChart.tex.j2 b/src/feelpp/benchmarking/json_report/figures/templates/tikz/groupedBarChart.tex.j2
@@ -23,9 +23,9 @@
       ybar,
       legend style={at={(0.5,-0.1)},anchor=north}
     ]
-    {% for var,name in zip(variables,names) %}
+    {% for var in variables %}
       \addplot table [x expr=\coordindex, y={{ var }}] {{'{#2}'}} ;
-      \addlegendentry{ {{name}} }
+      \addlegendentry{ {{var}} }
     {% endfor %}
 
     \end{axis}
diff --git a/src/feelpp/benchmarking/json_report/figures/templates/tikz/scatterChart.tex.j2 b/src/feelpp/benchmarking/json_report/figures/templates/tikz/scatterChart.tex.j2
@@ -21,10 +21,10 @@
       xticklabels from table={{'{#2}'}}{{'{'}}{{xaxis.parameter}}{{'}'}},
       cycle list name=color list, legend style={at={(0.5,-0.1)},anchor=north}
     ]
-    {% for var,name in zip(variables,names) %}
+    {% for var in variables %}
       {% if var not in fill_lines %}
       \addplot table [x expr=\coordindex, y={{ var }}] {{'{#2}'}} ;
-      \addlegendentry{ {{name}} }
+      \addlegendentry{ {{var}} }
       {% endif %}
     {% endfor %}
 
diff --git a/src/feelpp/benchmarking/json_report/figures/templates/tikz/stackedBarChart.tex.j2 b/src/feelpp/benchmarking/json_report/figures/templates/tikz/stackedBarChart.tex.j2
@@ -41,10 +41,10 @@
       {% if i > 1 %}
       \resetstackedplots
       {% endif %}
-      {% for var,name,color in zip(variables,names, colors) %}
+      {% for var,color in zip(variables, colors) %}
       \addplot+[ybar, bar width=0.2,point meta=y,draw=black,fill={{color}}{% if i > 1 %}, forget plot{% endif %} ] table [x expr=\coordindex+0.25*{{i}}, y={{ var }}] {\data{{i | inttouniquestr}}} ;
       {% if i == 1%}
-      \addlegendentry{ {{name}} }
+      \addlegendentry{ {{var}} }
       {% endif %}
       {% endfor %}
     {% endfor %}
diff --git a/src/feelpp/benchmarking/json_report/figures/tikzFigures.py b/src/feelpp/benchmarking/json_report/figures/tikzFigures.py
@@ -47,6 +47,7 @@ def createMultiindexFigure(self, df, **args):
             xaxis = self.config.xaxis,
             yaxis = self.config.yaxis,
             caption = self.config.title,
+            variables = df.columns.to_list(),
             secondary_axis = self.config.secondary_axis,
             anim_dimension_values = [str(dim) for dim in anim_dim_values],
             csv_filenames = [f"{dim}.csv" for dim in anim_dim_values],
@@ -64,6 +65,7 @@ def createSimpleFigure(self, df, **args):
             xaxis = self.config.xaxis,
             yaxis = self.config.yaxis,
             caption = self.config.title,
+            variables = df.columns.to_list(),
             csv_filenames = [f"{self.config.title}.csv"],
             **args
         )
diff --git a/src/feelpp/benchmarking/json_report/figures/transformationFactory.py b/src/feelpp/benchmarking/json_report/figures/transformationFactory.py
@@ -1,4 +1,5 @@
 import pandas as pd
+import numpy as np
 
 
 class TransformationStrategy:
@@ -153,8 +154,26 @@ def calculate(self,df):
             return pd.DataFrame(columns=["optimal","half-optimal"])
 
         if isinstance(pivot.index, pd.MultiIndex):
-            pivot = pivot.xs(pivot.index.get_level_values(self.dimensions["xaxis"].parameter).min(),level=self.dimensions["xaxis"].parameter,axis=0) / pivot
-            pivot["optimal"] = pivot.index.get_level_values(self.dimensions["xaxis"].parameter) / pivot.index.get_level_values(self.dimensions["xaxis"].parameter).min()
+            lvl = self.dimensions["xaxis"].parameter
+            outer = [n for n in pivot.index.names if n != lvl]
+            group_level = outer[0] if len(outer) == 1 else outer
+
+            results = []
+
+            for key, subdf in pivot.groupby(level=group_level):
+                vals = subdf.index.get_level_values(lvl)
+                ref = np.nanmin(vals)
+
+                base = subdf.xs(ref, level=lvl, axis=0)
+                speed = base / subdf
+
+                speed["optimal"] = vals / ref
+                speed["half-optimal"] = (speed["optimal"] - 1) / 2 + 1
+
+                results.append(speed)
+
+            pivot = pd.concat(results)
+
         else:
             pivot = pivot.loc[pivot.index.min(),:] / pivot
             pivot["optimal"] = pivot.index / pivot.index.min()
diff --git a/src/feelpp/benchmarking/json_report/schemas/jsonReport.py b/src/feelpp/benchmarking/json_report/schemas/jsonReport.py
@@ -121,7 +121,7 @@ def coerceListInput( cls, values ):
         else:
             raise TypeError(f"Expected dict or list at root, got {type(values)}")
 
-    def flattenContent(self, contents = None) -> list[Node]:
+    def flattenContent(self, contents = None) -> List[Node]:
         flattened = []
         if contents is None:
             contents = self.contents
diff --git a/src/feelpp/benchmarking/report/templates/overview.adoc.j2 b/src/feelpp/benchmarking/report/templates/overview.adoc.j2
@@ -12,7 +12,7 @@
     {% for key,key_plots in plots_by_child_repo.items() %}
 
         {%if self_id in key.split("|") %}
-            {% with report_data = {"aggregated_data":aggregated.get("data")}, report = ValidateJsonReport(key_plots)%}
+            {% with report_data = {"aggregated_data":{"aggregated_data":aggregated.get("data")}}, report = ValidateJsonReport(key_plots)%}
                 {% include "json2adoc_report.adoc.j2" %}
             {% endwith %}
 
