Report generators (#65)

* feat: add CriticalValueReportBuilder implementation.

* feat: add TimeComplexityReportBuilder implementation.

* feat: add template with chart for time-complexity reports

* feat: fully implemented PowerReportBuilder, TimeComplexityReportBuilder and CriticalValueReportBuilder.

* feat: add "report-mode" command to experiment configuration.

* test: add unit-tests.

* fix: fixed errors

* fix: fixed errors

* fix: auto-format code with ruff formatter

* fix: resolve mypy type checking errors

* fix: remove unused experiment_storage parameter.

* refactor: add report examples.

Author: quepum

pyproject.toml

@@ -14,6 +14,8 @@ dependencies = [
     "matplotlib>=3.8.0",
     "tqdm>=4.66.2",
     "fpdf2>=2.7.8",
+    "jinja2>=3.1.5",
+    "xhtml2pdf>=0.2.17",
     "pandas>=2.2.1",
     "sqlalchemy>=2.0.36",
     "python-rapidjson==1.20",

report_examples/critical_values_report.pdf

@@ -14,6 +14,7 @@
 from stattest.cli.commands.configure.report_builder_type.report_builder_type import (
     report_builder_type,
 )
+from stattest.cli.commands.configure.report_mode.report_mode import report_mode
 from stattest.cli.commands.configure.run_mode.run_mode import run_mode
 from stattest.cli.commands.configure.sample_sizes.sample_sizes import sample_sizes
 from stattest.cli.commands.configure.show.show import show
@@ -58,4 +59,5 @@ def _create_experiments_dir() -> None:
 cli.add_command(significance_levels)
 cli.add_command(criteria)
 cli.add_command(alternatives)
+cli.add_command(report_mode)
 cli.add_command(build_and_run)

report_examples/time_complexity_report.pdf

@@ -0,0 +1,31 @@
+from click import Context, argument, echo, pass_context
+
+from stattest.cli.commands.common.common import (
+    get_experiment_name_and_config,
+    save_experiment_config,
+)
+from stattest.cli.commands.configure.configure import configure
+from stattest.validation.cli.commands.configure.report_mode.report_mode import validate_report_mode
+
+
+@configure.command()
+@argument("mode")
+@pass_context
+def report_mode(ctx: Context, mode: str) -> None:
+    """
+    Configure experiment report mode.
+
+    :param ctx: context.
+    :param mode: report mode.
+    """
+
+    validate_report_mode(mode)
+    mode_lower = mode.lower()
+
+    experiment_name, experiment_config = get_experiment_name_and_config(ctx)
+
+    experiment_config["report_mode"] = mode_lower
+
+    save_experiment_config(ctx, experiment_name, experiment_config)
+
+    echo(f"Report mode of the experiment '{experiment_name}' is set to '{mode_lower}'.")

stattest/cli/cli/cli.py

@@ -24,6 +24,7 @@ def create(name: str) -> None:
             "executor_type": "standard",
             "report_builder_type": "standard",
             "run_mode": "reuse",
+            "report_mode": "with-chart",
         },
     }
 

stattest/cli/commands/configure/report_mode/report_mode.py

@@ -3,6 +3,7 @@
 from stattest.configuration.model.criterion.criterion import Criterion
 from stattest.configuration.model.experiment_type.experiment_type import ExperimentType
 from stattest.configuration.model.hypothesis.hypothesis import Hypothesis
+from stattest.configuration.model.report_mode.report_mode import ReportMode
 from stattest.configuration.model.run_mode.run_mode import RunMode
 from stattest.configuration.model.step_type.step_type import StepType
 
@@ -23,3 +24,4 @@ class ExperimentConfig:
     sample_sizes: list[int]
     monte_carlo_count: int
     criteria: list[Criterion]
+    report_mode: ReportMode

stattest/cli/commands/create/create.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class ReportMode(Enum):
+    """
+    Report mode (build report with chart or without chart).
+    """
+
+    WITH_CHART = "with-chart"
+    WITHOUT_CHART = "without-chart"

stattest/configuration/experiment_config/experiment_config.py

@@ -1,10 +1,12 @@
 from pathlib import Path
 
+from pysatl_criterion.cv_calculator.cv_calculator.cv_calculator import CVCalculator
 from pysatl_criterion.persistence.model.limit_distribution.limit_distribution import (
     ILimitDistributionStorage,
     LimitDistributionQuery,
 )
 from stattest.configuration.criteria_config.criteria_config import CriterionConfig
+from stattest.configuration.model.report_mode.report_mode import ReportMode
 from stattest.report.critical_value.critical_value import CriticalValueReportBuilder
 
 
@@ -21,36 +23,41 @@ def __init__(
         monte_carlo_count: int,
         result_storage: ILimitDistributionStorage,
         results_path: Path,
+        with_chart: ReportMode,
     ):
         self.criteria_config = criteria_config
         self.significance_levels = significance_levels
-        self.sizes = sample_sizes
+        self.sizes = sorted(sample_sizes)
         self.monte_carlo_count = monte_carlo_count
         self.result_storage = result_storage
         self.results_path = results_path
+        self.with_chart = with_chart
 
     def run(self) -> None:
         """
         Run standard critical value report building step.
         """
 
+        cv_values = []
         for criterion_config in self.criteria_config:
             for sample_size in self.sizes:
-                limit_distribution = self._get_limit_distribution_from_storage(
-                    storage=self.result_storage,
-                    criterion_config=criterion_config,
-                    sample_size=sample_size,
-                    monte_carlo_count=self.monte_carlo_count,
-                )
+                cv_calculator = CVCalculator(self.result_storage)
                 for significance_level in self.significance_levels:
-                    report_builder = CriticalValueReportBuilder(
-                        criterion_config=criterion_config,
-                        sample_size=sample_size,
-                        significance_level=significance_level,
-                        limit_distribution=limit_distribution,
-                        results_path=self.results_path,
+                    cv_value = cv_calculator.calculate_critical_value(
+                        criterion_config.criterion_code, sample_size, significance_level
                     )
-                    report_builder.build()
+
+                    cv_values.append(cv_value)
+
+        report_builder = CriticalValueReportBuilder(
+            criteria_config=self.criteria_config,
+            sample_sizes=self.sizes,
+            significance_levels=self.significance_levels,
+            cv_values=cv_values,
+            results_path=self.results_path,
+            with_chart=self.with_chart,
+        )
+        report_builder.build()
 
     def _get_limit_distribution_from_storage(
         self,
@@ -59,16 +66,14 @@ def _get_limit_distribution_from_storage(
         sample_size: int,
         monte_carlo_count: int,
     ) -> list[float]:
-        """
-        Get limit distribution from storage.
+        """Get limit distribution from storage.
 
         :param storage: storage.
         :param criterion_config: criterion configuration.
         :param sample_size: sample size.
         :param monte_carlo_count: monte carlo count.
 
-        :return: limit distribution.
-        """
+        :return: limit distribution."""
 
         query = LimitDistributionQuery(
             criterion_code=criterion_config.criterion_code,

stattest/configuration/model/report_mode/report_mode.py

@@ -2,6 +2,7 @@
 
 from stattest.configuration.criteria_config.criteria_config import CriterionConfig
 from stattest.configuration.model.alternative.alternative import Alternative
+from stattest.configuration.model.report_mode.report_mode import ReportMode
 from stattest.persistence.model.power.power import IPowerStorage, PowerQuery
 from stattest.report.power.power import PowerReportBuilder
 
@@ -20,40 +21,57 @@ def __init__(
         monte_carlo_count: int,
         result_storage: IPowerStorage,
         results_path: Path,
+        with_chart: ReportMode,
     ):
         self.criteria_config = criteria_config
         self.significance_levels = significance_levels
         self.alternatives = alternatives
-        self.sizes = sample_sizes
+        self.sizes = sorted(sample_sizes)
         self.monte_carlo_count = monte_carlo_count
         self.result_storage = result_storage
         self.results_path = results_path
+        self.with_chart = with_chart
 
     def run(self) -> None:
         """
         Run standard power report building step.
         """
 
+        power_data = self._collect_statistics()
+
+        builder = PowerReportBuilder(
+            criteria_config=self.criteria_config,
+            sample_sizes=self.sizes,
+            alternatives=self.alternatives,
+            significance_levels=self.significance_levels,
+            power_result=power_data,
+            results_path=self.results_path,
+            with_chart=self.with_chart,
+        )
+        builder.build()
+
+    def _collect_statistics(self) -> dict[str, dict[tuple[str, float], dict[int, list[bool]]]]:
+        """
+        Collect power results.
+
+        :return: {criterion_code -> (alt_name, alpha) -> {size: [bool]}}
+        """
+
+        power_data: dict[str, dict[tuple[str, float], dict[int, list[bool]]]] = {}
+
         for criterion_config in self.criteria_config:
             for alternative in self.alternatives:
-                for sample_size in self.sizes:
-                    for significance_level in self.significance_levels:
-                        results_criteria = self._get_power_result_from_storage(
-                            criterion_config=criterion_config,
-                            sample_size=sample_size,
-                            alternative=alternative,
-                            significance_level=significance_level,
-                        )
-
-                        report_builder = PowerReportBuilder(
+                for significance_level in self.significance_levels:
+                    for sample_size in self.sizes:
+                        result = self._get_power_result_from_storage(
                             criterion_config=criterion_config,
                             sample_size=sample_size,
                             alternative=alternative,
                             significance_level=significance_level,
-                            power_result=results_criteria,
-                            results_path=self.results_path,
                         )
-                        report_builder.build()
+                        key = (alternative.generator_name, significance_level)
+                        power_data[criterion_config.criterion_code][key][sample_size] = result
+        return power_data
 
     def _get_power_result_from_storage(
         self,