Experiments implementation

.github/workflows/ci.yaml

@@ -22,6 +22,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
       - name: Install Poetry
         run: pip install poetry==2.1.1
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "pysatl_criterion"]
+	path = pysatl_criterion
+	url = https://github.com/PySATL/pysatl-criterion.git

README.md

@@ -84,6 +84,9 @@ PowerCalculationWorker - calculates goodness-of-fit test power
 
 ```bash
 git clone https://github.com/PySATL/pysatl-experiment
+cd pysatl-experiment
+git submodule add https://github.com/PySATL/pysatl-criterion.git pysatl_criterion
+git submodule update --init --recursive
 ```
 
 Install dependencies:

graph_norm_experiment.py

@@ -1,12 +1,12 @@
 import multiprocessing
 
 from numpy import random as rd
+
 from pysatl_criterion.statistics.normal import (
     GraphEdgesNumberNormalityGofStatistic,
     GraphMaxDegreeNormalityGofStatistic,
     KolmogorovSmirnovNormalityGofStatistic,
 )
-
 from stattest.experiment import Experiment
 from stattest.experiment.configuration.configuration import (
     AlternativeConfiguration,

pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
     "rich==13.9.4",
     "click>=8.2.1",
     "dacite==1.9.2",
-    "pysatl-criterion @ git+https://github.com/PySATL/pysatl-criterion.git"
+    "pysatl-criterion @ ./pysatl_criterion"
 ]
 
 [build-system]

stattest/cli/cli/cli.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+
+from click import group, version_option
+
+from stattest.cli.commands.build_and_run.build_and_run import build_and_run
+from stattest.cli.commands.configure.alternatives.alternatives import alternatives
+from stattest.cli.commands.configure.configure import configure
+from stattest.cli.commands.configure.criteria.criteria import criteria
+from stattest.cli.commands.configure.executor_type.executor_type import executor_type
+from stattest.cli.commands.configure.experiment_type.experiment_type import experiment_type
+from stattest.cli.commands.configure.generator_type.generator_type import generator_type
+from stattest.cli.commands.configure.hypothesis.hypothesis import hypothesis
+from stattest.cli.commands.configure.monte_carlo_count.monte_carlo_count import monte_carlo_count
+from stattest.cli.commands.configure.report_builder_type.report_builder_type import (
+    report_builder_type,
+)
+from stattest.cli.commands.configure.run_mode.run_mode import run_mode
+from stattest.cli.commands.configure.sample_sizes.sample_sizes import sample_sizes
+from stattest.cli.commands.configure.show.show import show
+from stattest.cli.commands.configure.significance_levels.significance_levels import (
+    significance_levels,
+)
+from stattest.cli.commands.configure.storage_connection.storage_connection import storage_connection
+from stattest.cli.commands.create.create import create
+
+
+@group()
+@version_option()
+def cli() -> None:
+    """
+    PySATL-Experiment CLI.
+    """
+
+    _create_experiments_dir()
+
+
+def _create_experiments_dir() -> None:
+    """
+    Create experiments directory.
+    """
+    # pysatl-experiment/.experiments
+    folder_path = Path(__file__).resolve().parents[3] / ".experiments"
+    folder_path.mkdir(parents=False, exist_ok=True)
+
+
+cli.add_command(create)
+cli.add_command(configure)
+cli.add_command(experiment_type)
+cli.add_command(show)
+cli.add_command(storage_connection)
+cli.add_command(run_mode)
+cli.add_command(hypothesis)
+cli.add_command(generator_type)
+cli.add_command(executor_type)
+cli.add_command(report_builder_type)
+cli.add_command(sample_sizes)
+cli.add_command(monte_carlo_count)
+cli.add_command(significance_levels)
+cli.add_command(criteria)
+cli.add_command(alternatives)
+cli.add_command(build_and_run)

stattest/cli/commands/build_and_run/build_and_run.py

@@ -0,0 +1,64 @@
+from click import BadParameter, argument, command
+
+from stattest.cli.commands.common.common import get_experiment_config, read_experiment_data
+from stattest.configuration.experiment_config.experiment_config import ExperimentConfig
+from stattest.configuration.experiment_data.experiment_data import ExperimentData
+from stattest.experiment_new.experiment.experiment import Experiment
+from stattest.experiment_new.experiment_steps.experiment_steps import ExperimentSteps
+from stattest.factory.critical_value.critical_value import CriticalValueExperimentFactory
+from stattest.factory.power.power import PowerExperimentFactory
+from stattest.factory.time_complexity.time_complexity import TimeComplexityExperimentFactory
+from stattest.validation.cli.commands.build_and_run.build_and_run import validate_build_and_run
+from stattest.validation.cli.commands.common.common import if_experiment_exists
+
+
+@command()
+@argument("name")
+def build_and_run(name: str) -> None:
+    """
+    Build and run an experiment with the given name.
+
+    :param name: name of the experiment.
+    """
+
+    experiment_exists = if_experiment_exists(name)
+    if not experiment_exists:
+        raise BadParameter(f"Experiment with name {name} does not exists.")
+
+    experiment_data_dict = read_experiment_data(name)
+    experiment_config = get_experiment_config(experiment_data_dict)
+
+    experiment_data = validate_build_and_run(experiment_data_dict)
+
+    experiment_type = experiment_config["experiment_type"]
+
+    experiment_steps = _build_experiment(experiment_data, experiment_type)
+
+    experiment = Experiment(experiment_steps)
+
+    experiment.run_experiment()
+
+
+def _build_experiment(
+    experiment_data: ExperimentData[ExperimentConfig], experiment_type: str
+) -> ExperimentSteps:
+    """
+    Build experiment.
+
+    :param experiment_data: experiment data.
+    :param experiment_type: experiment type.
+
+    :return: experiment steps.
+    """
+
+    experiment_type_to_factory = {
+        "power": PowerExperimentFactory,
+        "critical_value": CriticalValueExperimentFactory,
+        "time_complexity": TimeComplexityExperimentFactory,
+    }
+
+    experiment_factory = experiment_type_to_factory[experiment_type]
+
+    experiment_steps = experiment_factory(experiment_data).create_experiment_steps()
+
+    return experiment_steps

stattest/cli/commands/common/common.py

@@ -0,0 +1,216 @@
+import json
+from enum import Enum
+from pathlib import Path
+from typing import cast
+
+from click import ClickException, Context
+
+from pysatl_criterion.statistics import (
+    AbstractExponentialityGofStatistic,
+    AbstractNormalityGofStatistic,
+    AbstractWeibullGofStatistic,
+)
+from pysatl_criterion.statistics.goodness_of_fit import AbstractGoodnessOfFitStatistic
+
+
+def create_experiment_path(name: str) -> Path:
+    """
+    Create experiment path.
+
+    :param name: name of the experiment.
+
+    :return: path to the experiment.
+    """
+
+    # pysatl-experiment/.experiments
+    experiments_dir = Path(__file__).resolve().parents[4] / ".experiments"
+    experiment_file_name = f"{name}.json"
+
+    experiment_path = experiments_dir / experiment_file_name
+
+    return experiment_path
+
+
+def create_result_path(name: str) -> Path:
+    """
+    Create experiment result path.
+
+    :param name: name of the experiment.
+
+    :return: path to the experiment result.
+    """
+
+    # pysatl-experiment/.results
+    experiments_dir = Path(__file__).resolve().parents[4] / ".results"
+    experiment_file_name = f"{name}.json"
+
+    experiment_path = experiments_dir / experiment_file_name
+
+    return experiment_path
+
+
+def save_experiment_data(experiment_name: str, experiment_data: dict) -> None:
+    """
+    Save experiment data.
+
+    :param experiment_name: path to the experiment.
+    :param experiment_data: experiment data.
+    """
+
+    experiment_path = create_experiment_path(experiment_name)
+    with Path.open(experiment_path, "w") as f:
+        json.dump(experiment_data, f)
+
+
+def read_experiment_data(experiment_name: str) -> dict:
+    """
+    Read experiment data.
+
+    :param experiment_name: path to the experiment.
+
+    :return: experiment data.
+    """
+
+    experiment_path = create_experiment_path(experiment_name)
+    with Path.open(experiment_path) as f:
+        data = json.load(f)
+
+    return dict(data)
+
+
+def list_possible_parameter_values(param_type: type[Enum]) -> str:
+    """
+    List possible parameter values.
+
+    :param param_type: parameter type.
+
+    :return: possible parameter values.
+    """
+
+    param_type_values = [item.value for item in param_type]
+    param_type_values_str = ", ".join(param_type_values)
+
+    return param_type_values_str
+
+
+def get_statistics_short_codes_for_hypothesis(hypothesis: str) -> list[str]:
+    """
+    Get statistics codes for hypothesis.
+
+    :param hypothesis: hypothesis.
+
+    :return: statistics codes for hypothesis.
+    """
+
+    hypothesis_to_base_class = {
+        "exponential": AbstractExponentialityGofStatistic,
+        "normal": AbstractNormalityGofStatistic,
+        "weibull": AbstractWeibullGofStatistic,
+    }
+
+    base_class = hypothesis_to_base_class[hypothesis]
+
+    valid_criteria_types = cast(
+        list[type[AbstractGoodnessOfFitStatistic]],
+        base_class.__subclasses__(),
+    )
+    valid_criteria_codes = [cls.code().split("_")[0] for cls in valid_criteria_types]
+
+    return valid_criteria_codes
+
+
+def get_experiment_data(ctx: Context) -> dict:
+    """
+    Get experiment data.
+
+    :param ctx: context.
+
+    :return: experiment data.
+    """
+
+    experiment_data = ctx.obj.get("experiment_data")
+    if experiment_data is None:
+        raise ClickException("Experiment is not created.")
+
+    return experiment_data
+
+
+def get_experiment_name(experiment_data: dict) -> str:
+    """
+    Get experiment name.
+
+    :param experiment_data: experiment data.
+
+    :return: experiment name.
+    """
+
+    experiment_name = experiment_data.get("name")
+    if experiment_name is None:
+        raise ClickException("Experiment name is not configured.")
+
+    return experiment_name
+
+
+def get_experiment_config(experiment_data: dict) -> dict:
+    """
+    Get experiment config.
+
+    :param experiment_data: experiment data.
+
+    :return: experiment config.
+    """
+
+    experiment_config = experiment_data.get("config")
+    if experiment_config is None:
+        raise ClickException("Experiment config is not configured.")
+
+    return experiment_config
+
+
+def get_experiment_name_and_config(ctx: Context) -> tuple[str, dict]:
+    """
+    Get experiment name and config from context.
+
+    :param ctx: context.
+
+    :return: experiment name and config.
+    """
+
+    experiment_data = get_experiment_data(ctx)
+    experiment_name = get_experiment_name(experiment_data)
+    experiment_config = get_experiment_config(experiment_data)
+
+    return experiment_name, experiment_config
+
+
+def save_experiment_config(ctx: Context, experiment_name: str, experiment_config: dict) -> None:
+    """
+    Save experiment config.
+
+    :param ctx: context.
+    :param experiment_name: experiment name.
+    :param experiment_config: experiment config.
+
+    :return: experiment config.
+    """
+
+    experiment_data = get_experiment_data(ctx)
+    experiment_data["config"] = experiment_config
+    save_experiment_data(experiment_name, experiment_data)
+
+
+def criteria_from_codes(codes: list[str]) -> list[dict]:
+    """
+    Convert criteria codes to criteria
+
+    :param codes: criteria codes.
+
+    :return: criteria.
+    """
+
+    criteria_data = []
+    for code in codes:
+        criterion = {"criterion_code": code, "parameters": []}
+        criteria_data.append(criterion)
+
+    return criteria_data