Merge branch 'develop' into 330_truncated

dweindl · web-flow · commit 2529bf99028e · 2025-02-11T12:12:59.000+01:00
diff --git a/petab/__init__.py b/petab/__init__.py
@@ -23,6 +23,8 @@ def __getattr__(name):
         return attr
     if name == "v1":
         return importlib.import_module("petab.v1")
+    if name == "v2":
+        return importlib.import_module("petab.v2")
     if name != "__path__":
         warn(
             f"Accessing `petab.{name}` is deprecated and will be removed in "
diff --git a/petab/v1/calculate.py b/petab/v1/calculate.py
@@ -106,10 +106,7 @@ def calculate_residuals_for_table(
     )
     residual_df[RESIDUAL] = residual_df[RESIDUAL].astype("float64")
     # matching columns
-    compared_cols = set(MEASUREMENT_DF_COLS)
-    compared_cols -= {MEASUREMENT}
-    compared_cols &= set(measurement_df.columns)
-    compared_cols &= set(simulation_df.columns)
+    compared_cols = set(measurement_df.columns) & set(simulation_df.columns)
 
     # compute noise formulas for observables
     noise_formulas = get_symbolic_noise_formulas(observable_df)
@@ -127,6 +124,16 @@ def calculate_residuals_for_table(
             raise ValueError(
                 f"Could not find simulation for measurement {row}."
             )
+        # if we have multiple matches, check that the rows are all identical
+        elif (
+            mask.sum() > 1
+            and simulation_df.loc[mask].drop_duplicates().shape[0] > 1
+        ):
+            raise ValueError(
+                f"Multiple different simulations found for measurement "
+                f"{row}:\n{simulation_df.loc[mask]}"
+            )
+
         simulation = simulation_df.loc[mask][SIMULATION].iloc[0]
         if scale:
             # apply scaling
@@ -343,10 +350,7 @@ def calculate_llh_for_table(
     llhs = []
 
     # matching columns
-    compared_cols = set(MEASUREMENT_DF_COLS)
-    compared_cols -= {MEASUREMENT}
-    compared_cols &= set(measurement_df.columns)
-    compared_cols &= set(simulation_df.columns)
+    compared_cols = set(measurement_df.columns) & set(simulation_df.columns)
 
     # compute noise formulas for observables
     noise_formulas = get_symbolic_noise_formulas(observable_df)
diff --git a/petab/v1/conditions.py b/petab/v1/conditions.py
@@ -60,9 +60,11 @@ def write_condition_df(df: pd.DataFrame, filename: str | Path) -> None:
 
     Arguments:
         df: PEtab condition table
-        filename: Destination file name
+        filename: Destination file name. The parent directory will be created
+            if necessary.
     """
     df = get_condition_df(df)
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
     df.to_csv(filename, sep="\t", index=True)
 
 
diff --git a/petab/v1/core.py b/petab/v1/core.py
@@ -60,8 +60,10 @@ def write_simulation_df(df: pd.DataFrame, filename: str | Path) -> None:
 
     Arguments:
         df: PEtab simulation table
-        filename: Destination file name
+        filename: Destination file name. The parent directory will be created
+            if necessary.
     """
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
     df.to_csv(filename, sep="\t", index=False)
 
 
diff --git a/petab/v1/mapping.py b/petab/v1/mapping.py
@@ -60,9 +60,11 @@ def write_mapping_df(df: pd.DataFrame, filename: str | Path) -> None:
 
     Arguments:
         df: PEtab mapping table
-        filename: Destination file name
+        filename: Destination file name. The parent directory will be created
+            if necessary.
     """
     df = get_mapping_df(df)
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
     df.to_csv(filename, sep="\t", index=True)
 
 
diff --git a/petab/v1/measurements.py b/petab/v1/measurements.py
@@ -58,9 +58,11 @@ def write_measurement_df(df: pd.DataFrame, filename: str | Path) -> None:
 
     Arguments:
         df: PEtab measurement table
-        filename: Destination file name
+        filename: Destination file name. The parent directory will be created
+            if necessary.
     """
     df = get_measurement_df(df)
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
     df.to_csv(filename, sep="\t", index=False)
 
 
diff --git a/petab/v1/observables.py b/petab/v1/observables.py
@@ -67,9 +67,11 @@ def write_observable_df(df: pd.DataFrame, filename: str | Path) -> None:
 
     Arguments:
         df: PEtab observable table
-        filename: Destination file name
+        filename: Destination file name. The parent directory will be created
+            if necessary.
     """
     df = get_observable_df(df)
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
     df.to_csv(filename, sep="\t", index=True)
 
 
diff --git a/petab/v1/parameters.py b/petab/v1/parameters.py
@@ -112,9 +112,11 @@ def write_parameter_df(df: pd.DataFrame, filename: str | Path) -> None:
 
     Arguments:
         df: PEtab parameter table
-        filename: Destination file name
+        filename: Destination file name. The parent directory will be created
+            if necessary.
     """
     df = get_parameter_df(df)
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
     df.to_csv(filename, sep="\t", index=True)
 
 
diff --git a/petab/v1/problem.py b/petab/v1/problem.py
@@ -13,10 +13,10 @@
 import pandas as pd
 from pydantic import AnyUrl, BaseModel, Field, RootModel
 
+from ..versions import get_major_version
 from . import (
     conditions,
     core,
-    format_version,
     mapping,
     measurements,
     observables,
@@ -290,13 +290,13 @@ def get_path(filename):
                 "petab.CompositeProblem.from_yaml() instead."
             )
 
-        if yaml_config[FORMAT_VERSION] not in {"1", 1, "1.0.0", "2.0.0"}:
+        major_version = get_major_version(yaml_config)
+        if major_version not in {1, 2}:
             raise ValueError(
                 "Provided PEtab files are of unsupported version "
-                f"{yaml_config[FORMAT_VERSION]}. Expected "
-                f"{format_version.__format_version__}."
+                f"{yaml_config[FORMAT_VERSION]}."
             )
-        if yaml_config[FORMAT_VERSION] == "2.0.0":
+        if major_version == 2:
             warn("Support for PEtab2.0 is experimental!", stacklevel=2)
             warn(
                 "Using petab.v1.Problem with PEtab2.0 is deprecated. "
@@ -321,7 +321,7 @@ def get_path(filename):
                 if config.parameter_file
                 else None
             )
-        if config.format_version.root in [1, "1", "1.0.0"]:
+        if major_version == 1:
             if len(problem0.sbml_files) > 1:
                 # TODO https://github.com/PEtab-dev/libpetab-python/issues/6
                 raise NotImplementedError(
@@ -1074,8 +1074,8 @@ def add_observable(
     def add_parameter(
         self,
         id_: str,
-        estimated: bool | str | int = True,
-        nominal_value=None,
+        estimate: bool | str | int = True,
+        nominal_value: Number | None = None,
         scale: str = None,
         lb: Number = None,
         ub: Number = None,
@@ -1089,7 +1089,7 @@ def add_parameter(
 
         Arguments:
             id_: The parameter id
-            estimated: Whether the parameter is estimated
+            estimate: Whether the parameter is estimated
             nominal_value: The nominal value of the parameter
             scale: The parameter scale
             lb: The lower bound of the parameter
@@ -1104,12 +1104,8 @@ def add_parameter(
         record = {
             PARAMETER_ID: [id_],
         }
-        if estimated is not None:
-            record[ESTIMATE] = [
-                int(estimated)
-                if isinstance(estimated, bool | int)
-                else estimated
-            ]
+        if estimate is not None:
+            record[ESTIMATE] = [int(estimate)]
         if nominal_value is not None:
             record[NOMINAL_VALUE] = [nominal_value]
         if scale is not None:
diff --git a/petab/v1/yaml.py b/petab/v1/yaml.py
@@ -12,15 +12,15 @@
 import yaml
 from pandas.io.common import get_handle
 
+from ..versions import parse_version
 from .C import *  # noqa: F403
 
 # directory with PEtab yaml schema files
 SCHEMA_DIR = Path(__file__).parent.parent / "schemas"
 # map of version number to validation schema
 SCHEMAS = {
-    "1": SCHEMA_DIR / "petab_schema.v1.0.0.yaml",
-    "1.0.0": SCHEMA_DIR / "petab_schema.v1.0.0.yaml",
-    "2.0.0": SCHEMA_DIR / "petab_schema.v2.0.0.yaml",
+    (1, 0): SCHEMA_DIR / "petab_schema.v1.0.0.yaml",
+    (2, 0): SCHEMA_DIR / "petab_schema.v2.0.0.yaml",
 }
 
 __all__ = [
@@ -71,14 +71,18 @@ def validate_yaml_syntax(
     yaml_config = load_yaml(yaml_config)
 
     if schema is None:
-        # try get PEtab version from yaml file
+        # try to get PEtab version from the yaml file
         #  if this is not the available, the file is not valid anyways,
         #  but let's still use the latest PEtab schema for full validation
+        version = yaml_config.get(FORMAT_VERSION, None)
         version = (
-            yaml_config.get(FORMAT_VERSION, None) or list(SCHEMAS.values())[-1]
+            parse_version(version)[:2]
+            if version
+            else list(SCHEMAS.values())[-1]
         )
+
         try:
-            schema = SCHEMAS[str(version)]
+            schema = SCHEMAS[version]
         except KeyError as e:
             raise ValueError(
                 "Unknown PEtab version given in problem "
@@ -234,8 +238,10 @@ def write_yaml(yaml_config: dict[str, Any], filename: str | Path) -> None:
 
     Arguments:
         yaml_config: Data to write
-        filename: File to create
+        filename: Destination file name. The parent directory will be created
+            if necessary.
     """
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
     with open(filename, "w") as outfile:
         yaml.dump(
             yaml_config, outfile, default_flow_style=False, sort_keys=False
diff --git a/petab/v2/experiments.py b/petab/v2/experiments.py
@@ -34,7 +34,9 @@ def write_experiment_df(df: pd.DataFrame, filename: str | Path) -> None:
 
     Arguments:
         df: PEtab experiments table
-        filename: Destination file name
+        filename: Destination file name. The parent directory will be created
+            if necessary.
     """
     df = get_experiment_df(df)
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
     df.to_csv(filename, sep="\t", index=False)
diff --git a/petab/v2/lint.py b/petab/v2/lint.py
@@ -228,11 +228,16 @@ class CheckValidPetabIdColumn(ValidationTask):
     """A task to check that a given column contains only valid PEtab IDs."""
 
     def __init__(
-        self, table_name: str, column_name: str, required_column: bool = True
+        self,
+        table_name: str,
+        column_name: str,
+        required_column: bool = True,
+        ignore_nan: bool = False,
     ):
         self.table_name = table_name
         self.column_name = column_name
         self.required_column = required_column
+        self.ignore_nan = ignore_nan
 
     def run(self, problem: Problem) -> ValidationIssue | None:
         df = getattr(problem, f"{self.table_name}_df")
@@ -248,7 +253,10 @@ def run(self, problem: Problem) -> ValidationIssue | None:
             return
 
         try:
-            check_ids(df[self.column_name].values, kind=self.column_name)
+            ids = df[self.column_name].values
+            if self.ignore_nan:
+                ids = ids[~pd.isna(ids)]
+            check_ids(ids, kind=self.column_name)
         except ValueError as e:
             return ValidationError(str(e))
 
@@ -308,21 +316,26 @@ def run(self, problem: Problem) -> ValidationIssue | None:
         except AssertionError as e:
             return ValidationError(str(e))
 
-        # TODO: introduce some option for validation partial vs full
+        # TODO: introduce some option for validation of partial vs full
         #  problem. if this is supposed to be a complete problem, a missing
         #  condition table should be an error if the measurement table refers
-        #  to conditions
-
-        # check that measured experiments
-        if problem.experiment_df is None:
-            return
-
+        #  to conditions, otherwise it should maximally be a warning
         used_experiments = set(problem.measurement_df[EXPERIMENT_ID].values)
-        available_experiments = set(
-            problem.experiment_df[EXPERIMENT_ID].unique()
+        # handle default-experiment
+        used_experiments = set(
+            filter(
+                lambda x: not pd.isna(x),
+                used_experiments,
+            )
+        )
+        # check that measured experiments exist
+        available_experiments = (
+            set(problem.experiment_df[EXPERIMENT_ID].unique())
+            if problem.experiment_df is not None
+            else set()
         )
         if missing_experiments := (used_experiments - available_experiments):
-            raise AssertionError(
+            return ValidationError(
                 "Measurement table references experiments that "
                 "are not specified in the experiments table: "
                 + str(missing_experiments)
@@ -826,6 +839,7 @@ def append_overrides(overrides):
     CheckMeasurementTable(),
     CheckConditionTable(),
     CheckExperimentTable(),
+    CheckValidPetabIdColumn("measurement", EXPERIMENT_ID, ignore_nan=True),
     CheckValidPetabIdColumn("experiment", EXPERIMENT_ID),
     CheckValidPetabIdColumn("experiment", CONDITION_ID),
     CheckExperimentConditionsExist(),
diff --git a/petab/v2/petab1to2.py b/petab/v2/petab1to2.py
diff --git a/petab/v2/problem.py b/petab/v2/problem.py
diff --git a/petab/versions.py b/petab/versions.py
diff --git a/tests/test_version.py b/tests/test_version.py
