GHA: Run tests on windows-latest, macos-latest, ubuntu-latest (#91)

* Run test suite on windows-latest, macos-latest, ubuntu-latest
* Fixed various issues related to temporary files on Windows
* Add support for pathlib.Path to some functions missed in #93 
* Fix loading remote yaml files on Windows

Closes #80

Co-authored-by: Dilan Pathirana <[email protected]>

Author: dweindl

.github/workflows/ci_tests.yml

@@ -8,10 +8,11 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
     strategy:
       matrix:
+        platform: [windows-latest, macos-latest, ubuntu-latest]
         python-version: [3.7, 3.8, 3.9, "3.10"]
+    runs-on: ${{ matrix.platform }}
 
     steps:
     - name: Check out repository
@@ -36,13 +37,18 @@ jobs:
         pip install -r .ci_pip_reqs.txt
         pip install -e .[reports,combine]
 
+    - name: Run flake8
+      run: |
+        python -m flake8 --exclude=build,doc,example,tmp --extend-ignore=F403,F405
+      if: matrix.platform == 'ubuntu-latest'
+
     - name: Run tests
       run: |
         pytest --cov --cov-report=xml tests
-        python -m flake8 --exclude=build,doc,example,tmp --extend-ignore=F403,F405
 
     - name: Coverage
       uses: codecov/codecov-action@v1
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: ./coverage.xml
+      if: matrix.platform == 'ubuntu-latest'

petab/core.py

@@ -179,7 +179,8 @@ def flatten_timepoint_specific_output_overrides(
 
 
 def concat_tables(
-        tables: Union[str, pd.DataFrame, Iterable[Union[pd.DataFrame, str]]],
+        tables: Union[str, Path, pd.DataFrame,
+                      Iterable[Union[pd.DataFrame, str, Path]]],
         file_parser: Optional[Callable] = None
 ) -> pd.DataFrame:
     """Concatenate DataFrames provided as DataFrames or filenames, and a parser
@@ -198,14 +199,14 @@ def concat_tables(
     if isinstance(tables, pd.DataFrame):
         return tables
 
-    if isinstance(tables, str):
+    if isinstance(tables, (str, Path)):
         return file_parser(tables)
 
     df = pd.DataFrame()
 
     for tmp_df in tables:
         # load from file, if necessary
-        if isinstance(tmp_df, str):
+        if isinstance(tmp_df, (str, Path)):
             tmp_df = file_parser(tmp_df)
 
         df = df.append(tmp_df, sort=False,

petab/problem.py

@@ -1,15 +1,17 @@
 """PEtab Problem class"""
 
 import os
-from pathlib import Path
 import tempfile
+from pathlib import Path, PurePosixPath
 from typing import Dict, Iterable, List, Optional, Union
+from urllib.parse import unquote, urlparse, urlunparse
 from warnings import warn
 
-import pandas as pd
 import libsbml
-from . import (parameter_mapping, measurements, conditions, parameters,
-               sampling, sbml, yaml, core, observables, format_version)
+import pandas as pd
+
+from . import (conditions, core, format_version, measurements, observables,
+               parameter_mapping, parameters, sampling, sbml, yaml)
 from .C import *  # noqa: F403
 
 __all__ = ['Problem', 'get_default_condition_file_name',
@@ -88,13 +90,18 @@ def __setstate__(self, state):
         self.__dict__.update(state)
 
     @staticmethod
-    def from_files(sbml_file: str = None,
-                   condition_file: str = None,
-                   measurement_file: Union[str, Iterable[str]] = None,
-                   parameter_file: Union[str, List[str]] = None,
-                   visualization_files: Union[str, Iterable[str]] = None,
-                   observable_files: Union[str, Iterable[str]] = None
-                   ) -> 'Problem':
+    def from_files(
+            sbml_file: Union[str, Path, None] = None,
+            condition_file: Union[str, Path, None] = None,
+            measurement_file: Union[str, Path,
+                                    Iterable[Union[str, Path]]] = None,
+            parameter_file: Union[str, Path,
+                                  Iterable[Union[str, Path]]] = None,
+            visualization_files: Union[str, Path,
+                                       Iterable[Union[str, Path]]] = None,
+            observable_files: Union[str, Path,
+                                    Iterable[Union[str, Path]]] = None
+    ) -> 'Problem':
         """
         Factory method to load model and tables from files.
 
@@ -146,18 +153,41 @@ def from_files(sbml_file: str = None,
                        visualization_df=visualization_df)
 
     @staticmethod
-    def from_yaml(yaml_config: Union[Dict, str]) -> 'Problem':
+    def from_yaml(yaml_config: Union[Dict, Path, str]) -> 'Problem':
         """
         Factory method to load model and tables as specified by YAML file.
 
         Arguments:
             yaml_config: PEtab configuration as dictionary or YAML file name
         """
+        if isinstance(yaml_config, Path):
+            yaml_config = str(yaml_config)
+
+        get_path = lambda filename: filename  # noqa: E731
         if isinstance(yaml_config, str):
-            path_prefix = os.path.dirname(yaml_config)
+            yaml_path = yaml_config
             yaml_config = yaml.load_yaml(yaml_config)
-        else:
-            path_prefix = ""
+
+            # yaml_config may be path or URL
+            path_url = urlparse(yaml_path)
+            if not path_url.scheme or \
+                    (path_url.scheme != 'file' and not path_url.netloc):
+                # a regular file path string
+                path_prefix = Path(yaml_path).parent
+                get_path = lambda filename: \
+                    path_prefix / filename  # noqa: E731
+            else:
+                # a URL
+                # extract parent path from
+                url_path = unquote(urlparse(yaml_path).path)
+                parent_path = str(PurePosixPath(url_path).parent)
+                path_prefix = urlunparse(
+                    (path_url.scheme, path_url.netloc, parent_path,
+                     path_url.params, path_url.query, path_url.fragment)
+                )
+                # need "/" on windows, not "\"
+                get_path = lambda filename: \
+                    f"{path_prefix}/{filename}"  # noqa: E731
 
         if yaml.is_composite_problem(yaml_config):
             raise ValueError('petab.Problem.from_yaml() can only be used for '
@@ -176,26 +206,21 @@ def from_yaml(yaml_config: Union[Dict, str]) -> 'Problem':
 
         if isinstance(yaml_config[PARAMETER_FILE], list):
             parameter_file = [
-                os.path.join(path_prefix, f)
-                for f in yaml_config[PARAMETER_FILE]
+                get_path(f) for f in yaml_config[PARAMETER_FILE]
             ]
         else:
-            parameter_file = os.path.join(
-                path_prefix, yaml_config[PARAMETER_FILE])
+            parameter_file = get_path(yaml_config[PARAMETER_FILE])
 
         return Problem.from_files(
-            sbml_file=os.path.join(path_prefix, problem0[SBML_FILES][0]),
-            measurement_file=[os.path.join(path_prefix, f)
+            sbml_file=get_path(problem0[SBML_FILES][0]),
+            measurement_file=[get_path(f)
                               for f in problem0[MEASUREMENT_FILES]],
-            condition_file=os.path.join(
-                path_prefix, problem0[CONDITION_FILES][0]),
+            condition_file=get_path(problem0[CONDITION_FILES][0]),
             parameter_file=parameter_file,
             visualization_files=[
-                os.path.join(path_prefix, f)
-                for f in problem0.get(VISUALIZATION_FILES, [])],
+                get_path(f) for f in problem0.get(VISUALIZATION_FILES, [])],
             observable_files=[
-                os.path.join(path_prefix, f)
-                for f in problem0.get(OBSERVABLE_FILES, [])]
+                get_path(f) for f in problem0.get(OBSERVABLE_FILES, [])]
         )
 
     @staticmethod
@@ -273,7 +298,7 @@ def from_combine(filename: Union[Path, str]) -> 'Problem':
     def to_files_generic(
         self,
         prefix_path: Union[str, Path],
-    ) -> None:
+    ) -> str:
         """Save a PEtab problem to generic file names.
 
         The PEtab problem YAML file is always created. PEtab data files are
@@ -318,15 +343,16 @@ def to_files_generic(
         return str(prefix_path / filenames['yaml_file'])
 
     def to_files(self,
-                 sbml_file: Optional[str] = None,
-                 condition_file: Optional[str] = None,
-                 measurement_file: Optional[str] = None,
-                 parameter_file: Optional[str] = None,
-                 visualization_file: Optional[str] = None,
-                 observable_file: Optional[str] = None,
-                 yaml_file: Optional[str] = None,
-                 prefix_path: Optional[Union[str, Path]] = None,
-                 relative_paths: bool = True,) -> None:
+                 sbml_file: Union[None, str, Path] = None,
+                 condition_file: Union[None, str, Path] = None,
+                 measurement_file: Union[None, str, Path] = None,
+                 parameter_file: Union[None, str, Path] = None,
+                 visualization_file: Union[None, str, Path] = None,
+                 observable_file: Union[None, str, Path] = None,
+                 yaml_file: Union[None, str, Path] = None,
+                 prefix_path: Union[None, str, Path] = None,
+                 relative_paths: bool = True,
+                 ) -> None:
         """
         Write PEtab tables to files for this problem
 

petab/visualize/data_overview.py

@@ -2,8 +2,9 @@
 Functions for creating an overview report of a PEtab problem
 """
 
-import os
+from pathlib import Path
 from shutil import copyfile
+from typing import Union
 
 import pandas as pd
 import petab
@@ -12,16 +13,21 @@
 __all__ = ['create_report']
 
 
-def create_report(problem: petab.Problem, model_name: str) -> None:
+def create_report(
+        problem: petab.Problem,
+        model_name: str,
+        output_path: Union[str, Path] = ''
+) -> None:
     """Create an HTML overview data / model overview report
 
     Arguments:
         problem: PEtab problem
         model_name: Name of the model, used for file name for report
+        output_path: Output directory
     """
 
-    template_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                                'templates')
+    template_dir = Path(__file__).absolute().parent / 'templates'
+    output_path = Path(output_path)
     template_file = "report.html"
 
     data_per_observable = get_data_per_observable(problem.measurement_df)
@@ -30,18 +36,17 @@ def create_report(problem: petab.Problem, model_name: str) -> None:
 
     # Setup template engine
     import jinja2
-    template_loader = jinja2.FileSystemLoader(
-        searchpath=template_dir)
+    template_loader = jinja2.FileSystemLoader(searchpath=template_dir)
     template_env = jinja2.Environment(loader=template_loader)
     template = template_env.get_template(template_file)
 
     # Render and save
     output_text = template.render(problem=problem, model_name=model_name,
                                   data_per_observable=data_per_observable,
                                   num_conditions=num_conditions)
-    with open(model_name + '.html', 'w') as html_file:
+    with open(output_path / f'{model_name}.html', 'w') as html_file:
         html_file.write(output_text)
-    copyfile(os.path.join(template_dir, 'mystyle.css'), 'mystyle.css')
+    copyfile(template_dir / 'mystyle.css', output_path / 'mystyle.css')
 
 
 def get_data_per_observable(measurement_df: pd.DataFrame) -> pd.DataFrame:
@@ -70,24 +75,3 @@ def get_data_per_observable(measurement_df: pd.DataFrame) -> pd.DataFrame:
     data_per_observable = data_per_observable.astype(int)
 
     return data_per_observable
-
-
-def main():
-    """Data overview generation with example data from the repository for
-    testing
-    """
-
-    root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                             '..', '..', 'doc/example/example_Fujita/')
-    problem = petab.Problem.from_files(
-        sbml_file=os.path.join(root_path, 'Fujita_model.xml'),
-        condition_file=os.path.join(root_path,
-                                    'Fujita_experimentalCondition.tsv'),
-        measurement_file=os.path.join(root_path, 'Fujita_measurementData.tsv'),
-        parameter_file=os.path.join(root_path, 'Fujita_parameters.tsv'),
-    )
-    create_report(problem, 'Fujita')
-
-
-if __name__ == '__main__':
-    main()

petab/yaml.py

@@ -19,7 +19,9 @@
            'create_problem_yaml']
 
 
-def validate(yaml_config: Union[Dict, str], path_prefix: Optional[str] = None):
+def validate(
+        yaml_config: Union[Dict, str, Path],
+        path_prefix: Union[None, str, Path] = None):
     """Validate syntax and semantics of PEtab config YAML
 
     Arguments:
@@ -37,7 +39,7 @@ def validate(yaml_config: Union[Dict, str], path_prefix: Optional[str] = None):
 
 
 def validate_yaml_syntax(
-        yaml_config: Union[Dict, str],
+        yaml_config: Union[Dict, str, Path],
         schema: Union[None, Dict, str] = None):
     """Validate PEtab YAML file syntax
 
@@ -61,8 +63,8 @@ def validate_yaml_syntax(
 
 
 def validate_yaml_semantics(
-        yaml_config: Union[Dict, str],
-        path_prefix: Optional[str] = None
+        yaml_config: Union[Dict, str, Path],
+        path_prefix: Union[None, str, Path] = None
 ):
     """Validate PEtab YAML file semantics