Merge pull request #906 from ICB-DCM/release_0_10_17

Release 0.10.17

Author: FFroehlich

.github/workflows/test-large-model.yml

@@ -4,7 +4,7 @@ on:
     branches:
       - develop
       - master
-      - feature_sparse_quadratures
+      - fix_896_perftestfiles
 
   pull_request:
     branches:
@@ -53,10 +53,11 @@ jobs:
         cd CS_Signalling_ERBB_RAS_AKT \
           && check_time.sh \
           petab_import amici_import_petab.py -v \
-            -s 'PEtab/CS_Signalling_ERBB_RAS_AKT_petab.xml' \
-            -c 'PEtab/conditions_petab.tsv' \
-            -m 'PEtab/measurements_petab.tsv' \
-            -p 'PEtab/parameters_petab.tsv' --no-compile
+            -n 'CS_Signalling_ERBB_RAS_AKT_petab' \
+            -s 'FroehlichKes2018/PEtab/CS_Signalling_ERBB_RAS_AKT_petab.xml' \
+            -c 'FroehlichKes2018/PEtab/conditions_petab.tsv' \
+            -m 'FroehlichKes2018/PEtab/measurements_petab.tsv' \
+            -p 'FroehlichKes2018/PEtab/parameters_petab.tsv' --no-compile
 
     # install model package
     - name: Install test model
@@ -79,4 +80,4 @@ jobs:
         check_time.sh forward_simulation_non_optimal_parameters tests/performance/test.py forward_simulation_non_optimal_parameters
     - name: adjoint_sensitivities_non_optimal_parameters
       run: |
-        check_time.sh adjoint_sensitivities_non_optimal_parameters tests/performance/test.py adjoint_sensitivities_non_optimal_parameters
+        check_time.sh adjoint_sensitivities_non_optimal_parameters tests/performance/test.py adjoint_sensitivities_non_optimal_parameters

.travis.yml

@@ -13,7 +13,7 @@ matrix:
     - os: linux
       dist: bionic
       language: python
-      python: 3.7
+      python: 3.8
       compiler: gcc
       addons:
         apt:
@@ -27,11 +27,21 @@ matrix:
             - libc6-dbg
       env:
         - ENABLE_GCOV_COVERAGE=TRUE
-        - CI_MODE=test
+        - CI_BUILD=TRUE
+        - CI_CPPUTEST=TRUE
+        - CI_PYTHON=TRUE
+        - CI_ARCHIVE=TRUE
+        - CI_NOTEBOOK=TRUE
       before_install:
         - ./scripts/buildValgrind.sh
       after_script:
+        # cpputest coverage cpp
+        - lcov --compat-libtool --no-external --directory ${BASE_DIR}/build/CMakeFiles/amici.dir/src --base-directory ${BASE_DIR} --capture --output-file coverage_cpp.info
+        # py coverage cpp
+        - lcov --compat-libtool --no-external --directory ${BASE_DIR}/python/sdist/build/temp.linux-x86_64-${TRAVIS_PYTHON_VERSION}/amici/src --base-directory ${BASE_DIR} --capture --output-file coverage_py.info
+        - lcov -a coverage_cpp.info -a coverage_py.info -o coverage.info
         - bash <(curl -s https://codecov.io/bash) -f coverage.info -X fix -F cpp
+        # py coverage py
         - bash <(curl -s https://codecov.io/bash) -f coverage_py.xml -F python
       before_deploy:
         - cd $BASE_DIR
@@ -43,12 +53,36 @@ matrix:
           branch: master
           tags: true
 
+    - os: linux
+      dist: bionic
+      language: python
+      python: 3.7
+      compiler: gcc
+      addons:
+        apt:
+          packages:
+            - libhdf5-serial-dev
+            - zlib1g-dev
+            - libatlas-base-dev
+            - g++-5
+            - libc6-dbg
+      env:
+        - CI_BUILD=TRUE
+        - CI_PYTHON=TRUE
+        - CI_NOTEBOOK=TRUE
+
     - os: osx
       osx_image: xcode11
       language: minimal
       compiler: clang
       env:
-        - CI_MODE=test
+        - CI_BUILD=TRUE
+        - CI_CPPUTEST=TRUE
+        - CI_CMAKE=TRUE
+        - CI_ARCHIVE=TRUE
+        - CI_CPPCHECK=TRUE
+        - CI_PYTHON=TRUE
+        - CI_NOTEBOOK=TRUE
       addons:
         homebrew:
           packages:
@@ -59,6 +93,7 @@ matrix:
           update: true
       before_install:
         - travis_wait brew link --overwrite gcc # fix linker warning regarding /usr/local/include/c++
+        - brew link --overwrite python # https://github.com/ICB-DCM/AMICI/issues/894
         - export -f travis_fold travis_nanoseconds travis_time_start travis_time_finish
         - pip3 install --user -U numpy
       after_success:
@@ -69,19 +104,21 @@ matrix:
       language: minimal
       compiler: clang
       env:
-        - CI_MODE=deploy
+        - CI_DOC=TRUE
       addons:
         homebrew:
           casks:
             - mactex
           packages:
-            - doxygen
+            # - doxygen # disabled because current 1.8.16 does not work with recent ghostscript
+            - bison # doxygen build
             - ragel
             - graphviz
           update: true
       before_install:
         - export PATH=/Users/travis/Library/Python/3.7/bin:/Library/TeX/texbin:$PATH
         - export -f travis_fold travis_nanoseconds travis_time_start travis_time_finish
+        - brew link --overwrite python # https://github.com/ICB-DCM/AMICI/issues/894
       after_success:
         - cd $BASE_DIR # cd to base dir for correct relative path in deploy
       deploy:
@@ -98,30 +135,31 @@ install:
     - export BASE_DIR=`pwd`
     # Build swig4.0 (not yet available with apt) to include pydoc in source distribution for pypi
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then scripts/downloadAndBuildSwig.sh && export PATH=${BASE_DIR}/ThirdParty/swig-4.0.1/install/bin:${PATH}; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export PYTHON_EXECUTABLE=$(which python3); fi # cmake wont be able to find python3 on its own ...
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CI_MODE" == "test" ]]; then pip3 install --upgrade pip==9.0.3 setuptools wheel pkgconfig scipy; fi
-    - if [[ "$TRAVIS_OS_NAME" != "linux" ]] && [[ "$CI_MODE" == "test" ]]; then pip3 install --user --upgrade pip==9.0.3 setuptools wheel pkgconfig scipy; fi
-    - if [[ "$CI_MODE" == "deploy" ]]; then pip3 install --user --upgrade pip==9.0.3 doxypypy; fi # pinning pip because of https://github.com/pypa/pip/issues/5240
-    - if [[ "$CI_MODE" == "test" ]]; then ./scripts/buildSuiteSparse.sh; fi
-    - if [[ "$CI_MODE" == "test" ]]; then ./scripts/buildSundials.sh; fi
-    - if [[ "$CI_MODE" == "test" ]]; then ./scripts/buildCpputest.sh; fi
-    - if [[ "$CI_MODE" == "test" ]]; then ./scripts/buildBNGL.sh; fi
-    - if [[ "$CI_MODE" == "test" ]]; then ./scripts/buildAmici.sh; fi
-    - if [[ "$CI_MODE" == "test" ]]; then ./scripts/installAmiciArchive.sh; fi
-    - if [[ "$CI_MODE" == "test" ]]; then ./scripts/installAmiciSource.sh; fi
+    - if [[ "$CI_DOC" == "TRUE" ]]; then pip3 install --user --upgrade pip==9.0.3 doxypypy; fi # pinning pip because of https://github.com/pypa/pip/issues/5240
+    - if [[ "$CI_DOC" == "TRUE" ]]; then export PATH="/usr/local/opt/bison/bin:$PATH"; LDFLAGS="-L/usr/local/opt/bison/lib" scripts/downloadAndBuildDoxygen.sh; fi
+    - if [[ "$CI_BUILD" == "TRUE" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip3 install --upgrade pip==9.0.3 setuptools wheel pkgconfig scipy; fi
+    - if [[ "$CI_BUILD" == "TRUE" ]] && [[ "$TRAVIS_OS_NAME" != "linux" ]]; then pip3 install --user --upgrade pip==9.0.3 setuptools wheel pkgconfig scipy; fi
+    - if [[ "$CI_BUILD" == "TRUE" ]]; then ./scripts/buildSuiteSparse.sh; fi
+    - if [[ "$CI_BUILD" == "TRUE" ]]; then ./scripts/buildSundials.sh; fi
+    - if [[ "$CI_BUILD" == "TRUE" ]]; then ./scripts/buildCpputest.sh; fi
+    - if [[ "$CI_PYTHON" == "TRUE" ]]; then ./scripts/buildBNGL.sh; fi
+    - if [[ "$CI_BUILD" == "TRUE" ]]; then ./scripts/buildAmici.sh; fi
+    - if [[ "$CI_ARCHIVE" == "TRUE" ]]; then ./scripts/installAmiciArchive.sh; fi
+    - if [[ "$CI_PYTHON" == "TRUE" ]]; then ./scripts/installAmiciSource.sh; fi
 
 script:
     - export -f travis_fold travis_nanoseconds travis_time_start travis_time_finish
     - export FOLD=$BASE_DIR/scripts/travis_wrap.sh
     - cd $BASE_DIR
-    - if [[ "$CI_MODE" == "test" ]]; then $FOLD notebooks "cd $BASE_DIR && scripts/runNotebook.sh python/examples/example_*/"; fi
-    - if [[ "$TRAVIS_OS_NAME" != "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD cpputest ./scripts/run-cpputest.sh; fi
-    - if [[ "$TRAVIS_OS_NAME" != "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD python-tests ./scripts/run-python-tests.sh; fi
-    - if [[ "$TRAVIS_OS_NAME" != "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD cmake ./tests/testCMakeCompilation.sh; fi
-    - if [[ "$CI_MODE" == "deploy" ]]; then $FOLD doxygen ./scripts/run-doxygen.sh; fi
-    - if [[ "$TRAVIS_OS_NAME" != "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD cppcheck ./scripts/run-cppcheck.sh; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD valgrind ./scripts/run-valgrind.sh; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD codecov ./scripts/run-codecov.sh; fi
+    - if [[ "$CI_CPPCHECK" == "TRUE" ]]; then $FOLD cppcheck ./scripts/run-cppcheck.sh; fi
+    - if [[ "$CI_NOTEBOOK" == "TRUE" ]]; then $FOLD notebooks "cd $BASE_DIR && scripts/runNotebook.sh python/examples/example_*/"; fi
+    - if [[ "$CI_PYTHON" == "TRUE" ]] && [[ "$ENABLE_GCOV_COVERAGE" == "TRUE" ]]; then $FOLD codecov ./scripts/run-codecov.sh; fi
+    - if [[ "$CI_PYTHON" == "TRUE" ]] && [[ "$ENABLE_GCOV_COVERAGE" != "TRUE" ]]; then $FOLD python-tests ./scripts/run-python-tests.sh; fi
+    # needs to be run after python tests
+    - if [[ "$CI_CMAKE" == "TRUE" ]]; then $FOLD cmake ./tests/testCMakeCompilation.sh; fi
+    - if [[ "$CI_CPPUTEST" == "TRUE" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then $FOLD valgrind ./scripts/run-valgrind.sh; fi
+    - if [[ "$CI_CPPUTEST" == "TRUE" ]] && [[ "$TRAVIS_OS_NAME" != "linux" ]]; then $FOLD cpputest ./scripts/run-cpputest.sh; fi
+    - if [[ "$CI_DOC" == "TRUE" ]]; then $FOLD doxygen ./scripts/run-doxygen.sh; fi
 
 after_failure:
     - $FOLD ls -alR

INSTALL.md

@@ -58,7 +58,7 @@ For Python-AMICI usage see
 To install development versions which have not been released to pypi yet,
 you can install AMICI with pip directly from GitHub using:
 
-    pip3 install -e https://github.com/icb-dcm/amici/archive/develop.zip#egg=amici\&subdirectory=python/sdist
+    pip3 install -e git+https://github.com/icb-dcm/amici.git@develop#egg=amici\&subdirectory=python/sdist
 
 Replace `develop` by the branch or commit you want to install.
 

ThirdParty/sundials/README.md

@@ -114,7 +114,7 @@ reporting work done with SUNDIALS:
 * Alan C. Hindmarsh, Peter N. Brown, Keith E. Grant, Steven L. Lee, Radu
 Serban, Dan E. Shumaker, and Carol S. Woodward. 2005. SUNDIALS: Suite of
 nonlinear and differential/algebraic equation solvers. ACM Trans. Math. Softw.
-31, 3 (September 2005), 363-396. DOI=http://dx.doi.org/10.1145/1089014.1089020
+31, 3 (September 2005), 363-396. DOI=https://doi.org/10.1145/1089014.1089020
 
 ## License ##
 SUNDIALS is released under the BSD 3-clause license. See the [LICENSE](./LICENSE)

documentation/CI.md

@@ -26,6 +26,27 @@ To run Python tests, run `../scripts/run-python-tests.sh` from anywhere
 (assumes build directory is `build/`) or run `make python-tests` in your build
 directory.
 
+### SBML Test Suite
+
+We test Python-AMICI SBML support using the test cases from the semantic
+[SBML Test Suite](https://github.com/sbmlteam/sbml-test-suite/). When making
+changes to the model import functions, make sure to run these tests.
+
+To run the SBML Test Suite test cases, the easiest way is:
+
+1. Running `scripts/installAmiciSource.sh` which
+   creates a virtual Python environment and performs a development installation
+   of AMICI from the current repository. (This needs to be run only once or
+   after AMICI model generation or C++ changes).
+
+2. Running `scripts/run-SBMLTestsuite.sh`. This will download the test cases
+   if necessary and run them all. A subset of test cases can be selected with
+   an optional argument (e.g. `scripts/run-SBMLTestsuite.sh 1,3-6,8`, to run
+   cases 1, 3, 4, 5, 6 and 8).
+
+   Once the test cases are available locally, for debugging it might be easier
+   to directly use `pytest` with `tests/testSBMLSuite.py`.
+
 
 ## Matlab tests (not included in CI pipeline)
 

documentation/amici_refs.bib

@@ -300,7 +300,7 @@ @article{LoosMoe2018
 	Title = {A Hierarchical, Data-Driven Approach to Modeling Single-Cell Populations Predicts Latent Causes of Cell-To-Cell Variability},
 	Volume = {6},
 	Year = {2018},
-	Bdsk-Url-1 = {http://dx.doi.org/10.1016/j.cels.2018.04.008}}
+	Bdsk-Url-1 = {https://doi.org/10.1016/j.cels.2018.04.008}}
 
 @article{MaierLoo2017,
 	Author = {Maier, C. and Loos, C. and Hasenauer, J.},
@@ -457,7 +457,7 @@ @Article{SchmiesterSch2019
   year     = {2019},
   month    = {07},
   issn     = {1367-4803},
-  abstract = {{Mechanistic models of biochemical reaction networks facilitate the quantitative understanding of biological processes and the integration of heterogeneous datasets. However, some biological processes require the consideration of comprehensive reaction networks and therefore large-scale models. Parameter estimation for such models poses great challenges, in particular when the data are on a relative scale.Here, we propose a novel hierarchical approach combining (i) the efficient analytic evaluation of optimal scaling, offset, and error model parameters with (ii) the scalable evaluation of objective function gradients using adjoint sensitivity analysis. We evaluate the properties of the methods by parameterizing a pan-cancer ordinary differential equation model (\\>1000 state variables, \\>4000 parameters) using relative protein, phospho-protein and viability measurements. The hierarchical formulation improves optimizer performance considerably. Furthermore, we show that this approach allows estimating error model parameters with negligible computational overhead when no experimental estimates are available, providing an unbiased way to weight heterogeneous data. Overall, our hierarchical formulation is applicable to a wide range of models, and allows for the efficient parameterization of large-scale models based on heterogeneous relative measurements.Supplementary information are available at Bioinformatics online. Supplementary code and data are available online at http://doi.org/10.5281/zenodo.3254429 and http://doi.org/10.5281/zenodo.3254441.}},
+  abstract = {{Mechanistic models of biochemical reaction networks facilitate the quantitative understanding of biological processes and the integration of heterogeneous datasets. However, some biological processes require the consideration of comprehensive reaction networks and therefore large-scale models. Parameter estimation for such models poses great challenges, in particular when the data are on a relative scale.Here, we propose a novel hierarchical approach combining (i) the efficient analytic evaluation of optimal scaling, offset, and error model parameters with (ii) the scalable evaluation of objective function gradients using adjoint sensitivity analysis. We evaluate the properties of the methods by parameterizing a pan-cancer ordinary differential equation model (\\>1000 state variables, \\>4000 parameters) using relative protein, phospho-protein and viability measurements. The hierarchical formulation improves optimizer performance considerably. Furthermore, we show that this approach allows estimating error model parameters with negligible computational overhead when no experimental estimates are available, providing an unbiased way to weight heterogeneous data. Overall, our hierarchical formulation is applicable to a wide range of models, and allows for the efficient parameterization of large-scale models based on heterogeneous relative measurements.Supplementary information are available at Bioinformatics online. Supplementary code and data are available online at https://doi.org/10.5281/zenodo.3254429 and https://doi.org/10.5281/zenodo.3254441.}},
   doi      = {10.1093/bioinformatics/btz581},
   eprint   = {http://oup.prod.sis.lan/bioinformatics/advance-article-pdf/doi/10.1093/bioinformatics/btz581/29004243/btz581.pdf},
   url      = {https://doi.org/10.1093/bioinformatics/btz581},

documentation/development.md

@@ -60,6 +60,7 @@ described below:
 * Write meaningful commit messages
 
 * Run all tests to ensure nothing was broken
+  ([more details](documentation/CI.md))
 
   * Run `scripts/buildAll.sh && scripts/run-cpputest.sh`.
 

documentation/gfx/amici_workflow.png

@@ -1253,6 +1253,8 @@ class Solver {
      *
      * @param which identifier of the backwards problem
      * @param ami_mem pointer to the forward solver memory instance
+     * @return A (void *) pointer to the CVODES memory allocated for the
+     * backward problem.
      */
     virtual void *getAdjBmem(void *ami_mem, int which) const = 0;