From 0a4dc6ce6890162fc7db1c17000456443e5c0c28 Mon Sep 17 00:00:00 2001 From: Unknown Date: Sat, 6 Oct 2018 13:12:22 +0100 Subject: [PATCH 01/19] travis typos again (trying to get build on development) --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d66dfa6..36db075 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ env: branches: only: - - master + - development install: - source devtools/travis-ci/install_miniconda.sh @@ -30,7 +30,7 @@ deploy: skip_cleanup: true local-dir: docs/_deploy/ on: - branch: Development + branch: master condition: "$CONDA_PY = 3.5" region: us-east-1 detect_encoding: true From 2019042f3c99abf618c541f38c1451882bc249e1 Mon Sep 17 00:00:00 2001 From: Unknown Date: Sat, 6 Oct 2018 13:14:08 +0100 Subject: [PATCH 02/19] ci on master and development --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 36db075..2ec8f16 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ env: branches: only: + - master - development install: From c1dd18de05008e41e05d0dd4eaf5b5a6e1a2fe15 Mon Sep 17 00:00:00 2001 From: Unknown Date: Sat, 6 Oct 2018 13:31:39 +0100 Subject: [PATCH 03/19] another travis ci type development --> Development --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2ec8f16..3340245 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ env: branches: only: + - Development - master - - development install: - source devtools/travis-ci/install_miniconda.sh From e006a4686f042d213b706ac5d7ccff55e333c093 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 14 Nov 2020 18:23:26 +0000 Subject: [PATCH 04/19] updated miniconda repo --- devtools/travis-ci/install_miniconda.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/devtools/travis-ci/install_miniconda.sh b/devtools/travis-ci/install_miniconda.sh index 0b5c760..0a3ff1b 100755 --- a/devtools/travis-ci/install_miniconda.sh +++ b/devtools/travis-ci/install_miniconda.sh @@ -1,7 +1,9 @@ #!/bin/bash MINICONDA=Miniconda3-latest-Linux-x86_64.sh -MINICONDA_MD5=$(curl -s https://repo.continuum.io/miniconda/ | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *\(.*\)<\/td> */\1/p') -wget https://repo.continuum.io/miniconda/$MINICONDA +MINICONDA_REPO=https://repo.anaconda.com/miniconda +MINICONDA_MD5=$(curl -s $MINICONDA_REPO | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *\(.*\)<\/td> */\1/p') + +wget $MINICONDA_REPO/$MINICONDA if [[ $MINICONDA_MD5 != $(md5sum $MINICONDA | cut -d ' ' -f 1) ]]; then echo "Miniconda MD5 mismatch" exit 1 From d2a86d40430eebfcf36e85143792646643d42a69 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 14 Nov 2020 18:31:34 +0000 Subject: [PATCH 05/19] typo in repo url --- devtools/travis-ci/install_miniconda.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/devtools/travis-ci/install_miniconda.sh b/devtools/travis-ci/install_miniconda.sh index 0a3ff1b..f68231f 100755 --- a/devtools/travis-ci/install_miniconda.sh +++ b/devtools/travis-ci/install_miniconda.sh @@ -1,8 +1,7 @@ #!/bin/bash MINICONDA=Miniconda3-latest-Linux-x86_64.sh MINICONDA_REPO=https://repo.anaconda.com/miniconda -MINICONDA_MD5=$(curl -s $MINICONDA_REPO | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *\(.*\)<\/td> */\1/p') - +MINICONDA_MD5=$(curl $MINICONDA_REPO/ | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *\(.*\)<\/td> */\1/p') wget $MINICONDA_REPO/$MINICONDA if [[ $MINICONDA_MD5 != $(md5sum $MINICONDA | cut -d ' ' -f 1) ]]; then echo "Miniconda MD5 mismatch" From 206de1d5aa1b95015f1b8dd70da0e7e98910a400 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 14 Nov 2020 18:47:13 +0000 Subject: [PATCH 06/19] added 3.7 and 3.8 to py matrix --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 52dd913..cc1aa3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,8 @@ env: matrix: - CONDA_PY=3.5 - CONDA_PY=3.6 - + - CONDA_PY=3.7 + - CONDA_PY=3.8 branches: only: - Development From dac6204504da7110283d7d69a38b3b0c87583df9 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 28 Nov 2020 14:39:40 +0000 Subject: [PATCH 07/19] using system joblib instead of sklearn. numpy decorators updated. Dropback to old methods still in place --- osprey/dataset_loaders.py | 2 +- osprey/fit_estimator.py | 7 ++++++- osprey/plugins/tests/test_plugin_pylearn2.py | 7 ++++++- osprey/tests/test_cli_worker_and_dump.py | 8 +++++++- osprey/tests/test_dataset_loader.py | 13 +++++++++++-- osprey/tests/test_search_space.py | 8 +++++++- osprey/tests/test_strategies.py | 7 ++++++- 7 files changed, 44 insertions(+), 8 deletions(-) diff --git a/osprey/dataset_loaders.py b/osprey/dataset_loaders.py index db45da4..2407dc4 100644 --- a/osprey/dataset_loaders.py +++ b/osprey/dataset_loaders.py @@ -219,7 +219,7 @@ class JoblibDatasetLoader(BaseDatasetLoader): short_name = 'joblib' def __init__(self, filenames, x_name=None, y_name=None, - system_joblib=False): + system_joblib=True): self.filenames = filenames self.x_name = x_name self.y_name = y_name diff --git a/osprey/fit_estimator.py b/osprey/fit_estimator.py index 7696473..8f3b4fd 100644 --- a/osprey/fit_estimator.py +++ b/osprey/fit_estimator.py @@ -7,7 +7,12 @@ import sklearn from sklearn.base import is_classifier, clone from sklearn.metrics.scorer import check_scoring -from sklearn.externals.joblib import Parallel, delayed +# TODO: Remove old import? externals.joblib deprecated as of 0.23 +try: + from joblib import Parallel, delayed +except ModuleNotFoundError: + from sklearn.externals.joblib import Parallel, delayed + from sklearn.model_selection import check_cv from sklearn.model_selection._validation import _safe_split, _score diff --git a/osprey/plugins/tests/test_plugin_pylearn2.py b/osprey/plugins/tests/test_plugin_pylearn2.py index 6c48a6b..99bde52 100644 --- a/osprey/plugins/tests/test_plugin_pylearn2.py +++ b/osprey/plugins/tests/test_plugin_pylearn2.py @@ -6,7 +6,12 @@ import unittest from argparse import Namespace import numpy as np -from numpy.testing.decorators import skipif +# TODO remove compat with py<=3.6 +try: + from numpy.testing import dec + skipif = dec.skipif +except ModuleNotFoundError: + from numpy.testing.decorators import skipif try: import pylearn2 diff --git a/osprey/tests/test_cli_worker_and_dump.py b/osprey/tests/test_cli_worker_and_dump.py index 47a0ca0..89f0e56 100644 --- a/osprey/tests/test_cli_worker_and_dump.py +++ b/osprey/tests/test_cli_worker_and_dump.py @@ -7,7 +7,13 @@ import subprocess import tempfile from distutils.spawn import find_executable -from numpy.testing.decorators import skipif +# TODO remove compat with py<=3.6 +try: + from numpy.testing import dec + skipif = dec.skipif +except ModuleNotFoundError: + from numpy.testing.decorators import skipif + from nose.plugins.skip import SkipTest diff --git a/osprey/tests/test_dataset_loader.py b/osprey/tests/test_dataset_loader.py index ee55e9e..2d38b4b 100644 --- a/osprey/tests/test_dataset_loader.py +++ b/osprey/tests/test_dataset_loader.py @@ -6,8 +6,17 @@ import numpy as np import sklearn.datasets -from sklearn.externals.joblib import dump -from numpy.testing.decorators import skipif +# TODO remove old import? +try: + from joblib import dump +except ModuleNotFoundError: + from sklearn.externals.joblib import dump +# TODO remove compat with py<=3.6 +try: + from numpy.testing import dec + skipif = dec.skipif +except ModuleNotFoundError: + from numpy.testing.decorators import skipif from osprey.dataset_loaders import (DSVDatasetLoader, FilenameDatasetLoader, JoblibDatasetLoader, HDF5DatasetLoader, diff --git a/osprey/tests/test_search_space.py b/osprey/tests/test_search_space.py index 945a85f..4cecd9b 100644 --- a/osprey/tests/test_search_space.py +++ b/osprey/tests/test_search_space.py @@ -4,7 +4,13 @@ import numpy as np import scipy.stats from six.moves import xrange -from numpy.testing.decorators import skipif +# backwards compat with python <=3.6 +try: + from numpy.testing import dec + skipif = dec.skipif +except ModuleNotFoundError: + from numpy.testing.decorators import skipif + try: from hyperopt import pyll except ImportError: diff --git a/osprey/tests/test_strategies.py b/osprey/tests/test_strategies.py index eb357d7..1cb5be1 100644 --- a/osprey/tests/test_strategies.py +++ b/osprey/tests/test_strategies.py @@ -3,7 +3,12 @@ import sys from six import iteritems import numpy as np -from numpy.testing.decorators import skipif +# TODO remove compat with py<=3.6 +try: + from numpy.testing import dec + skipif = dec.skipif +except ModuleNotFoundError: + from numpy.testing.decorators import skipif from osprey.search_space import SearchSpace from osprey.search_space import IntVariable, EnumVariable, FloatVariable From 6b6f37b5b76437d67a6e7acd952f76314cb04f08 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 28 Nov 2020 14:51:22 +0000 Subject: [PATCH 08/19] updated yaml.load call to include safeloader --- osprey/config.py | 2 +- osprey/tests/test_cli_skeleton.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osprey/config.py b/osprey/config.py index edf55b2..f533cd8 100644 --- a/osprey/config.py +++ b/osprey/config.py @@ -372,7 +372,7 @@ def sha1(self): def parse(f): - res = yaml.load(f) + res = yaml.load(f, Loader=yaml.SafeLoader) if res is None: res = {} diff --git a/osprey/tests/test_cli_skeleton.py b/osprey/tests/test_cli_skeleton.py index eaedfa2..45e0402 100644 --- a/osprey/tests/test_cli_skeleton.py +++ b/osprey/tests/test_cli_skeleton.py @@ -22,7 +22,7 @@ def test_1(): '-f', 'config.yaml']) assert os.path.exists('config.yaml') with open('config.yaml', 'rb') as f: - yaml.load(f) + yaml.load(f, Loader=yaml.SafeLoader) Config('config.yaml') finally: From b4ae4c8825bb31c29e231fc6833d24d78c9a05a2 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 28 Nov 2020 15:30:09 +0000 Subject: [PATCH 09/19] replaced ix with loc in dataframe subsetting --- osprey/plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osprey/plot.py b/osprey/plot.py index 6e3ee78..6398bc7 100644 --- a/osprey/plot.py +++ b/osprey/plot.py @@ -24,7 +24,7 @@ def nonconstant_parameters(data): assert len(data) > 0 df = pd.DataFrame([d['parameters'] for d in data]) # http://stackoverflow.com/a/20210048/1079728 - filtered = df.loc[:, (df != df.ix[0]).any()] + filtered = df.loc[:, (df != df.iloc[0]).any()] return filtered From 7a2b603c4e2ba850a6b1f637ceb92cdc8ee556e4 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 28 Nov 2020 15:31:27 +0000 Subject: [PATCH 10/19] increased samples in regression test as it wasn't passing --- osprey/tests/test_fit_estimator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osprey/tests/test_fit_estimator.py b/osprey/tests/test_fit_estimator.py index 607bc53..5ae9a8a 100644 --- a/osprey/tests/test_fit_estimator.py +++ b/osprey/tests/test_fit_estimator.py @@ -11,7 +11,7 @@ def test_1(): - X, y = make_regression(n_features=10) + X, y = make_regression(n_samples=300, n_features=10) lasso = Lasso() params = {'alpha': 2} From d12d1a928bbd9e1b18ad028033a2443b40506463 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 28 Nov 2020 15:39:10 +0000 Subject: [PATCH 11/19] updated build receipe to stop msmbuilder --- .travis.yml | 7 ++++--- devtools/conda-recipe/meta.yaml | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc1aa3a..c631476 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,14 +4,15 @@ sudo: false env: matrix: - - CONDA_PY=3.5 +# - CONDA_PY=3.5 - CONDA_PY=3.6 - CONDA_PY=3.7 - - CONDA_PY=3.8 +# - CONDA_PY=3.8 branches: only: - Development - - master + - update_to_py37 +# - master install: - source devtools/travis-ci/install_miniconda.sh diff --git a/devtools/conda-recipe/meta.yaml b/devtools/conda-recipe/meta.yaml index 6492571..0d198be 100644 --- a/devtools/conda-recipe/meta.yaml +++ b/devtools/conda-recipe/meta.yaml @@ -26,6 +26,7 @@ requirements: - bokeh - matplotlib - pandas + - joblib test: @@ -33,8 +34,8 @@ test: - nose - nose-timer - gpy - - msmbuilder - - msmb_data +# - msmbuilder +# - msmb_data - mdtraj - hyperopt - coverage From ff3b9442d67faa5f21239eb85aab613d9ee07ea1 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 28 Nov 2020 20:29:26 +0000 Subject: [PATCH 12/19] added test for pyemma estimator --- osprey/fit_estimator.py | 11 ++++++----- osprey/tests/test_fit_estimator.py | 17 +++++++++++++++++ osprey/utils.py | 28 ++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/osprey/fit_estimator.py b/osprey/fit_estimator.py index 8f3b4fd..89257da 100644 --- a/osprey/fit_estimator.py +++ b/osprey/fit_estimator.py @@ -17,7 +17,7 @@ from sklearn.model_selection._validation import _safe_split, _score from .utils import check_arrays, num_samples -from .utils import short_format_time, is_msmbuilder_estimator +from .utils import short_format_time, is_msmbuilder_estimator, is_pyemma_estimator if LooseVersion(sklearn.__version__) < LooseVersion('0.16.1'): @@ -45,7 +45,6 @@ def fit_and_score_estimator(estimator, parameters, cv, X, y=None, scoring=None, The scores on the training and test sets, as well as the mean test set score. """ - scorer = check_scoring(estimator, scoring=scoring) n_samples = num_samples(X) X, y = check_arrays(X, y, allow_lists=True, sparse_format='csr', @@ -141,9 +140,11 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, scoring_time = time.time() - start_time - msmbuilder_api = is_msmbuilder_estimator(estimator) - n_samples_test = num_samples(X_test, is_nested=msmbuilder_api) - n_samples_train = num_samples(X_train, is_nested=msmbuilder_api) + # With PyEMMA and MSMBuilder, observations are nested in separate lists not just in a single + # array. + is_nested = is_msmbuilder_estimator(estimator) or is_pyemma_estimator(estimator) + n_samples_test = num_samples(X_test, is_nested=is_nested) + n_samples_train = num_samples(X_train, is_nested=is_nested) if verbose > 2: msg += ", score=%f" % test_score if verbose > 1: diff --git a/osprey/tests/test_fit_estimator.py b/osprey/tests/test_fit_estimator.py index 5ae9a8a..4892099 100644 --- a/osprey/tests/test_fit_estimator.py +++ b/osprey/tests/test_fit_estimator.py @@ -44,3 +44,20 @@ def test_2(): verbose=0) np.testing.assert_array_equal(out['n_train_samples'], [11, 10]) np.testing.assert_array_equal(out['n_test_samples'], [10, 11]) + + +def test_3(): + try: + from pyemma.msm import MaximumLikelihoodMSM + except ImportError as e: + raise SkipTest(e) + + X = [np.random.randint(2, size=10), np.random.randint(2, size=11)] + out = fit_and_score_estimator(MaximumLikelihoodMSM(), {'score_k': 2}, + cv=2, + X=X, + y=None, + verbose=0) + # This shouldn't pass! + np.testing.assert_array_equal(out['n_train_samples'], [11, 10]) + np.testing.assert_array_equal(out['n_test_samples'], [10, 11]) diff --git a/osprey/utils.py b/osprey/utils.py index b4dee9c..685ec92 100644 --- a/osprey/utils.py +++ b/osprey/utils.py @@ -166,18 +166,34 @@ def expand_path(path, base='.'): return path +def estimator_in_package(estimator, package): + """ + checks to see if estimator is in package. If estimator is a skl Pipeline + then it checks each step + """ + package_estimators = import_all_estimators(package).values() + + out = estimator.__class__ in package_estimators + if isinstance(estimator, Pipeline): + out = any(step.__class__ in package_estimators + for name, step in estimator.steps) + return out + + def is_msmbuilder_estimator(estimator): try: import msmbuilder except ImportError: return False - msmbuilder_estimators = import_all_estimators(msmbuilder).values() + return estimator_in_package(estimator, msmbuilder) - out = estimator.__class__ in msmbuilder_estimators - if isinstance(estimator, Pipeline): - out = any(step.__class__ in msmbuilder_estimators - for name, step in estimator.steps) - return out + +def is_pyemma_estimator(estimator): + try: + import pyemma + except ImportError: + return False + return estimator_in_package(estimator, pyemma) def _assert_all_finite(X): From fafe4b829d6dae44a59ecfa5677950e516623ff2 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 28 Nov 2020 20:44:37 +0000 Subject: [PATCH 13/19] msmbuilder tests skipped properly --- osprey/tests/test_dataset_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osprey/tests/test_dataset_loader.py b/osprey/tests/test_dataset_loader.py index 2d38b4b..a2d6ece 100644 --- a/osprey/tests/test_dataset_loader.py +++ b/osprey/tests/test_dataset_loader.py @@ -186,7 +186,7 @@ def test_MDTrajDatasetLoader_1(): finally: shutil.rmtree(dirname) - +@skipif(not HAVE_MSMBUILDER, 'this test requires MSMBuilder') def test_MSMBuilderDatasetLoader_1(): # TODO Why does this work when other msmbuilder imports don't? from msmbuilder.dataset import dataset From 4842b8ada06d93b7d57a158bff21307898be86d4 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sat, 28 Nov 2020 21:06:44 +0000 Subject: [PATCH 14/19] updated sklearn import --- osprey/fit_estimator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/osprey/fit_estimator.py b/osprey/fit_estimator.py index 89257da..d6ec4c9 100644 --- a/osprey/fit_estimator.py +++ b/osprey/fit_estimator.py @@ -6,7 +6,10 @@ import numpy as np import sklearn from sklearn.base import is_classifier, clone -from sklearn.metrics.scorer import check_scoring +try: + from sklearn.metrics import check_scoring +except ModuleNotFoundError: + from sklearn.metrics.scorer import check_scoring # TODO: Remove old import? externals.joblib deprecated as of 0.23 try: from joblib import Parallel, delayed From 1eed06b92ca043f95f8ffbb9bca20a504de47564 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sun, 29 Nov 2020 10:41:08 +0000 Subject: [PATCH 15/19] test cli skeleton uses sklearn not msmbuilder --- osprey/tests/test_cli_skeleton.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osprey/tests/test_cli_skeleton.py b/osprey/tests/test_cli_skeleton.py index 45e0402..20ef85d 100644 --- a/osprey/tests/test_cli_skeleton.py +++ b/osprey/tests/test_cli_skeleton.py @@ -18,7 +18,7 @@ def test_1(): try: os.chdir(dirname) - subprocess.check_call([OSPREY_BIN, 'skeleton', '-t', 'msmbuilder', + subprocess.check_call([OSPREY_BIN, 'skeleton', '-t', 'sklearn', '-f', 'config.yaml']) assert os.path.exists('config.yaml') with open('config.yaml', 'rb') as f: From f440f9d2362b9d869d310e61a1c1a768352c7ddb Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sun, 29 Nov 2020 10:44:38 +0000 Subject: [PATCH 16/19] skipping msmbuilder cli_worker_and_dump tests if msmbuilder not installed --- osprey/tests/test_cli_worker_and_dump.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osprey/tests/test_cli_worker_and_dump.py b/osprey/tests/test_cli_worker_and_dump.py index 89f0e56..6d8928b 100644 --- a/osprey/tests/test_cli_worker_and_dump.py +++ b/osprey/tests/test_cli_worker_and_dump.py @@ -26,7 +26,7 @@ OSPREY_BIN = find_executable('osprey') -# @skipif(not HAVE_MSMBUILDER, 'this test requires MSMBuilder') +@skipif(not HAVE_MSMBUILDER, 'this test requires MSMBuilder') def test_msmbuilder_skeleton(): try: from msmbuilder.example_datasets import FsPeptide @@ -54,7 +54,7 @@ def test_msmbuilder_skeleton(): shutil.rmtree(dirname) -# @skipif(not HAVE_MSMBUILDER, 'this test requires MSMBuilder') +@skipif(not HAVE_MSMBUILDER, 'this test requires MSMBuilder') def test_msmb_feat_select_skeleton(): try: from msmbuilder.example_datasets import FsPeptide From bfc322f46c53976640684358bd9e10f6f8199e43 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sun, 29 Nov 2020 12:01:17 +0000 Subject: [PATCH 17/19] tests now test for optional imports, and msmbuilder now optional --- osprey/tests/test_cli_worker_and_dump.py | 22 ++++++++-------- osprey/tests/test_config.py | 20 +++++++++------ osprey/tests/test_fit_estimator.py | 32 ++++++++++++++++-------- osprey/tests/test_search_space.py | 20 +++++++++------ osprey/tests/test_strategies.py | 17 ++++++++++--- osprey/tests/test_trials.py | 1 + osprey/tests/test_utils.py | 2 ++ 7 files changed, 75 insertions(+), 39 deletions(-) diff --git a/osprey/tests/test_cli_worker_and_dump.py b/osprey/tests/test_cli_worker_and_dump.py index 6d8928b..d9ec606 100644 --- a/osprey/tests/test_cli_worker_and_dump.py +++ b/osprey/tests/test_cli_worker_and_dump.py @@ -14,9 +14,9 @@ except ModuleNotFoundError: from numpy.testing.decorators import skipif -from nose.plugins.skip import SkipTest - +# from nose.plugins.skip import SkipTest +# TODO need Pyemma analogues of these tests. try: __import__('msmbuilder.example_datasets') HAVE_MSMBUILDER = True @@ -28,10 +28,11 @@ @skipif(not HAVE_MSMBUILDER, 'this test requires MSMBuilder') def test_msmbuilder_skeleton(): - try: - from msmbuilder.example_datasets import FsPeptide - except ImportError as e: - raise SkipTest(e) + from msmbuilder.example_datasets import FsPeptide + # try: + # from msmbuilder.example_datasets import FsPeptide + # except ImportError as e: + # raise SkipTest(e) assert OSPREY_BIN is not None cwd = os.path.abspath(os.curdir) @@ -56,10 +57,11 @@ def test_msmbuilder_skeleton(): @skipif(not HAVE_MSMBUILDER, 'this test requires MSMBuilder') def test_msmb_feat_select_skeleton(): - try: - from msmbuilder.example_datasets import FsPeptide - except ImportError as e: - raise SkipTest(e) + from msmbuilder.example_datasets import FsPeptide + # try: + # from msmbuilder.example_datasets import FsPeptide + # except ImportError as e: + # raise SkipTest(e) assert OSPREY_BIN is not None cwd = os.path.abspath(os.curdir) diff --git a/osprey/tests/test_config.py b/osprey/tests/test_config.py index 57eccb9..dd3090e 100644 --- a/osprey/tests/test_config.py +++ b/osprey/tests/test_config.py @@ -14,6 +14,7 @@ from osprey.search_space import IntVariable, FloatVariable, EnumVariable from osprey.strategies import RandomSearch, HyperoptTPE, Bayes +import warnings os.environ['OSPREYRC'] = ' ' @@ -24,21 +25,24 @@ def test_estimator_pickle(): with tempfile.NamedTemporaryFile('w+b', 0) as f: cPickle.dump(KMeans(), f) - config = Config.fromdict({ 'estimator': {'pickle': f.name} }, check_fields=False) + assert isinstance(config.estimator(), KMeans) def test_estimator_eval_1(): - config = Config.fromdict({ - 'estimator': { - 'eval': 'KMeans()', - 'eval_scope': 'sklearn', - } - }, check_fields=False) - assert isinstance(config.estimator(), KMeans) + # TODO: why is this the only call to sklearn that produces the FutureWarnings? What about eval_1? + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=FutureWarning) + config = Config.fromdict({ + 'estimator': { + 'eval': 'KMeans()', + 'eval_scope': 'sklearn', + } + }, check_fields=False) + assert isinstance(config.estimator(), KMeans) def test_estimator_eval_2(): diff --git a/osprey/tests/test_fit_estimator.py b/osprey/tests/test_fit_estimator.py index 4892099..d2b8f1c 100644 --- a/osprey/tests/test_fit_estimator.py +++ b/osprey/tests/test_fit_estimator.py @@ -9,6 +9,24 @@ from osprey.fit_estimator import fit_and_score_estimator +try: + __import__('msmbuilder.msm') + HAVE_MSMBUILDER = True +except: + HAVE_MSMBUILDER = False + +try: + __import__('pyemma.msm') + HAVE_PYEMMA = True +except: + HAVE_PYEMMA = False + +# TODO remove compat with py<=3.6 +try: + from numpy.testing import dec + skipif = dec.skipif +except ModuleNotFoundError: + from numpy.testing.decorators import skipif def test_1(): X, y = make_regression(n_samples=300, n_features=10) @@ -29,12 +47,9 @@ def test_1(): [g.cv_results_['split{}_test_score'.format(i)] for i in range(cv)]) assert np.all(out['test_scores'] == test_scores) - +@skipif(not HAVE_MSMBUILDER, "this test requires MSMBuilder") def test_2(): - try: - from msmbuilder.msm import MarkovStateModel - except ImportError as e: - raise SkipTest(e) + from msmbuilder.msm import MarkovStateModel X = [np.random.randint(2, size=10), np.random.randint(2, size=11)] out = fit_and_score_estimator(MarkovStateModel(), {'verbose': False}, @@ -46,11 +61,9 @@ def test_2(): np.testing.assert_array_equal(out['n_test_samples'], [10, 11]) +@skipif(not HAVE_PYEMMA, "this test requires PyEMMA") def test_3(): - try: - from pyemma.msm import MaximumLikelihoodMSM - except ImportError as e: - raise SkipTest(e) + from pyemma.msm import MaximumLikelihoodMSM X = [np.random.randint(2, size=10), np.random.randint(2, size=11)] out = fit_and_score_estimator(MaximumLikelihoodMSM(), {'score_k': 2}, @@ -58,6 +71,5 @@ def test_3(): X=X, y=None, verbose=0) - # This shouldn't pass! np.testing.assert_array_equal(out['n_train_samples'], [11, 10]) np.testing.assert_array_equal(out['n_test_samples'], [10, 11]) diff --git a/osprey/tests/test_search_space.py b/osprey/tests/test_search_space.py index 4cecd9b..25d42d0 100644 --- a/osprey/tests/test_search_space.py +++ b/osprey/tests/test_search_space.py @@ -13,8 +13,9 @@ try: from hyperopt import pyll + HAVE_HYPEROPT = True except ImportError: - pass + HAVE_HYPEROPT = False from osprey.search_space import SearchSpace from osprey.search_space import IntVariable, FloatVariable, EnumVariable @@ -56,8 +57,9 @@ def test_2_1(): values = [s.rvs()['a'] for _ in xrange(100)] _run_chi2_test(values, bin_edges=range(5)) - -@skipif('hyperopt.pyll' not in sys.modules, 'this test requires hyperopt') +# TODO : why does old skipif check in sys.modules rather than test import? +# @skipif('hyperopt.pyll' not in sys.modules, 'this test requires hyperopt') +@skipif(not HAVE_HYPEROPT, "this test requires hyperopt") def test_2_2(): s = SearchSpace() s.add_int('a', 0, 3) @@ -74,8 +76,8 @@ def test_3_1(): assert all(-2 < v < 2 for v in values) _run_chi2_test(values, bin_edges=np.linspace(-2, 2, 10)) - -@skipif('hyperopt.pyll' not in sys.modules, 'this test requires hyperopt') +# @skipif('hyperopt.pyll' not in sys.modules, 'this test requires hyperopt') +@skipif(not HAVE_HYPEROPT, "this test requires hyperopt") def test_3_2(): s = SearchSpace() s.add_float('b', -2, 2) @@ -94,8 +96,8 @@ def test_4_1(): assert all(v in [True, False] for v in values) _run_chi2_test(np.array(values, dtype=int), bin_edges=range(3)) - -@skipif('hyperopt.pyll' not in sys.modules, 'this test requires hyperopt') +# @skipif('hyperopt.pyll' not in sys.modules, 'this test requires hyperopt') +@skipif(not HAVE_HYPEROPT, "this test requires hyperopt") def test_4_2(): s = SearchSpace() s.add_enum('c', [True, False]) @@ -120,7 +122,8 @@ def test_5_1(): _run_chi2_test(values, bin_edges) -@skipif('hyperopt.pyll' not in sys.modules, 'this test requires hyperopt') +# @skipif('hyperopt.pyll' not in sys.modules, 'this test requires hyperopt') +@skipif(not HAVE_HYPEROPT, "this test requires hyperopt") def test_5_2(): s = SearchSpace() s.add_float('a', 1e-5, 1, warp='log') @@ -163,6 +166,7 @@ def test_gp_3(): assert 10.0 == v.point_from_unit(v.point_to_unit(10.1)) +# TODO : enum variables should map to corners of hypercube def test_gp_4(): v = EnumVariable('name', ['a', 'b', 'c']) assert 'a' == v.point_from_unit(v.point_to_unit('a')) diff --git a/osprey/tests/test_strategies.py b/osprey/tests/test_strategies.py index 1cb5be1..ce8b687 100644 --- a/osprey/tests/test_strategies.py +++ b/osprey/tests/test_strategies.py @@ -16,8 +16,15 @@ try: from hyperopt import hp, fmin, tpe, Trials + HAVE_HYPEROPT = True except: - pass + HAVE_HYPEROPT = False + +try: + import gpy + HAVE_GPY = True +except: + HAVE_GPY = False def test_random(): @@ -65,6 +72,7 @@ def test_check_repeated_params(): assert not grid_search4.is_repeated_suggestion(params, history) +@skipif(not HAVE_HYPEROPT, "this test requires Hyperopt") def hyperopt_x2_iterates(n_iters=100): iterates = [] trials = Trials() @@ -82,6 +90,7 @@ def fn(params): return np.array(iterates) +@skipif(not HAVE_HYPEROPT, "this test requires Hyperopt") def our_x2_iterates(n_iters=100): history = [] searchspace = SearchSpace() @@ -100,7 +109,8 @@ def fn(params): return np.array([h[0]['x'] for h in history]) -@skipif('hyperopt.fmin' not in sys.modules, 'this test requires hyperopt') +# @skipif('hyperopt.fmin' not in sys.modules, 'this test requires hyperopt') +@skipif(not HAVE_HYPEROPT, "this test requires Hyperopt") def test_1(): ours = our_x2_iterates(25) ref = hyperopt_x2_iterates(25) @@ -108,7 +118,8 @@ def test_1(): np.testing.assert_array_equal(ref, ours) -@skipif('GPy' not in sys.modules, 'this test requires GPy') +# @skipif('GPy' not in sys.modules, 'this test requires GPy') +@skipif(not HAVE_GPY, "this test requires GPy") def test_gp(): searchspace = SearchSpace() searchspace.add_float('x', -10, 10) diff --git a/osprey/tests/test_trials.py b/osprey/tests/test_trials.py index 8d10a5d..2d58ef8 100644 --- a/osprey/tests/test_trials.py +++ b/osprey/tests/test_trials.py @@ -7,6 +7,7 @@ from osprey.trials import make_session, Trial +# TODO : create test that pulls single project from db of multiple projects def test_1(): cwd = os.path.abspath(os.curdir) diff --git a/osprey/tests/test_utils.py b/osprey/tests/test_utils.py index 02fee63..ffbd8bd 100644 --- a/osprey/tests/test_utils.py +++ b/osprey/tests/test_utils.py @@ -9,6 +9,7 @@ import numpy as np from sklearn.base import BaseEstimator + def test_is_json_serializable(): a = BaseEstimator() b = np.array([1, 2, 3], dtype=int) @@ -21,6 +22,7 @@ def test_is_json_serializable(): for obj in [e, f, g, h, i]: assert(is_json_serializable(obj)) + def test_dict_merge_1(): base = {'a': 1} top = {'a': 2} From 9bf9f955abfa31e6bbb8dfb0bbbc2cd0dcc740c2 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sun, 29 Nov 2020 12:06:19 +0000 Subject: [PATCH 18/19] build receip doesn't inlcude msmbuilder in test --- devtools/conda-recipe/meta.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/devtools/conda-recipe/meta.yaml b/devtools/conda-recipe/meta.yaml index 0d198be..193b11d 100644 --- a/devtools/conda-recipe/meta.yaml +++ b/devtools/conda-recipe/meta.yaml @@ -34,8 +34,6 @@ test: - nose - nose-timer - gpy -# - msmbuilder -# - msmb_data - mdtraj - hyperopt - coverage From a564309a95b54dd82ad915c19281730faf8591b5 Mon Sep 17 00:00:00 2001 From: rob arbon Date: Sun, 29 Nov 2020 12:09:34 +0000 Subject: [PATCH 19/19] updated changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d7825c..27aead7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,7 +20,7 @@ New Features + Added ``n_jobs`` flag for ``osprey worker`` to control how many threads are used for cross-validation. + Added the ability to specify three different acquisition functions for the gaussian processes strategy: expected improvement `ei`, upper confidence bound, `ucb` and the original Osprey function (the default), `osprey`. + Refactored code: GP strategy has been replaced by a general Bayes strategy. This will eventually allow different surrogate functions, other than Gaussian Process, to be specified. - ++ Update Python compatability: Now compatable with Python 3.7. MSMBuilder is now an optional. Bug Fixes ~~~~~~~~~