From 2712cb022b6868b3f7c42bc34bf85d5181f81a82 Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 12 Oct 2022 10:18:49 +0100 Subject: [PATCH 1/5] change install to check_package --- scivision/io/__init__.py | 2 +- scivision/io/installer.py | 26 +++++--------------------- scivision/io/reader.py | 13 +++++-------- 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/scivision/io/__init__.py b/scivision/io/__init__.py index af07c2c0..477af772 100644 --- a/scivision/io/__init__.py +++ b/scivision/io/__init__.py @@ -1,3 +1,3 @@ from .reader import load_pretrained_model, load_dataset, _parse_url, _get_model_configs -from .installer import install_package, package_from_config +from .installer import check_package from .wrapper import PretrainedModel, Datasource \ No newline at end of file diff --git a/scivision/io/installer.py b/scivision/io/installer.py index 1ede7618..cfc91ae9 100644 --- a/scivision/io/installer.py +++ b/scivision/io/installer.py @@ -16,7 +16,7 @@ def _package_exists(config: dict) -> bool: return True -def package_from_config(config: dict, branch: str = "main") -> str: +def _package_from_config(config: dict, branch: str = "main") -> str: """Given a config return the pip install string.""" install_str = config["url"] if install_str.endswith(".git"): @@ -25,31 +25,15 @@ def package_from_config(config: dict, branch: str = "main") -> str: return f"git+{install_str}@{install_branch}#egg={config['import']}" -def _install(package, pip_install_args=None): - """Install a package using pip.""" - - if pip_install_args is None: - pip_install_args = [] - - subprocess.check_call( - [sys.executable, "-m", "pip", "install", *pip_install_args, package] - ) - - -def install_package( +def check_package( config: dict, - allow_install=False, # allowed values: True, False, or the string "force" branch: str = "main", ): - """Install the python package if it doesn't exist.""" - package = package_from_config(config, branch) + """Check if the python package exists.""" + package = _package_from_config(config, branch) exists = _package_exists(config) - if allow_install == "force" or (allow_install and not exists): - # if a package is not already installed, there is little harm - # to passing the extra arguments, so these cases are combined - _install(package, pip_install_args=["--force-reinstall", "--no-cache-dir"]) - elif not exists: + if not exists: raise Exception( "Package does not exist. Try installing it with: \n" f"`!pip install -e {package}`" diff --git a/scivision/io/reader.py b/scivision/io/reader.py index 18d09008..833a9228 100644 --- a/scivision/io/reader.py +++ b/scivision/io/reader.py @@ -9,7 +9,7 @@ import yaml from ..koala import koala -from .installer import install_package +from .installer import check_package from .wrapper import PretrainedModel, Datasource import warnings @@ -120,7 +120,6 @@ def _get_model_configs( def load_pretrained_model( path: os.PathLike, branch: str = "main", - allow_install: bool = False, model_selection: str = "default", load_multiple: bool = False, *args, @@ -134,8 +133,6 @@ def load_pretrained_model( The filename, path or URL of a pretrained model description. branch : str, default = main Specify the name of a github branch if loading from github. - allow_install : bool, default = False - Allow installation of remote package via pip. model_selection : str, default = default Specify the name of the model if there is > 1 in the model repo package. load_multiple : bool, default = False @@ -170,8 +167,8 @@ def load_pretrained_model( # make sure a model at least has an input to the function assert "X" in config["prediction_fn"]["args"].keys() - # try to install the package if necessary - install_package(config, allow_install=allow_install, branch=branch) + # Raise exception if package not installed and suggest how + check_package(config, branch=branch) loaded_models.append(PretrainedModel(config)) if load_multiple: @@ -245,7 +242,7 @@ def load_data_from_plugin( The dataset to be visualised, loaded via xarray. """ - # install the package - install_package(config, allow_install=True, branch=branch) + # Raise exception if package not installed and suggest how + check_package(config, branch=branch) return Datasource(config) From 7ec4b46da97767027e6a6afb641ad65d77cc330c Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 12 Oct 2022 10:19:36 +0100 Subject: [PATCH 2/5] rename installer to checker --- scivision/io/__init__.py | 2 +- scivision/io/{installer.py => checker.py} | 0 scivision/io/reader.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename scivision/io/{installer.py => checker.py} (100%) diff --git a/scivision/io/__init__.py b/scivision/io/__init__.py index 477af772..597b0513 100644 --- a/scivision/io/__init__.py +++ b/scivision/io/__init__.py @@ -1,3 +1,3 @@ from .reader import load_pretrained_model, load_dataset, _parse_url, _get_model_configs -from .installer import check_package +from .checker import check_package from .wrapper import PretrainedModel, Datasource \ No newline at end of file diff --git a/scivision/io/installer.py b/scivision/io/checker.py similarity index 100% rename from scivision/io/installer.py rename to scivision/io/checker.py diff --git a/scivision/io/reader.py b/scivision/io/reader.py index 833a9228..f15e3e09 100644 --- a/scivision/io/reader.py +++ b/scivision/io/reader.py @@ -9,7 +9,7 @@ import yaml from ..koala import koala -from .installer import check_package +from .checker import check_package from .wrapper import PretrainedModel, Datasource import warnings From 5b16b462ac5dc18159fc53cd6b1a0091dec3df9e Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 12 Oct 2022 10:30:03 +0100 Subject: [PATCH 3/5] update so tests run --- scivision/io/__init__.py | 2 +- scivision/io/checker.py | 4 ++-- tests/conftest.py | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scivision/io/__init__.py b/scivision/io/__init__.py index 597b0513..7f5c9f66 100644 --- a/scivision/io/__init__.py +++ b/scivision/io/__init__.py @@ -1,3 +1,3 @@ from .reader import load_pretrained_model, load_dataset, _parse_url, _get_model_configs -from .checker import check_package +from .checker import check_package, package_from_config from .wrapper import PretrainedModel, Datasource \ No newline at end of file diff --git a/scivision/io/checker.py b/scivision/io/checker.py index cfc91ae9..16406f48 100644 --- a/scivision/io/checker.py +++ b/scivision/io/checker.py @@ -16,7 +16,7 @@ def _package_exists(config: dict) -> bool: return True -def _package_from_config(config: dict, branch: str = "main") -> str: +def package_from_config(config: dict, branch: str = "main") -> str: """Given a config return the pip install string.""" install_str = config["url"] if install_str.endswith(".git"): @@ -30,7 +30,7 @@ def check_package( branch: str = "main", ): """Check if the python package exists.""" - package = _package_from_config(config, branch) + package = package_from_config(config, branch) exists = _package_exists(config) if not exists: diff --git a/tests/conftest.py b/tests/conftest.py index 3cb115d9..92ef2166 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ -from scivision.io import install_package +from scivision.io import package_from_config from scivision.io import PretrainedModel +import subprocess +import sys import fsspec import yaml import pytest @@ -31,4 +33,4 @@ def KOALA(request): # Install the model package so it can be used in tests -install_package(imagenet_model_config, allow_install=True) +subprocess.check_call([sys.executable, "-m", "pip", "install", package_from_config(imagenet_model_config)]) From 4493bbc338faff167dc25757578e8c10a8a7f70a Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 12 Oct 2022 10:35:40 +0100 Subject: [PATCH 4/5] update so data plugin test still works --- tests/conftest.py | 11 +++++++++++ tests/test_data_plugin.yml | 8 ++++++++ 2 files changed, 19 insertions(+) create mode 100644 tests/test_data_plugin.yml diff --git a/tests/conftest.py b/tests/conftest.py index 92ef2166..dc3fac8f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,3 +34,14 @@ def KOALA(request): # Install the model package so it can be used in tests subprocess.check_call([sys.executable, "-m", "pip", "install", package_from_config(imagenet_model_config)]) + + +# Set up some global vars for tests that require an example data plugin +file = fsspec.open('tests/test_data_plugin.yml') +with file as config_file: + stream = config_file.read() + data_plugin_config = yaml.safe_load(stream) + + +# Install the data plugin package so it can be used in tests +subprocess.check_call([sys.executable, "-m", "pip", "install", package_from_config(data_plugin_config)]) diff --git a/tests/test_data_plugin.yml b/tests/test_data_plugin.yml new file mode 100644 index 00000000..14140a6f --- /dev/null +++ b/tests/test_data_plugin.yml @@ -0,0 +1,8 @@ +name: scivision_sentinel2_stac +url: https://github.com/alan-turing-institute/scivision_sentinel2_stac.git +import: scivision_sentinel2_stac +class: scivision_sentinel2_stac +args: None +func: + call: get_images + args: None \ No newline at end of file From 425605ce3a6b4c9275527c6992614071bed6bef4 Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 12 Oct 2022 11:07:03 +0100 Subject: [PATCH 5/5] flake8 --- scivision/io/checker.py | 2 -- tests/conftest.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scivision/io/checker.py b/scivision/io/checker.py index 16406f48..8ae7fc97 100644 --- a/scivision/io/checker.py +++ b/scivision/io/checker.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- import importlib -import subprocess -import sys def _package_exists(config: dict) -> bool: diff --git a/tests/conftest.py b/tests/conftest.py index dc3fac8f..a7bcdbe2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ def KOALA(request): with file as config_file: stream = config_file.read() data_plugin_config = yaml.safe_load(stream) - - + + # Install the data plugin package so it can be used in tests subprocess.check_call([sys.executable, "-m", "pip", "install", package_from_config(data_plugin_config)])