Refactoring of test (new base test and ocr tests was moved)

New Import Fix by using safe_imports

Author: sokolegg

Makefile

@@ -1,18 +1,16 @@
 SHELL:=/usr/bin/env bash
 
-.PHONY: lint
+.PHONY: lint unit package test jupyter
+
 lint:
 	flake8 piper
 
-.PHONY: unit
 unit:
-	pytest -vs tests/running_piper_test.py::TestCompose
+	pytest -vs tests/envs_test.py::TestCompose
 
-.PHONY: package
 package:
 	pip check
 
-.PHONY: test
 test: package unit
 
 jupyter:

piper/__init__.py

@@ -0,0 +1,10 @@
+from piper.configurations import get_configuration
+from piper.imports import activate_safe_import
+
+configuration = get_configuration()
+if configuration.ignore_import_errors:
+    """
+    Piper activates safe import if configured True.
+    This ignores any import errors for safe imports in piper.base.executors
+    """
+    activate_safe_import()

piper/base/executors.py

@@ -1,3 +1,11 @@
+# from piper.base.docker import PythonTesseractImage
+from piper.base.backend.utils import (render_fast_api_backend,
+                                      render_fast_api_tsrct_backend)
+from piper.base.docker import PythonImage
+from piper.configurations import get_configuration
+from piper.envs import get_env, is_current_env, is_docker_env
+from piper.utils import docker_utils as du
+
 import inspect
 import os
 import sys
@@ -12,14 +20,6 @@
 from loguru import logger
 from pydantic import BaseModel  # , BytesObject, ListOfStringsObject
 
-# from piper.base.docker import PythonTesseractImage
-from piper.base.backend.utils import (render_fast_api_backend,
-                                      render_fast_api_tsrct_backend)
-from piper.base.docker import PythonImage
-from piper.configurations import get_configuration
-from piper.envs import get_env, is_current_env, is_docker_env
-from piper.utils import docker_utils as du
-
 
 class BaseExecutor:
     pass
@@ -89,7 +89,7 @@ async def __call__(self, *args, **kwargs):
 def copy_piper(path: str):
     cfg = get_configuration()
     from distutils.dir_util import copy_tree
-    copy_tree(cfg.piper_path, f"{path}/piper")
+    copy_tree(cfg.piper_path, f"{path}piper")
 
 
 def copy_scripts(path: str, scripts: Dict[str, str]):

piper/configurations.py

@@ -1,15 +1,15 @@
 class Configuration:
-    # path = "/Users/olegsokolov/PycharmProjects/piper/applications"
-    path = "./piper_new_out/"
-    test_path = "./piper_test_out/"
-    piper_path = "piper"
-    default_env = "docker"
-    docker_app_port = 8788
+    path: str = "./piper_new_out/"
+    test_path: str = "./piper_test_out/"
+    piper_path: str = "../piper"
+    default_env: str = "docker"
+    docker_app_port: int = 8788
 
-    name_venv = "venv_test"
-    number = 10
+    name_venv: str = "venv_test"
+    number: int = 10
 
-    env = 'compose'
+    env: str = 'compose'
+    ignore_import_errors: bool = False
 
     # start time and counter
     wait_on_iter = 0.5

piper/imports.py

@@ -0,0 +1,91 @@
+import builtins
+from types import ModuleType
+
+from piper.utils.logger_utils import logger
+
+
+def _empty_import():
+    logger.error("Import Not Installed Yet!")
+    raise ImportError
+
+
+real_import = _empty_import
+
+
+class PiperDummyModule(ModuleType):
+
+    def __init__(self, name):
+        super().__init__(name)
+        logger.info(f"Piper emulates {name} module")
+
+    def __getattr__(self, name):
+        return PiperDummyModule(name)
+
+    __all__ = []
+
+
+def try_import(name, globals={}, locals={}, fromlist=[], level=0):
+    """
+    This import replace real Python import with fake import which returns warning only and PiperDummyModule.
+    """
+    try:
+        return real_import(name, globals, locals, fromlist, level)
+    except ImportError as e:
+        logger.warning(f"Piper ignores ImportError and module {name} replaced by dummy module. ImportError: {e}")
+        module = PiperDummyModule(name)
+        return module
+
+
+"""
+Here Piper saves default Python *import* only.
+"""
+if builtins.__import__ != try_import:
+    real_import = builtins.__import__
+
+
+def set_ignore_import_errors(ignore: bool = True):
+    if ignore:
+        builtins.__import__ = try_import
+    else:
+        builtins.__import__ = real_import
+
+
+def activate_safe_import():
+    """
+    Activate piper safe import with try_import function.
+    Piper needs safe import to ignore imports in Executors examples.
+    For instance if you want to use Pandas in your CustomExecutor normally you have to *import pandas*
+    But we don't want to install everything for every executor in default Python (where Piper is installed)
+    For that you have to ignore every Executors dependencies.
+
+    Otherwise, you can wrap buy yourself every Executors import with try_import
+    or you can use directly only *requirements* field in your CustomExecutor.
+
+    """
+    logger.info(f"Piper activates safe import")
+    set_ignore_import_errors(ignore=True)
+
+
+def deactivate_safe_import():
+    logger.info(f"Piper deactivates safe import")
+    set_ignore_import_errors(ignore=False)
+
+
+class safe_import:
+    """
+    Context manager to activate safe import on some part of imports.
+    For instance:
+
+        with safe_import():
+            import foo
+        import bar
+
+    foo would be ignored and replaced by PiperDummyModule
+    boo wouldn't be replaced (you can catch ImportError)
+    """
+
+    def __enter__(self):
+        activate_safe_import()
+
+    def __exit__(self, type, value, traceback):
+        deactivate_safe_import()

piper/services/__init__.py

@@ -1,8 +1,11 @@
-import json
 import sys
 
-import spacy
-from fastapi.responses import JSONResponse
+from piper.imports import safe_import
+
+with safe_import():
+    import spacy
+    from fastapi.responses import JSONResponse
+
 from loguru import logger
 from pydantic import BaseModel
 
@@ -25,16 +28,6 @@ class ListOfStringsObject(BaseModel):
     value: list
 
 
-class TestMessageAdder(FastAPIExecutor):
-
-    def __init__(self, appender="TEST", **kwargs):
-        self.appender = appender
-        super().__init__(**kwargs)
-
-    async def run(self, message: StringValue) -> StringValue:
-        return StringValue(value=(message.value + self.appender))
-
-
 class TesseractRecognizer(FastAPITesseractExecutor):
     '''
         Tesseract OCR implementation service

piper/utils/tesrct_utils.py

@@ -1,7 +1,10 @@
-import cv2
-import numpy as np
-import pdf2image
-import pytesseract
+from piper.imports import safe_import
+with safe_import():
+    import cv2
+    import numpy as np
+    import pdf2image
+    import pytesseract
+
 import requests
 from loguru import logger
 

tests/base_test.py

@@ -0,0 +1,46 @@
+# pytest -vs tests/base_test.py::TestPiperBase
+import asyncio
+import os
+import sys
+import time
+
+root_dir = os.path.join(os.path.realpath(os.path.pardir), 'piper')
+sys.path.insert(1, root_dir)
+
+
+class TestPiperBase:
+
+    def test_simple_executor(self):
+        from piper.base.executors import FastAPIExecutor
+        from piper.services import StringValue
+        from piper.configurations import get_configuration
+        from piper.envs import CurrentEnv, DockerEnv
+
+        class TestMessageAdder(FastAPIExecutor):
+
+            def __init__(self, appender="TEST", **kwargs):
+                self.appender = appender
+                super().__init__(**kwargs)
+
+            async def run(self, message: StringValue) -> StringValue:
+                return StringValue(value=(message.value + self.appender))
+
+        cfg = get_configuration()
+        loop = asyncio.get_event_loop()
+        init_value = "TEST_container_"
+        x = StringValue(value=init_value)
+        need_result = f'{init_value}TEST'
+
+        with CurrentEnv() as env:
+            adder = TestMessageAdder(appender="!", port=cfg.docker_app_port)
+            result = loop.run_until_complete(adder(x))
+            print(result)
+            assert result.get('value') == need_result
+
+        with DockerEnv() as env:
+            adder = TestMessageAdder(appender="!", port=cfg.docker_app_port)
+            result = loop.run_until_complete(adder(x))
+            print(result)
+            adder.rm_container()
+            assert result.get('value') == need_result
+

tests/container_test.py

@@ -1,11 +1,5 @@
-# pytest -vs tests/running_piper_test.py::TestDocker
+# pytest -vs tests/envs_test.py::TestDocker
 import time
-from pathlib import Path
-from shlex import split
-from subprocess import check_call
-from tempfile import TemporaryDirectory
-from venv import create
-
 import requests
 
 from piper.envs import ComposeEnv, VirtualEnv