Use lazy loading for packages

faster initialization (paramiko import is extremely expensive)

Author: penguinolog

exec_helpers/__init__.py

@@ -16,31 +16,27 @@
 
 from __future__ import annotations
 
+# Standard Library
+import importlib
+import typing
+import warnings
+
 # Local Implementation
-from . import async_api
-from ._helpers import mask_command
-from ._ssh_helpers import HostsSSHConfigs
-from ._ssh_helpers import SSHConfig
-from .api import ExecHelper
 from .exceptions import CalledProcessError
 from .exceptions import ExecCalledProcessError
 from .exceptions import ExecHelperError
 from .exceptions import ExecHelperNoKillError
 from .exceptions import ExecHelperTimeoutError
 from .exceptions import ParallelCallExceptionsError
 from .exceptions import ParallelCallProcessError
-from .exec_result import ExecResult
-from .proc_enums import ExitCodes
-from .ssh import SSHClient
-from .ssh_auth import SSHAuth
-from .subprocess import Subprocess  # nosec  # Expected
 
 try:
     # Local Implementation
     from ._version import version as __version__
 except ImportError:
     pass
 
+# noinspection PyUnresolvedReferences
 __all__ = (
     "ExecHelperError",
     "ExecCalledProcessError",
@@ -49,18 +45,65 @@
     "ParallelCallProcessError",
     "ExecHelperNoKillError",
     "ExecHelperTimeoutError",
+    # pylint: disable=undefined-all-variable
+    # lazy load
+    # API
+    "async_api",
+    "ExitCodes",
+    "ExecResult",
     "ExecHelper",
-    "SSHClient",
     "mask_command",
+    # Expensive
+    "Subprocess",
+    "SSHClient",
     "SSHAuth",
     "SSHConfig",
     "HostsSSHConfigs",
-    "Subprocess",
-    "ExitCodes",
-    "ExecResult",
-    "async_api",
+    # deprecated
+    "ParallelCallExceptions",
 )
 
+__locals: typing.Dict[str, typing.Any] = locals()  # use mutable access for pure lazy loading
+
+__lazy_load_modules: typing.Sequence[str] = ("async_api",)
+
+__lazy_load_parent_modules: typing.Dict[str, str] = {
+    "HostsSSHConfigs": "_ssh_helpers",
+    "SSHConfig": "_ssh_helpers",
+    "SSHClient": "ssh",
+    "SSHAuth": "ssh_auth",
+    "Subprocess": "subprocess",
+    # API
+    "ExitCodes": "proc_enums",
+    "ExecResult": "exec_result",
+    "ExecHelper": "api",
+    "mask_command": "_helpers",
+    "ParallelCallExceptions": "exceptions",
+}
+
+_deprecated: typing.Dict[str, str] = {"ParallelCallExceptions": "ParallelCallExceptionsError"}
+
+
+def __getattr__(name: str) -> typing.Any:
+    """Get attributes lazy.
+
+    :return: attribute by name
+    :raises AttributeError: attribute is not defined for lazy load
+    """
+    if name in _deprecated:
+        warnings.warn(f"{name} is deprecated in favor of {_deprecated[name]}", DeprecationWarning)
+    if name in __lazy_load_modules:
+        mod = importlib.import_module(f"{__package__}.{name}")
+        __locals[name] = mod
+        return mod
+    if name in __lazy_load_parent_modules:
+        mod = importlib.import_module(f"{__package__}.{__lazy_load_parent_modules[name]}")
+        obj = getattr(mod, name)
+        __locals[name] = obj
+        return obj
+    raise AttributeError(f"{name} not found in {__package__}")
+
+
 __author__ = "Alexey Stepanov"
 __author_email__ = "[email protected]"
 __maintainers__ = {

exec_helpers/_ssh_base.py

@@ -42,14 +42,6 @@
 from exec_helpers import exec_result
 from exec_helpers import proc_enums
 from exec_helpers import ssh_auth
-from exec_helpers.api import CalledProcessErrorSubClassT
-from exec_helpers.api import CommandT
-from exec_helpers.api import ErrorInfoT
-from exec_helpers.api import ExpectedExitCodesT
-from exec_helpers.api import LogMaskReT
-from exec_helpers.api import OptionalStdinT
-from exec_helpers.api import OptionalTimeoutT
-from exec_helpers.proc_enums import ExitCodeT
 
 # Local Implementation
 from . import _helpers
@@ -62,6 +54,16 @@
     import socket
     import types
 
+    # Package Implementation
+    from exec_helpers.api import CalledProcessErrorSubClassT
+    from exec_helpers.api import CommandT
+    from exec_helpers.api import ErrorInfoT
+    from exec_helpers.api import ExpectedExitCodesT
+    from exec_helpers.api import LogMaskReT
+    from exec_helpers.api import OptionalStdinT
+    from exec_helpers.api import OptionalTimeoutT
+    from exec_helpers.proc_enums import ExitCodeT
+
 __all__ = ("SSHClientBase", "SshExecuteAsyncResult", "SupportPathT")
 
 KeepAlivePeriodT = typing.Union[int, bool]

exec_helpers/async_api/__init__.py

@@ -17,9 +17,51 @@
 .. versionadded:: 3.0.0
 """
 
-__all__ = ("ExecHelper", "ExecResult", "Subprocess")
+# noinspection PyUnresolvedReferences
+__all__ = (
+    # pylint: disable=undefined-all-variable
+    # lazy load
+    # API
+    "ExecHelper",
+    "ExecResult",
+    # Expensive
+    "Subprocess",
+)
 
-# Local Implementation
-from .api import ExecHelper
-from .exec_result import ExecResult
-from .subprocess import Subprocess  # nosec  # Expected
+# Standard Library
+import importlib
+import typing
+import warnings
+
+__locals: typing.Dict[str, typing.Any] = locals()  # use mutable access for pure lazy loading
+
+__lazy_load_modules: typing.Sequence[str] = ()
+
+__lazy_load_parent_modules: typing.Dict[str, str] = {
+    "Subprocess": "subprocess",
+    # API
+    "ExecResult": "exec_result",
+    "ExecHelper": "api",
+}
+
+_deprecated: typing.Dict[str, str] = {}
+
+
+def __getattr__(name: str) -> typing.Any:
+    """Get attributes lazy.
+
+    :return: attribute by name
+    :raises AttributeError: attribute is not defined for lazy load
+    """
+    if name in _deprecated:
+        warnings.warn(f"{name} is deprecated in favor of {_deprecated[name]}", DeprecationWarning)
+    if name in __lazy_load_modules:
+        mod = importlib.import_module(f"{__package__}.{name}")
+        __locals[name] = mod
+        return mod
+    if name in __lazy_load_parent_modules:
+        mod = importlib.import_module(f"{__package__}.{__lazy_load_parent_modules[name]}")
+        obj = getattr(mod, name)
+        __locals[name] = obj
+        return obj
+    raise AttributeError(f"{name} not found in {__package__}")

exec_helpers/async_api/api.py

@@ -40,7 +40,6 @@
 from exec_helpers.api import OptionalStdinT
 from exec_helpers.api import OptionalTimeoutT
 from exec_helpers.async_api import exec_result
-from exec_helpers.proc_enums import ExitCodeT
 
 # Local Implementation
 from .. import _helpers
@@ -49,7 +48,20 @@
     # Standard Library
     import types
 
-__all__ = ("ExecHelper",)
+    # Package Implementation
+    from exec_helpers.proc_enums import ExitCodeT
+
+__all__ = (
+    "ExecHelper",
+    "CalledProcessErrorSubClassT",
+    "OptionalStdinT",
+    "OptionalTimeoutT",
+    "CommandT",
+    "LogMaskReT",
+    "ErrorInfoT",
+    "ChRootPathSetT",
+    "ExpectedExitCodesT",
+)
 
 
 class ExecuteContext(typing.AsyncContextManager[api.ExecuteAsyncResult], abc.ABC):

exec_helpers/async_api/subprocess.py

@@ -34,17 +34,8 @@
 from exec_helpers import exceptions
 from exec_helpers import proc_enums
 from exec_helpers import subprocess
-from exec_helpers.api import CalledProcessErrorSubClassT
-from exec_helpers.api import CommandT
-from exec_helpers.api import ErrorInfoT
-from exec_helpers.api import ExpectedExitCodesT
-from exec_helpers.api import LogMaskReT
-from exec_helpers.api import OptionalTimeoutT
 from exec_helpers.async_api import api
 from exec_helpers.async_api import exec_result
-from exec_helpers.exec_result import OptionalStdinT
-from exec_helpers.subprocess import CwdT
-from exec_helpers.subprocess import EnvT
 
 # Local Implementation
 from .. import _log_templates
@@ -54,6 +45,17 @@
     # Standard Library
     import types
 
+    # Package Implementation
+    from exec_helpers.api import CalledProcessErrorSubClassT
+    from exec_helpers.api import CommandT
+    from exec_helpers.api import ErrorInfoT
+    from exec_helpers.api import ExpectedExitCodesT
+    from exec_helpers.api import LogMaskReT
+    from exec_helpers.api import OptionalTimeoutT
+    from exec_helpers.exec_result import OptionalStdinT
+    from exec_helpers.subprocess import CwdT
+    from exec_helpers.subprocess import EnvT
+
 __all__ = ("Subprocess", "SubprocessExecuteAsyncResult")
 
 

exec_helpers/exceptions.py

@@ -21,14 +21,14 @@
 
 # Package Implementation
 from exec_helpers import proc_enums
-from exec_helpers.proc_enums import ExitCodeT
 
 # Local Implementation
 from . import _log_templates
 
 if typing.TYPE_CHECKING:
     # Package Implementation
     from exec_helpers import exec_result  # noqa: F401  # pylint: disable=cyclic-import
+    from exec_helpers.proc_enums import ExitCodeT
 
 __all__ = (
     "ExecHelperError",

exec_helpers/exec_result.py

@@ -29,7 +29,6 @@
 # Package Implementation
 from exec_helpers import exceptions
 from exec_helpers import proc_enums
-from exec_helpers.proc_enums import ExitCodeT
 
 try:
     # noinspection PyPackageRequirements
@@ -63,6 +62,9 @@
     # noinspection PyPackageRequirements
     import logwrap
 
+    # Package Implementation
+    from exec_helpers.proc_enums import ExitCodeT
+
 __all__ = ("ExecResult", "OptionalStdinT")
 
 LOGGER: logging.Logger = logging.getLogger(__name__)

exec_helpers/ssh.py

@@ -22,10 +22,13 @@
 import os
 import pathlib
 import posixpath
+import typing
 
 # Local Implementation
 from . import _ssh_base
-from ._ssh_base import SupportPathT
+
+if typing.TYPE_CHECKING:
+    from ._ssh_base import SupportPathT
 
 __all__ = ("SSHClient",)
 

exec_helpers/subprocess.py

@@ -36,13 +36,6 @@
 from exec_helpers import exceptions
 from exec_helpers import exec_result
 from exec_helpers import proc_enums
-from exec_helpers.api import CalledProcessErrorSubClassT
-from exec_helpers.api import CommandT
-from exec_helpers.api import ErrorInfoT
-from exec_helpers.api import ExpectedExitCodesT
-from exec_helpers.api import LogMaskReT
-from exec_helpers.api import OptionalStdinT
-from exec_helpers.api import OptionalTimeoutT
 
 # Local Implementation
 from . import _log_templates
@@ -52,6 +45,15 @@
     # Standard Library
     import types
 
+    # Package Implementation
+    from exec_helpers.api import CalledProcessErrorSubClassT
+    from exec_helpers.api import CommandT
+    from exec_helpers.api import ErrorInfoT
+    from exec_helpers.api import ExpectedExitCodesT
+    from exec_helpers.api import LogMaskReT
+    from exec_helpers.api import OptionalStdinT
+    from exec_helpers.api import OptionalTimeoutT
+
 __all__ = ("Subprocess", "SubprocessExecuteAsyncResult", "EnvT", "CwdT")
 
 EnvT = typing.Optional[