Ability to load plugins in non-web situations (#1)

Refactor plugin loading out of Application, to make it possible to load
plugins in non-web situations (scripts, cron jobs, etc.).
Document how to bootstrap Webware for command-line scripts.

Author: jasonhildebrand

docs/appdev.rst

@@ -363,6 +363,25 @@ Now run this file in your IDE in debug mode. For instance, in PyCharm, right-cli
 Some IDEs like PyCharm can also debug remote processes. This could be useful to debug a test or production server.
 
 
+Bootstrap Webware from Command line
+-----------------------------------
+
+You may be in a situation where you want to execute some part of your Webware app from the command line, for example to implement a cron job or 
+maintenance script.  In these situations you probably don't want to instantiate a full-fledged `Application` -- some of the downsides are that doing so
+would cause standard output and standard error to be redirected to the log file, and that it sets up the session sweeper, task manager, etc. 
+But you may still need access to plugins such as MiscUtils, MiddleKit, which you may not be able to import directly.
+
+Here is a lightweight approach which allows you to bootstrap Webware and plugins::
+
+   import webware
+   webware.add_to_python_path()
+   webware.load_plugins('/your/app/directory')
+
+   # now plugins are available...
+   import MiscUtils
+   import MiddleKit
+
+
 How do I Develop an App?
 ------------------------
 
@@ -381,3 +400,5 @@ The answer to that question might not seem clear after being deluged with all th
 * With this additional knowledge, create more sophisticated pages.
 
 * If you need to secure your pages using a login screen, you'll want to look at the SecurePage, LoginPage, and SecureCountVisits examples in ``Examples``. You'll need to modify them to suit your particular needs.
+
+

webware/Application.py

@@ -19,8 +19,6 @@
 
 from time import time, localtime
 
-import pkg_resources
-
 from MiscUtils import NoDefault
 from MiscUtils.Funcs import asclocaltime
 from MiscUtils.NamedValueAccess import valueForName
@@ -32,9 +30,9 @@
 from ExceptionHandler import ExceptionHandler
 from HTTPRequest import HTTPRequest
 from HTTPExceptions import HTTPException, HTTPSessionExpired
-from PlugIn import PlugIn
 from Transaction import Transaction
 from WSGIStreamOut import WSGIStreamOut
+from PlugInLoader import PlugInLoader
 
 import URLParser
 
@@ -237,6 +235,7 @@ def __init__(self, path=None, settings=None, development=None):
         if self.setting('UseSessionSweeper'):
             self.startSessionSweeper()
 
+        self._plugInLoader = None
         self.loadPlugIns()
 
         self._wasShutDown = False
@@ -1093,19 +1092,7 @@ def loadPlugIn(self, name, module):
         May return None if loading was unsuccessful (in which case this method
         prints a message saying so). Used by `loadPlugIns` (note the **s**).
         """
-        try:
-            plugIn = PlugIn(self, name, module)
-            willNotLoadReason = plugIn.load()
-            if willNotLoadReason:
-                print(f'    Plug-in {name} cannot be loaded because:\n'
-                      f'    {willNotLoadReason}')
-                return None
-            plugIn.install()
-        except Exception:
-            print()
-            print(f'Plug-in {name} raised exception.')
-            raise
-        return plugIn
+        return self._plugInLoader(name, module)
 
     def loadPlugIns(self):
         """Load all plug-ins.
@@ -1115,24 +1102,8 @@ def loadPlugIns(self):
         Application at startup time, just before listening for requests.
         See the docs in `PlugIn` for more info.
         """
-        plugInNames = set(self.setting('PlugIns'))
-        plugInNames.add('Webware')
-        plugIns = [
-            (entry_point.name, entry_point.load())
-            for entry_point
-            in pkg_resources.iter_entry_points('webware.plugins')
-            if entry_point.name in plugInNames
-        ]
-
-        print('Plug-ins list:', ', '.join(
-            name for name, _module in plugIns if name != 'Webware'))
-
-        # Now that we have our plug-in list, load them...
-        for name, module in plugIns:
-            plugIn = self.loadPlugIn(name, module)
-            if plugIn:
-                self._plugIns[name] = plugIn
-        print()
+        self._plugInLoader = loader = PlugInLoader(self)
+        self._plugIns = loader.loadPlugIns(self.setting('PlugIns'))
 
     # endregion Plug-in loading
 

webware/MockApplication.py

@@ -0,0 +1,71 @@
+import os
+
+from ConfigurableForServerSidePath import ConfigurableForServerSidePath
+
+
+class MockImportManager(object):
+
+    def recordFile(self, filename, isfile=None):
+        pass
+
+
+defaultConfig = dict(
+    CacheDir='Cache',
+    PlugIns=['MiscUtils', 'WebUtils', 'TaskKit', 'UserKit', 'PSP'],
+)
+
+
+class MockApplication(ConfigurableForServerSidePath):
+    """
+    A minimal implementation which is compatible with Application
+    and which is sufficient to load plugins.
+    """
+
+    def __init__(self, path=None, settings=None, development=None):
+        ConfigurableForServerSidePath.__init__(self)
+        if path is None:
+            path = os.getcwd()
+        self._serverSidePath = os.path.abspath(path)
+        self._webwarePath = os.path.abspath(os.path.dirname(__file__))
+        if development is None:
+            development = bool(os.environ.get('WEBWARE_DEVELOPMENT'))
+        self._development = development
+
+        appConfig = self.config()  # get and cache the configuration
+        if settings:
+            appConfig.update(settings)
+        self._cacheDir = self.serverSidePath(self.setting('CacheDir') or 'Cache')
+        from MiscUtils.PropertiesObject import PropertiesObject
+        props = PropertiesObject(os.path.join(
+            self._webwarePath, 'Properties.py'))
+        self._webwareVersion = props['version']
+        self._webwareVersionString = props['versionString']
+        self._imp = MockImportManager()
+        for path in (self._cacheDir,):
+            if path and not os.path.exists(path):
+                os.makedirs(path)
+
+    def defaultConfig(self):
+        return defaultConfig
+
+    def configReplacementValues(self):
+        """Get config values that need to be escaped."""
+        return dict(
+            ServerSidePath=self._serverSidePath,
+            WebwarePath=self._webwarePath,
+            Development=self._development)
+
+    def configFilename(self):
+        return self.serverSidePath('Configs/Application.config')
+
+    def serverSidePath(self, path=None):
+        if path:
+            return os.path.normpath(
+                os.path.join(self._serverSidePath, path))
+        return self._serverSidePath
+
+    def hasContext(self, context):
+        return False
+
+    def addServletFactory(self, factory):
+        pass

webware/PlugIn.py

@@ -81,13 +81,14 @@ def __init__(self, application, name, module):
         self._cacheDir = os.path.join(self._app._cacheDir, self._name)
         self._examplePages = self._examplePagesContext = None
 
-    def load(self):
+    def load(self, verbose=True):
         """Loads the plug-in into memory, but does not yet install it.
 
         Will return None on success, otherwise a message (string) that says
         why the plug-in could not be loaded.
         """
-        print(f'Loading plug-in: {self._name} at {self._path}')
+        if verbose:
+            print(f'Loading plug-in: {self._name} at {self._path}')
 
         # Grab the Properties.py
         self._properties = PropertiesObject(

webware/PlugInLoader.py

@@ -0,0 +1,60 @@
+import pkg_resources
+
+from PlugIn import PlugIn
+
+
+class PlugInLoader(object):
+
+    def __init__(self, app):
+        self.app = app
+        self._plugIns = {}
+
+    def loadPlugIn(self, name, module, verbose=True):
+        """Load and return the given plug-in.
+
+        May return None if loading was unsuccessful (in which case this method
+        prints a message saying so). Used by `loadPlugIns` (note the **s**).
+        """
+        try:
+            plugIn = PlugIn(self.app, name, module)
+            willNotLoadReason = plugIn.load(verbose=verbose)
+            if willNotLoadReason:
+                print(f'    Plug-in {name} cannot be loaded because:\n'
+                      f'    {willNotLoadReason}')
+                return None
+            plugIn.install()
+        except Exception:
+            print()
+            print(f'Plug-in {name} raised exception.')
+            raise
+        return plugIn
+
+    def loadPlugIns(self, plugInNames, verbose=True):
+        """Load all plug-ins.
+
+        A plug-in allows you to extend the functionality of Webware without
+        necessarily having to modify its source. Plug-ins are loaded by
+        Application at startup time, just before listening for requests.
+        See the docs in `PlugIn` for more info.
+        """
+        plugInNames = set(plugInNames)
+        plugInNames.add('Webware')
+        plugIns = [
+            (entry_point.name, entry_point.load())
+            for entry_point
+            in pkg_resources.iter_entry_points('webware.plugins')
+            if entry_point.name in plugInNames
+        ]
+
+        if verbose:
+            print('Plug-ins list:', ', '.join(
+                name for name, _module in plugIns if name != 'Webware'))
+
+        # Now that we have our plug-in list, load them...
+        for name, module in plugIns:
+            plugIn = self.loadPlugIn(name, module, verbose=verbose)
+            if plugIn:
+                self._plugIns[name] = plugIn
+        if verbose:
+            print()
+        return self._plugIns

webware/__init__.py

@@ -1 +1,17 @@
 """Webware for Python"""
+
+import sys
+
+
+def add_to_python_path():
+    webwarePath = __path__[0]
+    if webwarePath not in sys.path:
+        sys.path.insert(0, webwarePath)
+
+
+def load_plugins(path, settings=None, development=None):
+    from MockApplication import MockApplication
+    from PlugInLoader import PlugInLoader
+    app = MockApplication(path, settings, development)
+    loader = PlugInLoader(app)
+    loader.loadPlugIns(app.setting('PlugIns'), verbose=False)