Load service plugins from both modules and files

Author: amotl

CHANGES.rst

@@ -7,7 +7,7 @@ in progress
 ===========
 
 
-2021-06-02 0.21.0
+2021-06-03 0.21.0
 =================
 - [xmpp] Add slixmpp plugin and documentation. Thanks, Remi!
 - [mysql] Fix unicode vs byte issue for Python 3. Thanks, Filip!
@@ -18,6 +18,7 @@ in progress
   responds with MQTT publish. Thanks, Jörg!
 - [core] Remove "os.chdir" as it is apparently not needed anymore. Thanks, Dan!
 - [ci] Run tests on Python 3.9, remove testing on Python 3.5
+- [core] Load service plugins from both modules and files
 
 
 2020-10-20 0.20.0

HANDBOOK.md

@@ -23,9 +23,8 @@ I've written an introductory post, explaining [what mqttwarn can be used for](ht
     + [Custom functions](#custom-functions)
     + [Templates](#templates)
   * [Periodic tasks](#periodic-tasks)
-  * [Running with Docker](#running-with-docker)
-    + [Run the Image](#run-the-image)
-    + [Build the image](#build-the-image)
+  * [Running with Docker](#docker)
+  * [Loading external services](#loading-external-services)
   * [Examples](#examples)
     + [Low battery notifications](#low-battery-notifications)
     + [Producing JSON](#producing-json)
@@ -3404,6 +3403,55 @@ pinger = 10.5; now=true
 In order to run `mqttwarn` on Docker, please follow up at [DOCKER.md](DOCKER.md).
 
 
+## Loading external services
+
+In order to bring in custom emitter machinery to `mqttwarn` in form of service
+plugins, there are two options.
+
+
+### Service plugin from package
+
+This configuration snippet outlines how to load a custom plugin from a Python
+module referenced in "dotted" notation.
+
+```ini
+[defaults]
+; name the service providers you will be using.
+launch    = log, file, tests.acme.foobar
+
+[test/plugin-module]
+; echo '{"name": "temperature", "value": 42.42}' | mosquitto_pub -h localhost -t test/plugin-module -l
+targets = tests.acme.foobar:default
+format = {name}: {value}
+
+[config:tests.acme.foobar]
+targets = {
+    'default'  : [ 'default' ],
+  }
+```
+
+### Service plugin from file
+
+This configuration snippet outlines how to load a custom plugin from a Python
+file referenced by file name.
+
+```ini
+[defaults]
+; name the service providers you will be using.
+launch    = log, file, tests/acme/foobar.py
+
+[test/plugin-file]
+; echo '{"name": "temperature", "value": 42.42}' | mosquitto_pub -h localhost -t test/plugin-file -l
+targets = tests/acme/foobar.py:default
+format = {name}: {value}
+
+[config:tests/acme/foobar.py]
+targets = {
+    'default'  : [ 'default' ],
+  }
+```
+
+
 ## Examples ##
 
 This section contains some examples of how `mqttwarn` can be used with some more complex configurations.

mqttwarn/core.py

@@ -532,20 +532,27 @@ def load_services(services):
 
         module = cf.g('config:' + service, 'module', service)
 
-        if '.' in module:
+        if module.endswith(".py"):
+            modulefile = module
+
+        elif '.' in module:
+            logger.debug('Trying to load service "{}" from module "{}"'.format(service, module))
             try:
                 service_plugins[service]['module'] = load_module_by_name(module)
                 logger.info('Successfully loaded service "{}" from module "{}"'.format(service, module))
+                continue
             except Exception as ex:
                 logger.exception('Unable to load service "{}" from module "{}": {}'.format(service, module, ex))
 
         else:
             modulefile = resource_filename('mqttwarn.services', module + '.py')
-            try:
-                service_plugins[service]['module'] = load_module_from_file(modulefile)
-                logger.info('Successfully loaded service "{}"'.format(service))
-            except Exception as ex:
-                logger.exception('Unable to load service "{}" from file "{}": {}'.format(service, modulefile, ex))
+
+        logger.debug('Trying to load service "{}" from file "{}"'.format(service, modulefile))
+        try:
+            service_plugins[service]['module'] = load_module_from_file(modulefile)
+            logger.info('Successfully loaded service "{}"'.format(service))
+        except Exception as ex:
+            logger.exception('Unable to load service "{}" from file "{}": {}'.format(service, modulefile, ex))
 
 
 def connect():

tests/selftest.ini

@@ -16,7 +16,7 @@
 ; MQTT
 ; ----
 
-hostname     = 'localhost'  ; default
+hostname     = 'localhost'
 port         = 1883
 username     = None
 password     = None
@@ -55,7 +55,7 @@ loglevel  = DEBUG
 functions = 'tests/selftest.py'
 
 ; name the service providers you will be using.
-launch    = log, file, tests.acme.foobar
+launch    = log, file, tests.acme.foobar, tests/acme/foobar.py
 
 [config:log]
 targets = {
@@ -78,6 +78,11 @@ targets = {
     'default'  : [ 'default' ],
   }
 
+[config:tests/acme/foobar.py]
+targets = {
+    'default'  : [ 'default' ],
+  }
+
 
 ; -------
 ; Targets
@@ -103,11 +108,16 @@ format = {name}: {value}
 targets = file:test-2
 format = {item}
 
-[test/plugin-1]
-; echo '{"name": "temperature", "value": 42.42}' | mosquitto_pub -h localhost -t test/plugin-1 -l
+[test/plugin-module]
+; echo '{"name": "temperature", "value": 42.42}' | mosquitto_pub -h localhost -t test/plugin-module -l
 targets = tests.acme.foobar:default
 format = {name}: {value}
 
+[test/plugin-file]
+; echo '{"name": "temperature", "value": 42.42}' | mosquitto_pub -h localhost -t test/plugin-file -l
+targets = tests/acme/foobar.py:default
+format = {name}: {value}
+
 [test/datamap]
 targets = log:info
 datamap = datamap_dummy()

tests/test_core.py

@@ -178,7 +178,24 @@ def test_plugin_module(caplog):
         core_bootstrap(configfile=configfile)
 
         # Signal mocked MQTT message to the core machinery for processing
-        send_message(topic='test/plugin-1', payload='{"name": "temperature", "value": 42.42}')
+        send_message(topic='test/plugin-module', payload='{"name": "temperature", "value": 42.42}')
+
+        # Proof that the message has been routed to the "log" plugin properly
+        assert 'Plugin invoked' in caplog.text, caplog.text
+
+
+def test_plugin_file(caplog):
+    """
+    Check if using a module with dotted name also works.
+    """
+
+    with caplog.at_level(logging.DEBUG):
+
+        # Bootstrap the core machinery without MQTT
+        core_bootstrap(configfile=configfile)
+
+        # Signal mocked MQTT message to the core machinery for processing
+        send_message(topic='test/plugin-file', payload='{"name": "temperature", "value": 42.42}')
 
         # Proof that the message has been routed to the "log" plugin properly
         assert 'Plugin invoked' in caplog.text, caplog.text