Merge "Add connmon implementation"

Author: zzzeek

README.rst

@@ -214,6 +214,7 @@ collectd.d/sqlalchemy.conf file, assuming a system-installed sqlalchemy-collectd
 
 	        # set to "debug" to show messages received
 	        loglevel "info"
+
 	    </Module>
 	</Plugin>
 
@@ -424,4 +425,44 @@ explicitly referred to by absolute path within a conf file, and then to
 make matters worse when this option is used, we have to uncomment the location
 of the default types.db file in the central collectd.conf else it will
 no longer be able to find it.  Given the choice between "very nice names"
-and "no need to set up three separate config files", we chose the latter :)
+and "no need to set up three separate config files", we chose the latter :)
+
+connmon mode
+============
+
+As an added feature, the **connmon** UX has now been integrated into SQLAlchemy-collectd.
+This is a console application that displays a "top"-like display of the current
+status of connections.
+
+Using the configuration above, we can add a "monitor" line to our collectd
+server configuration::
+
+
+    LoadPlugin python
+    <Plugin python>
+        LogTraces true
+
+        Import "sqlalchemy_collectd.server.plugin"
+
+        <Module "sqlalchemy_collectd.server.plugin">
+            # ipv4 only for the moment
+            listen "0.0.0.0" 25827
+
+            # set to "debug" to show messages received
+            loglevel "info"
+
+            # connmon monitor port
+            monitor "localhost" 25828
+        </Module>
+    </Plugin>
+
+We can now run "connmon" on localhost port 25828::
+
+    connmon --port 25828
+
+Screenshot of connmon:
+
+|connmon_screenshot|
+
+.. |connmon_screenshot| image:: connmon.png
+   :width: 800

connmon.png

@@ -4,15 +4,14 @@ Hello World Example
 
 A minimal example to see sqlalchemy-collectd do something.
 
-
 Step One - Run collectd as a console app
 ========================================
 
 In one terminal window, run collectd as an interactive application with
 the -f flag::
 
-	# cd examples/helloworld
-	# collectd -f -C collectdconsole.conf
+    # cd examples/helloworld
+    # collectd -f -C collectdconsole.conf
 
 The ``collectdconsole.conf`` includes a relative path to the
 sqlalchemy-collectd checkout as the module path.
@@ -23,28 +22,39 @@ Step Two - Run the demo program
 This program uses a SQLite database for starters.    It spins up
 five processes with 20 threads each::
 
-	# python run_queries.py
+    # python run_queries.py
 
 Step Three - watch collectd console
 ===================================
 
 Output looks something like::
 
-	$ collectd -f -C collectdconsole.conf
-	[2018-02-11 18:29:35] plugin_load: plugin "logfile" successfully loaded.
-	[2018-02-11 18:29:35] [info] plugin_load: plugin "write_log" successfully loaded.
-	[2018-02-11 18:29:35] [info] plugin_load: plugin "python" successfully loaded.
-	[2018-02-11 18:29:35] [info] sqlalchemy_collectd plugin version 0.0.1
-	[2018-02-11 18:29:35] [info] Python version: 3.6.4 (default, Jan 23 2018, 22:28:37)
-	[GCC 7.2.1 20170915 (Red Hat 7.2.1-2)]
-	[2018-02-11 18:29:35] [info] Initialization complete, entering read-loop.
-	[2018-02-11 18:29:45] [info] write_log values:
-	[{"values":[0],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"run_queries.py","type":"count","type_instance":"checkedin"}]
-	[2018-02-11 18:29:45] [info] write_log values:
-	[{"values":[0],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"run_queries.py","type":"count","type_instance":"detached"}]
-	[2018-02-11 18:29:45] [info] write_log values:
-	[{"values":[6],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"run_queries.py","type":"count","type_instance":"numprocs"}]
-	[2018-02-11 18:29:45] [info] write_log values:
-	[{"values":[6],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"run_queries.py","type":"count","type_instance":"numpools"}]
-	[2018-02-11 18:29:45] [info] write_log values:
-	[{"values":[28],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"host","type":"count","type_instance":"checkedout"}]
+    $ collectd -f -C collectdconsole.conf
+    [2018-02-11 18:29:35] plugin_load: plugin "logfile" successfully loaded.
+    [2018-02-11 18:29:35] [info] plugin_load: plugin "write_log" successfully loaded.
+    [2018-02-11 18:29:35] [info] plugin_load: plugin "python" successfully loaded.
+    [2018-02-11 18:29:35] [info] sqlalchemy_collectd plugin version 0.0.1
+    [2018-02-11 18:29:35] [info] Python version: 3.6.4 (default, Jan 23 2018, 22:28:37)
+    [GCC 7.2.1 20170915 (Red Hat 7.2.1-2)]
+    [2018-02-11 18:29:35] [info] Initialization complete, entering read-loop.
+    [2018-02-11 18:29:45] [info] write_log values:
+    [{"values":[0],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"run_queries.py","type":"count","type_instance":"checkedin"}]
+    [2018-02-11 18:29:45] [info] write_log values:
+    [{"values":[0],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"run_queries.py","type":"count","type_instance":"detached"}]
+    [2018-02-11 18:29:45] [info] write_log values:
+    [{"values":[6],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"run_queries.py","type":"count","type_instance":"numprocs"}]
+    [2018-02-11 18:29:45] [info] write_log values:
+    [{"values":[6],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"run_queries.py","type":"count","type_instance":"numpools"}]
+    [2018-02-11 18:29:45] [info] write_log values:
+    [{"values":[28],"dstypes":["gauge"],"dsnames":["value"],"time":1518391784.793,"interval":10.000,"host":"photon2","plugin":"sqlalchemy","plugin_instance":"host","type":"count","type_instance":"checkedout"}]
+
+Step Four - Try out Connmon
+===========================
+
+The **connmon** console client is an optional feature that connects to the
+SQLAlchemy-collectd plugin running inside the collectd server to print live
+stats.  Once the above steps are all running, try out the client::
+
+    # connmon
+
+

examples/helloworld/README.rst

@@ -29,6 +29,10 @@ LoadPlugin python
 
         # set to "debug" to show messages received
         loglevel "info"
+
+        # connmon monitor port
+        monitor "localhost" 25828
+
     </Module>
 </Plugin>
 

examples/helloworld/collectdconsole.conf

@@ -1,17 +1,20 @@
-from sqlalchemy import create_engine
+import logging
+import multiprocessing
 import random
-import time
 import threading
-import multiprocessing
+import time
+
+from sqlalchemy import create_engine
 
-import logging
 logging.basicConfig()
 logging.getLogger("sqlalchemy_collectd").setLevel(logging.DEBUG)
 
-e = create_engine("sqlite:///file.db?plugin=collectd")
 
+def worker(appname):
+    e = create_engine(
+        "sqlite:///file.db?plugin=collectd&collectd_program_name=%s" % appname
+    )
 
-def worker():
     e.dispose()
 
     def screw_w_pool():
@@ -25,7 +28,7 @@ def screw_w_pool():
                 conn.close()
             time.sleep(random.randint(1, 5))
 
-    threads = [threading.Thread(target=screw_w_pool) for i in range(20)]
+    threads = [threading.Thread(target=screw_w_pool) for i in range(5)]
     for t in threads:
         t.daemon = True
         t.start()
@@ -34,11 +37,14 @@ def screw_w_pool():
         time.sleep(1)
 
 
-procs = [multiprocessing.Process(target=worker) for i in range(5)]
+procs = [
+    multiprocessing.Process(target=worker, args=(("worker %d" % (i % 20)),))
+    for i in range(50)
+]
 for proc in procs:
-    time.sleep(.5)
+    time.sleep(0.5)
     proc.daemon = True
     proc.start()
 
 while True:
-    time.sleep(1)
+    time.sleep(1)

examples/helloworld/run_queries.py

@@ -68,8 +68,9 @@ def run_tests(self):
     zip_safe=False,
     install_requires=requires,
     entry_points={
+        "console_scripts": ["connmon = sqlalchemy_collectd.connmon.main:main"],
         "sqlalchemy.plugins": [
             "collectd = sqlalchemy_collectd.client.plugin:Plugin"
-        ]
+        ],
     },
 )

setup.py

@@ -18,7 +18,7 @@ def _check_threads_started():
     if _WORKER_THREAD is None or _PID != ospid:
 
         _PID = ospid
-        _WORKER_THREAD = threading.Thread(target=_process, args=(5,))
+        _WORKER_THREAD = threading.Thread(target=_process, args=(2,))
         _WORKER_THREAD.daemon = True
         _WORKER_THREAD.start()