Merge pull request #43 from dylan171/server_perf

Improvements to RecCeiver performance

Author: shroffk

debian/changelog

@@ -1,3 +1,15 @@
+recsync (1.3.6) unstable; urgency=low
+
+  * Send PING message immediately when DONE message is received
+  * Commit transactions sequentially and support cancellation
+  * Collect and log more performance metrics
+  * Remove the extra newline character from the log messages
+  * Add support for cfstore commit cancellation
+  * Allow the ChannelFinder client size limit to be configured
+  * Improve cfstore performance by hashing PV info by PV name instead of RID
+
+ -- Dylan Maxwell <maxwelld@frib.msu.edu>  Fri, 14 Mar 2019 10:11:12 -0400
+
 recsync (1.3.5) unstable; urgency=medium
 
   * Fix bug with client that caused RIDs to be reused when more than 65536 records

server/demo.conf

@@ -44,6 +44,10 @@ procs = show
 # Doesn't effect IOC clients
 #commitInterval = 5.0
 
+# Maximum size before committing updates 
+# The default is 0 (which is no limit)
+#commitSizeLimit = 0
+
 # Maximum concurrent "active" clients
 # to allow.
 #maxActive = 20
@@ -66,6 +70,13 @@ idkey = 42
 # cf-store application
 # Uncomment line below to turn on the feature to add alias records to channelfinder
 # alias = on
+# The size limit for finding channels (ie the value of the '~size' query parameter)
+# If not specified then the fallback is the server default
+# findSizeLimit = 10000
+# Mark all channels as 'Inactive' when processor is stopped (default: True)
+#cleanOnStop = True
+# Mark all channels as 'Inactive' when processor is started (default: True)
+#cleanOnStart = True
 # Debug output file location.
 # Produces no file when not defined.
 # debug_file_loc = /home/devuser/recsyncdata.txt

server/recceiver/application.py

@@ -21,7 +21,12 @@ class Log2Twisted(logging.StreamHandler):
     """
     def __init__(self):
         super(Log2Twisted,self).__init__(stream=self)
-        self.write = log.msg
+        # The Twisted log publisher adds a newline, so strip the newline added by the Python log handler.
+        if sys.version_info < (3,2):
+            self.write = lambda *args, **kwargs: log.msg(*[ str(a).strip() for a in args ], **kwargs)
+        else:
+            self.terminator = ""  #  the 'terminator' attribute was added to StreamHandler in Python v3.2
+            self.write = log.msg
     def flush(self):
         pass
 
@@ -33,6 +38,7 @@ def __init__(self, config):
         self.annperiod = float(config.get('announceInterval', '15.0'))
         self.tcptimeout = float(config.get('tcptimeout', '15.0'))
         self.commitperiod = float(config.get('commitInterval', '5.0'))
+        self.commitSizeLimit = int(config.get('commitSizeLimit', '0'))
         self.maxActive = int(config.get('maxActive', '20'))
         self.bind, _sep, portn = config.get('bind', '').strip().partition(':')
         self.addrlist = []
@@ -65,6 +71,7 @@ def privilegedStartService(self):
         self.tcpFactory = CastFactory()
         self.tcpFactory.protocol.timeout = self.tcptimeout
         self.tcpFactory.session.timeout = self.commitperiod
+        self.tcpFactory.session.trlimit = self.commitSizeLimit
         self.tcpFactory.maxActive = self.maxActive
 
         # Attaching CastFactory to ProcessorController