improved multiprocessing, stripped down amount of OIDs scanned by the SNMP Traffic Sensor

Author: konstantinpae

probe.py

@@ -93,6 +93,8 @@ def main():
                 'protocol': config['protocol'],
                 'key': key_sha1
             }
+            procs = []
+            out_queue = multiprocessing.Queue()
             task = False
             while not task:
                 json_payload_data = []
@@ -112,30 +114,35 @@ def main():
             gc.collect()
             if str(json_response) != '[]':
                 json_response_chunks = [json_response[i:i + 10] for i in range(0, len(json_response), 10)]
-                procs = []
-
                 for element in json_response_chunks:
-                    out_queue = multiprocessing.Queue()
                     for part in element:
                         if config['debug']:
                             logging.debug(part)
                         for sensor in sensor_list:
                             if part['kind'] == sensor.get_kind():
+                                print part['kind']
                                 p = multiprocessing.Process(target=sensor.get_data, args=(part, out_queue),
                                                             name=part['kind'])
                                 procs.append(p)
                                 p.start()
                             else:
                                 pass
                         gc.collect()
-                    for i in range(len(procs)):
-                        json_payload_data.append(out_queue.get())
-                    for p in procs:
-                        p.join(2)
-                        procs = []
-                        p.terminate()
-                        del p
-                    del out_queue
+                    try:
+                        while len(json_payload_data) < len(element):
+                            out = out_queue.get()
+                            json_payload_data.append(out)
+                    except Exception as e:
+                        logging.error(e)
+                        print e
+                        pass
+                    #print len(json_response_chunks)
+                    #print len(json_payload_data)
+                        #p.join()
+                        #p.terminate()
+                        #del p
+                    #del out_queue
+                    #procs = []
                     url_data = mini_probe.create_url(config, 'data')
                     try:
                         request_data = requests.post(url_data, data=json.dumps(json_payload_data), verify=False)
@@ -157,6 +164,14 @@ def main():
                 time.sleep(int(config['baseinterval']) / 3)
 
             # Delete some stuff used in the loop and run the garbage collector
+            for p in procs:
+                if not p.is_alive():
+                    p.join()
+                    p.terminate()
+                    del p
+                #p.join()
+                #p.terminate()
+                #del p
             del json_response
             del json_payload_data
             gc.collect()

sensors/snmptraffic.py

@@ -30,6 +30,7 @@
     sys.exit()
 
 
+
 class SNMPTraffic(object):
 
     def __init__(self):
@@ -122,25 +123,43 @@ def get_sensordef():
     def snmp_get(self, target, countertype, community, port, ifindex):
         if countertype == "1":
             data = []
-            oid_endings = range(1, 19)
-            for number in oid_endings:
-                data.append("1.3.6.1.2.1.2.2.1.%s.%s" % (str(number), str(ifindex)))
+            #data2 = []
+            #oid_endings = range(1, 19)
+            #for number in oid_endings:
+                #data.append("1.3.6.1.2.1.2.2.1.%s.%s" % (str(number), str(ifindex)))
+            data.append("1.3.6.1.2.1.2.2.1.10.%s" % str(ifindex))
+            data.append("1.3.6.1.2.1.2.2.1.16.%s" % str(ifindex))
         else:
             data = []
-            oid_endings = range(1, 20)
-            for number in oid_endings:
-                data.append("1.3.6.1.2.1.31.1.1.1.%s.%s" % (str(number), str(ifindex)))
+            #oid_endings = range(1, 20)
+            #for number in oid_endings:
+                #data.append("1.3.6.1.2.1.31.1.1.1.%s.%s" % (str(number), str(ifindex)))
+            data.append("1.3.6.1.2.1.31.1.1.1.6.%s" % str(ifindex))
+            data.append("1.3.6.1.2.1.31.1.1.1.10.%s" % str(ifindex))
         snmpget = cmdgen.CommandGenerator()
         error_indication, error_status, error_index, var_binding = snmpget.getCmd(
             cmdgen.CommunityData(community), cmdgen.UdpTransportTarget((target, port)), *data)
         if countertype == "1":
-            traffic_in = str(long(var_binding[9][1]))
-            traffic_out = str(long(var_binding[15][1]))
-            traffic_total = str(long(var_binding[9][1]) + long(var_binding[15][1]))
+            #print var_binding[0][1]
+            #print var_binding[1][1]
+            #sys.exit()
+            #traffic_in = str(long(var_binding[9][1]))
+            traffic_in = str(long(var_binding[0][1]))
+            #traffic_out = str(long(var_binding[15][1]))
+            traffic_out = str(long(var_binding[1][1]))
+            #traffic_total = str(long(var_binding[9][1]) + long(var_binding[15][1]))
+            traffic_total = str(long(var_binding[0][1]) + long(var_binding[1][1]))
         else:
-            traffic_in = str(long(var_binding[5][1]))
-            traffic_out = str(long(var_binding[9][1]))
-            traffic_total = str(long(var_binding[5][1]) + long(var_binding[9][1]))
+            #print var_binding[0][1]
+            #print var_binding[1][1]
+
+            #sys.exit()
+            #traffic_in = str(long(var_binding[5][1]))
+            traffic_in = str(long(var_binding[0][1]))
+            #traffic_out = str(long(var_binding[9][1]))
+            traffic_out = str(long(var_binding[1][1]))
+            #traffic_total = str(long(var_binding[5][1]) + long(var_binding[9][1]))
+            traffic_total = str(long(var_binding[0][1]) + long(var_binding[1][1]))
 
         channellist = [
             {