#384 add ability to cap off parallel hook processes

Author: pliablepixels

zmeventnotification.ini

@@ -229,6 +229,23 @@ use_hooks = yes
 
 # NOTE: This entire section is only valid if use_hooks is yes above
 
+# When a hook is invoked, the ES forks a child. If you are in a situation
+# where your motion sensititivy in ZM is not set properly, you may land up
+# triggering hundreds of child processes of zm_detect that may potentially
+# crash your system. Note that there are global locks around the ML code which
+# are controlled by xxx_max_processes in the objectconfig/mlapiconfig.files
+# which will avoid parallel running of models. But this is if you are facing issues
+# by the simple fact that too many zm_detect processes are forked (which will apply
+# whether you use mlapi or not). While I do feel the core issue is that you need 
+# to fix your ZM sensitivity, this parameter helps control.
+
+# NOTE: When you put in value for this, any hooks that attempt to kick off 
+# beyond this limit will simply be ignored. There is no queueing.
+
+# A value of 0 (default) means there are no limits
+max_parallel_hooks=0
+
+
 # Shell script name here to be called every time an alarm is detected
 # the script will get passed $1=alarmEventID,  $2=alarmMonitorId
 # $3 monitor Name, $4 alarm cause 

zmeventnotification.pl

@@ -138,7 +138,9 @@
     '/var/lib/zmeventnotification/misc/escontrol_interface.dat',
   DEFAULT_FCM_DATE_FORMAT => '%I:%M %p, %d-%b',
   DEFAULT_FCM_ANDROID_PRIORITY=>'high',
-  DEFAULT_MAX_FCM_PER_MONTH_PER_TOKEN => 8000
+  DEFAULT_MAX_FCM_PER_MONTH_PER_TOKEN => 8000,
+
+  DEFAULT_MAX_PARALLEL_HOOKS => 0,
 };
 
 # connection state
@@ -173,6 +175,7 @@
 my $es_terminate = 0;
 
 my $child_forks = 0;    # Global tracker of active children
+my $parallel_hooks = 0; # Global tracker for active hooks
 my $total_forks = 0;    # Global tracker of all forks since start
 
 # Declare options.
@@ -279,6 +282,8 @@
 
 my %fcm_tokens_map;
 
+my $max_parallel_hooks= 0;
+
 my %monitors            = ();
 my %active_events       = ();
 my $monitor_reload_time = 0;
@@ -696,6 +701,12 @@ sub loadEsConfigSettings {
     DEFAULT_EVENT_END_NOTIFY_ON_HOOK_SUCCESS
   );
 
+  $max_parallel_hooks = config_get_val(
+    $config, 'hook',
+    'max_parallel_hooks',
+    DEFAULT_MAX_PARALLEL_HOOKS
+  );
+
   # get channels and convert to hash
 
   %event_start_notify_on_hook_fail = map { $_ => 1 }
@@ -807,10 +818,11 @@ sub print_config {
 Monitor rules JSON file................${\(value_or_undefined($es_rules_file))}
 
 Use Hooks............................. ${\(yes_or_no($use_hooks))}
+Max Parallel Hooks.................... ${\(value_or_undefined($max_parallel_hooks))}
 Hook Script on Event Start ........... ${\(value_or_undefined($event_start_hook))}
 User Script on Event Start.............${\(value_or_undefined($event_start_hook_notify_userscript))}
 Hook Script on Event End.............. ${\(value_or_undefined($event_end_hook))}
-User Script on Event End.............${\(value_or_undefined($event_end_hook_notify_userscript))}
+User Script on Event End...............${\(value_or_undefined($event_end_hook_notify_userscript))}
 Hook Skipped monitors................. ${\(value_or_undefined($hook_skip_monitors))}
 
 Notify on Event Start (hook success).. ${\(value_or_undefined($event_start_notify_on_hook_success))}
@@ -2374,6 +2386,17 @@ sub processJobs {
         delete( $active_events{$mid}->{$eid} );
         $child_forks--;
       }
+      elsif ( $job eq 'update_parallel_hooks' ) {
+        if ($msg eq "add") {
+          $parallel_hooks++;
+        }
+        elsif ($msg eq "del") {
+          $parallel_hooks--;
+        } 
+        else { 
+          printError ("Parallel hooks update: command not understood: $msg");
+        }
+      }
       elsif ( $job eq 'mqtt_publish' ) {
         my ( $id, $topic, $payload ) = split( '--SPLIT--', $msg );
         printDebug( "Job: MQTT Publish on topic: $topic", 2 );
@@ -3861,7 +3884,10 @@ sub processNewAlarmsInFork {
           if ( $cmd =~ /^(.*)$/ ) {
             $cmd = $1;
           }
+          print WRITER "update_parallel_hooks--TYPE--add\n";
           my $res = `$cmd`;
+          print WRITER "update_parallel_hooks--TYPE--del\n";
+
           chomp($res);
           my ( $resTxt, $resJsonString ) = parseDetectResults($res);
           $hookResult      = $? >> 8;
@@ -4093,7 +4119,11 @@ sub processNewAlarmsInFork {
           if ( $cmd =~ /^(.*)$/ ) {
             $cmd = $1;
           }
+
+          print WRITER "update_parallel_hooks--TYPE--add\n";
           my $res = `$cmd`;
+          print WRITER "update_parallel_hooks--TYPE--del\n";
+
           chomp($res);
           my ( $resTxt, $resJsonString ) = parseDetectResults($res);
           $hookResult = $? >> 8;
@@ -4420,7 +4450,7 @@ sub initSocketServer {
         exit(0);
       }
       my $elapsed_time_min =  ceil((time() - $es_start_time)/60);
-      printDebug( "----------> Tick START (active forks:$child_forks, total forks:$total_forks, running for:$elapsed_time_min min)<--------------", 2 );
+      printDebug( "----------> Tick START (active forks:$child_forks, total forks:$total_forks, active hooks: $parallel_hooks running for:$elapsed_time_min min)<--------------", 2 );
       if ( $restart_interval
         && ( ( time() - $es_start_time ) > $restart_interval ) )
       {
@@ -4446,7 +4476,7 @@ sub initSocketServer {
       checkConnection();
       processJobs();
 
-      printDebug( "There are $child_forks active child forks...", 2 );
+      printDebug( "There are $child_forks active child forks & $parallel_hooks zm_detect processes running...", 2 );
       my (@newEvents) = checkNewEvents();
 
       #print Dumper(\@newEvents);
@@ -4468,6 +4498,10 @@ sub initSocketServer {
       }
 
       foreach (@newEvents) {
+        if (($parallel_hooks >= $max_parallel_hooks) && ($max_parallel_hooks != 0)) {
+          printError("There are $parallel_hooks hooks running as of now. This exceeds your set limit of max_parallel_hooks=$max_parallel_hooks. Ignoring this event. Either increase your max_parallel_hooks value, or, adjust your ZM motion sensitivity ");
+          return;
+        }
         $child_forks++;
         $total_forks++;
         if ($cpid = fork() ) {
@@ -4523,7 +4557,7 @@ sub initSocketServer {
       }
 
       check_for_duplicate_token();
-      printDebug( "---------->Tick END (active forks:$child_forks, total forks:$total_forks)<--------------", 2 );
+      printDebug( "---------->Tick END (active forks:$child_forks, total forks:$total_forks, active hooks: $parallel_hooks)<--------------", 2 );
     },
 
     # called when a new connection comes in