Heartbeat to file for process/container runtimes

When running in the context of a simple process or a docker container we
have to heartbeat to a file for liveness checks.

Author: agrare

workers/event_catcher/event_catcher.rb

@@ -1,13 +1,16 @@
 require_relative "event_parser"
+require "active_support/core_ext/numeric/time"
+require "more_core_extensions/core_ext/string/to_i_with_method"
 
 class EventCatcher
-  def initialize(ems, endpoint, authentication, messaging, logger, page_size = 20)
+  def initialize(ems, endpoint, authentication, settings, messaging, logger, page_size = 20)
     @ems            = ems
     @endpoint       = endpoint
     @authentication = authentication
     @logger         = logger
     @messaging      = messaging
     @page_size      = page_size
+    @settings       = settings
   end
 
   def run!
@@ -45,7 +48,7 @@ def stop!
 
   private
 
-  attr_reader :ems, :endpoint, :authentication, :logger, :messaging, :page_size
+  attr_reader :ems, :endpoint, :authentication, :logger, :messaging, :page_size, :settings
 
   def connect
     vim_opts = {
@@ -135,14 +138,36 @@ def messaging_client
   end
 
   def notify_started
-    SdNotify.ready if ENV.fetch("NOTIFY_SOCKET", nil)
+    if ENV.fetch("NOTIFY_SOCKET", nil)
+      SdNotify.ready
+    elsif ENV.fetch("WORKER_HEARTBEAT_FILE", nil)
+      heartbeat_to_file
+    end
   end
 
   def heartbeat
-    SdNotify.watchdog if ENV.fetch("NOTIFY_SOCKET", nil)
+    if ENV.fetch("NOTIFY_SOCKET", nil)
+      SdNotify.watchdog
+    elsif ENV.fetch("WORKER_HEARTBEAT_FILE", nil)
+      heartbeat_to_file
+    end
   end
 
   def notify_stopping
     SdNotify.stopping if ENV.fetch("NOTIFY_SOCKET", nil)
   end
+
+  def heartbeat_to_file
+    heartbeat_file = ENV.fetch("WORKER_HEARTBEAT_FILE")
+
+    File.write(heartbeat_file, heartbeat_timeout)
+  end
+
+  def heartbeat_timeout
+    timeout   = settings.dig(:workers, :worker_base, :event_catcher, :event_catcher_vmware, :heartbeat_timeout)
+    timeout ||= settings.dig(:workers, :worker_base, :defaults, :heartbeat_timeout)
+    timeout ||= "2.minutes"
+
+    Time.now.to_i + timeout.to_i_with_method
+  end
 end

workers/event_catcher/worker

@@ -5,8 +5,10 @@ require "bundler/inline"
 gemfile(false) do
   source "https://rubygems.org"
 
+  gem "activesupport", "~>6.1.6"
   gem "manageiq-loggers", "~>1.0"
   gem "manageiq-messaging", "~> 1.0"
+  gem "more_core_extensions"
   gem "rbvmomi2", "~> 3.3"
   if ENV.fetch("APPLIANCE", nil)
     gem "sd_notify"
@@ -40,8 +42,9 @@ def main(args)
   messaging      = args["messaging"].symbolize_keys
   endpoint       = ems["endpoints"].detect { |ep| ep["role"] == "default" }
   authentication = ems["authentications"].detect { |auth| auth["authtype"] == "default" }
+  settings       = args["settings"]
 
-  EventCatcher.new(ems, endpoint, authentication, messaging, logger).run!
+  EventCatcher.new(ems, endpoint, authentication, settings, messaging, logger).run!
 end
 
 def parse_args