Add draining for cc uploader (#4296)

Author: kathap

lib/cloud_controller/drain.rb

@@ -9,6 +9,7 @@ class Drain
       NGINX_FINAL_TIMEOUT = 10
       CCNG_FINAL_TIMEOUT = 20
       SLEEP_INTERVAL = 1
+      CC_UPLOADER_FINAL_TIMEOUT_SECONDS = 900
 
       def initialize(log_path)
         @log_path = log_path
@@ -57,6 +58,22 @@ def shutdown_delayed_worker(pid_path, timeout=15)
         send_signal('KILL', pid, process_name)
       end
 
+      def shutdown_cc_uploader(pid_path)
+        pid = File.read(pid_path).to_i
+        process_name = File.basename(pid_path, '.pid')
+        # Initiate shutdown.
+        send_signal('TERM', pid, process_name)
+
+        # Wait some additional time for cc_uploader to be terminated; otherwise write an error log message.
+        log_shutdown_error(pid, process_name) unless wait_for_shutdown(pid, process_name, CC_UPLOADER_FINAL_TIMEOUT_SECONDS)
+
+        # force shutdown
+        return if terminated?(pid, process_name)
+
+        log_info("Forcefully shutting down process '#{process_name}' with pid '#{pid}'")
+        send_signal('KILL', pid, process_name)
+      end
+
       private
 
       def send_signal(signal, pid, process_name)

spec/unit/lib/cloud_controller/drain_spec.rb

@@ -159,5 +159,40 @@ def log_contents
         end
       end
     end
+
+    describe '#shutdown_cc_uploader' do
+      it 'sends TERM to the cc_uploader process specified in the pid file' do
+        expect(Process).to receive(:kill).with('TERM', pid)
+
+        drain.shutdown_cc_uploader(pid_path)
+
+        log_contents do |log|
+          expect(log).to include("Sending signal 'TERM' to process '#{pid_name}' with pid '#{pid}'")
+        end
+      end
+
+      it 'waits 900s after sending TERM' do
+        allow(Process).to receive(:getpgid).with(pid).and_return(1)
+
+        drain.shutdown_cc_uploader(pid_path)
+
+        expect(drain).to have_received(:sleep).exactly(900).times
+        log_contents do |log|
+          expect(log).to include("Process '#{pid_name}' with pid '#{pid}' is still running - this indicates an error in the shutdown procedure!")
+        end
+      end
+
+      it 'sends KILL to the cc_uploader process if it is still running after 900s' do
+        allow(Process).to receive(:getpgid).with(pid).and_return(1)
+        allow(Process).to receive(:kill).with('TERM', pid)
+        expect(Process).to receive(:kill).with('KILL', pid)
+
+        drain.shutdown_cc_uploader(pid_path)
+
+        log_contents do |log|
+          expect(log).to include("Forcefully shutting down process '#{pid_name}' with pid '#{pid}'")
+        end
+      end
+    end
   end
 end