Fix ContainerExecProc joinWithTimeout deadlock

This change updates the `ContainerExecProc#kill` method
to force the finished countdown latch to decrement. It has
been observed in some high load clusters where the
`joinWithTimeout` timeout is reached but the proc continues
to be blocked.

When `joinWithTimeout` is called, the `kill` method is called if the
task does not complete in time.

https://github.com/jenkinsci/jenkins/blob/368f1ccbc967a85c0ff801f3729cb77a269afd41/core/src/main/java/hudson/Proc.java#L165

But if `kill` fails to trigger the `finished` countdown latch then the
`join` method will continue to wait indefinitely.

https://github.com/jenkinsci/kubernetes-plugin/blob/676ab933d12ad8b25e4d7f78594a32066aad2569/src/main/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecProc.java#L100

By forcing `finished.countDown()` after `close` the join should be unblocked even if the `ctl-c` command didn't trigger the exec listener. `countDown` is a no-op if the latch is already zero.

Author: cronik

src/main/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecProc.java

@@ -89,7 +89,15 @@ public void kill() throws IOException, InterruptedException {
         } catch (IOException e) {
             LOGGER.log(Level.FINE, "Proc kill failed, ignoring", e);
         } finally {
-            close();
+            try {
+                close();
+            } finally {
+                // Force finished latch count down to ensure join unblocks.
+                // In the wild it has been observed that sometimes the countdown
+                // latch is not triggered resulting in joinWithTimeout never
+                // completing even after the timeout interruption has been triggered.
+                finished.countDown();
+            }
         }
     }
 

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecProcTest.java

@@ -0,0 +1,43 @@
+package org.csanchez.jenkins.plugins.kubernetes.pipeline;
+
+import hudson.model.TaskListener;
+import io.fabric8.kubernetes.client.dsl.ExecWatch;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.jvnet.hudson.test.JenkinsRule;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ContainerExecProcTest {
+
+    @Mock
+    private ExecWatch execWatch;
+
+    @Rule
+    public JenkinsRule j = new JenkinsRule();
+
+    /**
+     * This test validates that {@link hudson.Proc#joinWithTimeout(long, TimeUnit, TaskListener)} is
+     * terminated properly when the finished countdown latch is never triggered by the watch.
+     */
+    @Test(timeout = 20000)
+    public void testJoinWithTimeoutFinishCountDown() throws Exception {
+        AtomicBoolean alive = new AtomicBoolean(true);
+        CountDownLatch finished = new CountDownLatch(1);
+        ByteArrayOutputStream stdin = new ByteArrayOutputStream();
+        PrintStream ps = new PrintStream(new ByteArrayOutputStream());
+        TaskListener listener = TaskListener.NULL;
+        ContainerExecProc proc = new ContainerExecProc(execWatch, alive, finished, stdin, ps);
+        int exitCode = proc.joinWithTimeout(1, TimeUnit.SECONDS, listener);
+        assertEquals(-1, exitCode);
+    }
+}