servlet: add extra 5s to AsyncContext timeout (grpc#12321)

Currently there is a race between 2 tasks scheduled to handle request
timeout
* servlet AsyncContext timeout
* gRPC Context

which can cause instability. This change makes the gRPC Context timeout happen first.

Author: panchenko

servlet/src/main/java/io/grpc/servlet/ServletAdapter.java

@@ -22,6 +22,7 @@
 import static java.util.logging.Level.FINE;
 import static java.util.logging.Level.FINEST;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.io.BaseEncoding;
 import io.grpc.Attributes;
 import io.grpc.ExperimentalApi;
@@ -134,10 +135,9 @@ public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOEx
     }
 
     Long timeoutNanos = headers.get(TIMEOUT_KEY);
-    if (timeoutNanos == null) {
-      timeoutNanos = 0L;
-    }
-    asyncCtx.setTimeout(TimeUnit.NANOSECONDS.toMillis(timeoutNanos));
+    asyncCtx.setTimeout(timeoutNanos != null
+        ? TimeUnit.NANOSECONDS.toMillis(timeoutNanos) + ASYNC_TIMEOUT_SAFETY_MARGIN
+        : 0);
     StatsTraceContext statsTraceCtx =
         StatsTraceContext.newServerContext(streamTracerFactories, method, headers);
 
@@ -164,6 +164,12 @@ public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOEx
     asyncCtx.addListener(new GrpcAsyncListener(stream, logId));
   }
 
+  /**
+   * Deadlines are managed via Context, servlet async timeout is not supposed to happen.
+   */
+  @VisibleForTesting
+  static final long ASYNC_TIMEOUT_SAFETY_MARGIN = 5_000;
+
   // This method must use Enumeration and its members, since that is the only way to read headers
   // from the servlet api.
   @SuppressWarnings("JdkObsolete")

servlet/src/test/java/io/grpc/servlet/ServletServerBuilderTest.java

@@ -80,7 +80,7 @@ public void scheduledExecutorService() throws Exception {
     ServletAdapter servletAdapter = serverBuilder.buildServletAdapter();
     servletAdapter.doPost(request, response);
 
-    verify(asyncContext).setTimeout(1);
+    verify(asyncContext).setTimeout(1 + ServletAdapter.ASYNC_TIMEOUT_SAFETY_MARGIN);
 
     // The following just verifies that scheduler is populated to the transport.
     // It doesn't matter what tasks (such as handshake timeout and request deadline) are actually