netty/jetty: review performance degradation on 4.x

- fix #3771
- ref #3770

Author: jknack

modules/jooby-netty/src/main/java/io/jooby/internal/netty/NettyEventLoopGroup.java

@@ -0,0 +1,52 @@
+/*
+ * Jooby https://jooby.io
+ * Apache License Version 2.0 https://jooby.io/LICENSE.txt
+ * Copyright 2014 Edgar Espina
+ */
+package io.jooby.internal.netty;
+
+import java.util.concurrent.TimeUnit;
+
+import io.netty.channel.EventLoopGroup;
+
+public class NettyEventLoopGroup {
+  private final EventLoopGroup parent;
+  private final EventLoopGroup child;
+  private boolean closed;
+
+  public NettyEventLoopGroup(NettyTransport transport, boolean single, int ioThreads) {
+    child = transport.createEventLoop(ioThreads, "eventloop", 100);
+    if (single) {
+      parent = child;
+    } else {
+      parent = transport.createEventLoop(1, "acceptor", 50);
+    }
+  }
+
+  public EventLoopGroup getParent() {
+    return parent;
+  }
+
+  public EventLoopGroup getChild() {
+    return child;
+  }
+
+  public void shutdown() {
+    if (!closed) {
+      closed = true;
+      if (parent == child) {
+        shutdown(child, 0);
+      } else {
+        try {
+          shutdown(parent, 0);
+        } finally {
+          shutdown(child, 2);
+        }
+      }
+    }
+  }
+
+  private void shutdown(EventLoopGroup eventLoopGroup, int quietPeriod) {
+    eventLoopGroup.shutdownGracefully(quietPeriod, 15, TimeUnit.SECONDS);
+  }
+}

modules/jooby-netty/src/main/java/io/jooby/internal/netty/NettyHandler.java

@@ -39,13 +39,13 @@ public class NettyHandler extends ChannelInboundHandlerAdapter {
   private NettyContext context;
 
   public NettyHandler(
-      NettyDateService dateService,
+      NettyDateService serverDate,
       List<Jooby> applications,
       long maxRequestSize,
       int bufferSize,
       boolean defaultHeaders,
       boolean http2) {
-    this.serverDate = dateService;
+    this.serverDate = serverDate;
     this.applications = applications;
     this.ctxSelector = Context.Selector.create(applications);
     this.maxRequestSize = maxRequestSize;

modules/jooby-netty/src/main/java/io/jooby/internal/netty/NettyPipeline.java

@@ -6,6 +6,7 @@
 package io.jooby.internal.netty;
 
 import java.util.List;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.function.Supplier;
 
 import io.jooby.Jooby;
@@ -20,7 +21,6 @@
 public class NettyPipeline extends ChannelInitializer<SocketChannel> {
   private static final String H2_HANDSHAKE = "h2-handshake";
   private final SslContext sslContext;
-  private final NettyDateService serverDate;
   private final HttpDecoderConfig decoderConfig;
   private final List<Jooby> applications;
   private final long maxRequestSize;
@@ -32,7 +32,6 @@ public class NettyPipeline extends ChannelInitializer<SocketChannel> {
 
   public NettyPipeline(
       SslContext sslContext,
-      NettyDateService dateService,
       HttpDecoderConfig decoderConfig,
       List<Jooby> applications,
       long maxRequestSize,
@@ -42,7 +41,6 @@ public NettyPipeline(
       boolean expectContinue,
       Integer compressionLevel) {
     this.sslContext = sslContext;
-    this.serverDate = dateService;
     this.decoderConfig = decoderConfig;
     this.applications = applications;
     this.maxRequestSize = maxRequestSize;
@@ -53,9 +51,14 @@ public NettyPipeline(
     this.compressionLevel = compressionLevel;
   }
 
-  private NettyHandler createHandler() {
+  private NettyHandler createHandler(ScheduledExecutorService executor) {
     return new NettyHandler(
-        serverDate, applications, maxRequestSize, bufferSize, defaultHeaders, http2);
+        new NettyDateService(executor),
+        applications,
+        maxRequestSize,
+        bufferSize,
+        defaultHeaders,
+        http2);
   }
 
   @Override
@@ -79,7 +82,7 @@ public void initChannel(SocketChannel ch) {
 
       p.addLast(H2_HANDSHAKE, handshake);
       additionalHandlers(p);
-      p.addLast("handler", createHandler());
+      p.addLast("handler", createHandler(ch.eventLoop()));
     } else {
       http11(p);
     }
@@ -121,6 +124,6 @@ private void http11(ChannelPipeline p) {
     p.addLast("decoder", new NettyRequestDecoder(decoderConfig));
     p.addLast("encoder", new NettyResponseEncoder());
     additionalHandlers(p);
-    p.addLast("handler", createHandler());
+    p.addLast("handler", createHandler(p.channel().eventLoop()));
   }
 }

modules/jooby-netty/src/main/java/io/jooby/netty/NettyServer.java

@@ -52,16 +52,13 @@ public class NettyServer extends Server.Base {
     System.setProperty("__server_.name", NAME);
   }
 
-  private static final int _50 = 50;
-
-  private static final int _100 = 100;
-
-  private EventLoopGroup acceptorloop;
-  private EventLoopGroup eventloop;
+  private NettyEventLoopGroup eventLoop;
   private ExecutorService worker;
 
   private List<Jooby> applications;
   private NettyOutputFactory outputFactory;
+  private boolean singleEventLoopGroup =
+      System.getProperty("io.netty.eventLoopGroup", "parent-child").equals("single");
 
   /**
    * Creates a server.
@@ -90,6 +87,18 @@ public NettyServer(@NonNull ServerOptions options) {
 
   public NettyServer() {}
 
+  /**
+   * Configure netty to use a single event loop group. When true, the acceptor and eventLoop share
+   * the same {@link EventLoopGroup}. Default is <code>true</code>.
+   *
+   * @param value True for shared {@link EventLoopGroup}.
+   * @return This server.
+   */
+  public NettyServer setSingleEventLoopGroup(boolean value) {
+    this.singleEventLoopGroup = value;
+    return this;
+  }
+
   @Override
   public OutputFactory getOutputFactory() {
     if (outputFactory == null) {
@@ -127,22 +136,14 @@ public Server start(@NonNull Jooby... application) {
       var classLoader = applications.get(0).getClassLoader();
 
       var transport = NettyTransport.transport(classLoader);
-
-      /* Acceptor event-loop */
-      // this.acceptorloop = transport.createEventLoop(1, "acceptor", _50);
-
-      /* Event loop: processing connections, parsing messages and doing engine's internal work */
-      this.eventloop = transport.createEventLoop(options.getIoThreads(), "eventloop", _100);
-      this.acceptorloop = eventloop;
-      var dateService = new NettyDateService(eventloop);
+      eventLoop = new NettyEventLoopGroup(transport, singleEventLoopGroup, options.getIoThreads());
 
       var outputFactory = (NettyOutputFactory) getOutputFactory();
       var allocator = outputFactory.getAllocator();
       var http2 = options.isHttp2() == Boolean.TRUE;
       /* Bootstrap: */
       if (!options.isHttpsOnly()) {
-        var http =
-            newBootstrap(allocator, transport, newPipeline(options, null, dateService, http2));
+        var http = newBootstrap(allocator, transport, newPipeline(options, null, http2), eventLoop);
         http.bind(options.getHost(), options.getPort()).get();
       }
 
@@ -155,8 +156,7 @@ public Server start(@NonNull Jooby... application) {
         var clientAuth = sslOptions.getClientAuth();
         var sslContext = wrap(javaSslContext, toClientAuth(clientAuth), protocol, http2);
         var https =
-            newBootstrap(
-                allocator, transport, newPipeline(options, sslContext, dateService, http2));
+            newBootstrap(allocator, transport, newPipeline(options, sslContext, http2), eventLoop);
         portInUse = options.getSecurePort();
         https.bind(options.getHost(), portInUse).get();
       } else if (options.isHttpsOnly()) {
@@ -177,9 +177,12 @@ public Server start(@NonNull Jooby... application) {
   }
 
   private ServerBootstrap newBootstrap(
-      ByteBufAllocator allocator, NettyTransport transport, NettyPipeline factory) {
+      ByteBufAllocator allocator,
+      NettyTransport transport,
+      NettyPipeline factory,
+      NettyEventLoopGroup group) {
     return transport
-        .configure(acceptorloop, eventloop)
+        .configure(eventLoop.getParent(), eventLoop.getChild())
         .childHandler(factory)
         .childOption(ChannelOption.ALLOCATOR, allocator)
         .childOption(ChannelOption.SO_REUSEADDR, true)
@@ -194,8 +197,7 @@ private ClientAuth toClientAuth(SslOptions.ClientAuth clientAuth) {
     };
   }
 
-  private NettyPipeline newPipeline(
-      ServerOptions options, SslContext sslContext, NettyDateService dateService, boolean http2) {
+  private NettyPipeline newPipeline(ServerOptions options, SslContext sslContext, boolean http2) {
     var decoderConfig =
         new HttpDecoderConfig()
             .setMaxInitialLineLength(_4KB)
@@ -205,7 +207,6 @@ private NettyPipeline newPipeline(
             .setTrailersFactory(HEADERS);
     return new NettyPipeline(
         sslContext,
-        dateService,
         decoderConfig,
         applications,
         options.getMaxRequestSize(),
@@ -222,9 +223,7 @@ public synchronized Server stop() {
     // only for jooby build where close events may take longer.
     NettyWebSocket.all.clear();
 
-    // required after Netty 4.2
-    shutdown(acceptorloop, 0);
-    shutdown(eventloop, 2);
+    eventLoop.shutdown();
     if (worker != null) {
       worker.shutdown();
       worker = null;

tests/pom.xml

@@ -383,4 +383,23 @@
       </plugin>
     </plugins>
   </build>
+
+  <profiles>
+    <profile>
+      <id>netty-mac_aarch_64</id>
+      <activation>
+        <os>
+          <family>mac</family>
+        </os>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>io.netty</groupId>
+          <artifactId>netty-transport-native-kqueue</artifactId>
+          <version>${netty.version}</version>
+          <classifier>osx-aarch_64</classifier>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
 </project>

tests/src/test/java/examples/Performance.java

@@ -14,7 +14,7 @@
 import io.jooby.Jooby;
 import io.jooby.ServerOptions;
 import io.jooby.StartupSummary;
-import io.jooby.jetty.JettyServer;
+import io.jooby.netty.NettyServer;
 
 public class Performance extends Jooby {
 
@@ -43,6 +43,6 @@ public class Performance extends Jooby {
 
   public static void main(final String[] args) {
     System.setProperty("io.netty.disableHttpHeadersValidation", "true");
-    runApp(args, new JettyServer(new ServerOptions()), EVENT_LOOP, Performance::new);
+    runApp(args, new NettyServer(new ServerOptions()), EVENT_LOOP, Performance::new);
   }
 }