Change netty pool chunk size to respect G1 region (#62410)

Currently the netty pool chunk size defaults to 16MB. The number does
not play well with the G1GC which causes this to consume entire regions.
Additionally, we normally allocated arrays of size 64KB or less. This
means that Elasticsearch could handle a smaller pool chunk size to play
nicer with the G1GC.

Author: Tim-Brooks

modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.transport;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -49,8 +51,14 @@ public class Netty4Plugin extends Plugin implements NetworkPlugin {
     public static final String NETTY_TRANSPORT_NAME = "netty4";
     public static final String NETTY_HTTP_TRANSPORT_NAME = "netty4";
 
+    private static final Logger logger = LogManager.getLogger(Netty4Plugin.class);
+
     private final SetOnce<SharedGroupFactory> groupFactory = new SetOnce<>();
 
+    public Netty4Plugin() {
+        logger.info("creating NettyAllocator with the following configs: " + NettyAllocator.getAllocatorDescription());
+    }
+
     @Override
     public List<Setting<?>> getSettings() {
         return Arrays.asList(

modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyAllocator.java

@@ -29,32 +29,68 @@
 import io.netty.channel.socket.nio.NioServerSocketChannel;
 import io.netty.channel.socket.nio.NioSocketChannel;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
 public class NettyAllocator {
 
     private static final ByteBufAllocator ALLOCATOR;
+    private static final String DESCRIPTION;
 
     private static final String USE_UNPOOLED = "es.use_unpooled_allocator";
     private static final String USE_NETTY_DEFAULT = "es.unsafe.use_netty_default_allocator";
+    private static final String USE_NETTY_DEFAULT_CHUNK = "es.unsafe.use_netty_default_chunk_and_page_size";
 
     static {
         if (Booleans.parseBoolean(System.getProperty(USE_NETTY_DEFAULT), false)) {
             ALLOCATOR = ByteBufAllocator.DEFAULT;
+            DESCRIPTION = "[name=netty_default, factors={es.unsafe.use_netty_default_allocator=true}]";
         } else {
+            final long heapSizeInBytes = JvmInfo.jvmInfo().getMem().getHeapMax().getBytes();
+            final boolean g1gcEnabled = Boolean.parseBoolean(JvmInfo.jvmInfo().useG1GC());
+            final long g1gcRegionSizeInBytes = JvmInfo.jvmInfo().getG1RegionSize();
+            final boolean g1gcRegionSizeIsKnown = g1gcRegionSizeInBytes != -1;
+            ByteSizeValue heapSize = new ByteSizeValue(heapSizeInBytes);
+            ByteSizeValue g1gcRegionSize = new ByteSizeValue(g1gcRegionSizeInBytes);
+
             ByteBufAllocator delegate;
-            if (useUnpooled()) {
+            if (useUnpooled(heapSizeInBytes, g1gcEnabled, g1gcRegionSizeIsKnown, g1gcRegionSizeInBytes)) {
                 delegate = UnpooledByteBufAllocator.DEFAULT;
+                DESCRIPTION = "[name=unpooled, factors={es.unsafe.use_unpooled_allocator=" + userForcedUnpooled()
+                    + ", g1gc_enabled=" + g1gcEnabled
+                    + ", g1gc_region_size=" + g1gcRegionSize
+                    + ", heap_size=" + heapSize + "}]";
             } else {
                 int nHeapArena = PooledByteBufAllocator.defaultNumHeapArena();
-                int pageSize = PooledByteBufAllocator.defaultPageSize();
-                int maxOrder = PooledByteBufAllocator.defaultMaxOrder();
+                int pageSize;
+                int maxOrder;
+                if (useDefaultChunkAndPageSize()) {
+                    pageSize = PooledByteBufAllocator.defaultPageSize();
+                    maxOrder = PooledByteBufAllocator.defaultMaxOrder();
+                } else {
+                    pageSize = 8192;
+                    if (g1gcEnabled == false || g1gcRegionSizeIsKnown == false || g1gcRegionSizeInBytes >= (4 * 1024 * 1024)) {
+                        // This combined with a 8192 page size = 1 MB chunk sizes
+                        maxOrder = 7;
+                    } else if (g1gcRegionSizeInBytes >= (2 * 1024 * 1024)) {
+                        // This combined with a 8192 page size = 512 KB chunk sizes
+                        maxOrder = 6;
+                    } else {
+                        // This combined with a 8192 page size = 256 KB chunk sizes
+                        maxOrder = 5;
+                    }
+                }
                 int tinyCacheSize = PooledByteBufAllocator.defaultTinyCacheSize();
                 int smallCacheSize = PooledByteBufAllocator.defaultSmallCacheSize();
                 int normalCacheSize = PooledByteBufAllocator.defaultNormalCacheSize();
                 boolean useCacheForAllThreads = PooledByteBufAllocator.defaultUseCacheForAllThreads();
                 delegate = new PooledByteBufAllocator(false, nHeapArena, 0, pageSize, maxOrder, tinyCacheSize,
                     smallCacheSize, normalCacheSize, useCacheForAllThreads);
+                ByteSizeValue chunkSize = new ByteSizeValue(pageSize << maxOrder);
+                DESCRIPTION = "[name=elasticsearch_configured, chunk_size=" + chunkSize
+                    + ", factors={es.unsafe.use_netty_default_chunk_and_page_size=" + useDefaultChunkAndPageSize()
+                    + ", g1gc_enabled=" + g1gcEnabled
+                    + ", g1gc_region_size=" + g1gcRegionSize + "}]";
             }
             ALLOCATOR = new NoDirectBuffers(delegate);
         }
@@ -64,6 +100,10 @@ public static ByteBufAllocator getAllocator() {
         return ALLOCATOR;
     }
 
+    public static String getAllocatorDescription() {
+        return DESCRIPTION;
+    }
+
     public static Class<? extends Channel> getChannelType() {
         if (ALLOCATOR instanceof NoDirectBuffers) {
             return CopyBytesSocketChannel.class;
@@ -80,12 +120,34 @@ public static Class<? extends ServerChannel> getServerChannelType() {
         }
     }
 
-    private static boolean useUnpooled() {
+    private static boolean useUnpooled(long heapSizeInBytes, boolean g1gcEnabled, boolean g1gcRegionSizeIsKnown, long g1RegionSize) {
+        if (userForcedUnpooled()) {
+            return true;
+        } else if (heapSizeInBytes <= 1 << 30) {
+            // If the heap is 1GB or less we use unpooled
+            return true;
+        } else if (g1gcEnabled == false) {
+            return false;
+        } else {
+            // If the G1GC is enabled and the region size is known and is less than 1MB we use unpooled.
+            boolean g1gcRegionIsLessThan1MB = g1RegionSize < 1 << 20;
+            return (g1gcRegionSizeIsKnown && g1gcRegionIsLessThan1MB);
+        }
+    }
+
+    private static boolean userForcedUnpooled() {
         if (System.getProperty(USE_UNPOOLED) != null) {
             return Booleans.parseBoolean(System.getProperty(USE_UNPOOLED));
         } else {
-            long heapSize = JvmInfo.jvmInfo().getMem().getHeapMax().getBytes();
-            return heapSize <= 1 << 30;
+            return false;
+        }
+    }
+
+    private static boolean useDefaultChunkAndPageSize() {
+        if (System.getProperty(USE_NETTY_DEFAULT_CHUNK) != null) {
+            return Booleans.parseBoolean(System.getProperty(USE_NETTY_DEFAULT_CHUNK));
+        } else {
+            return false;
         }
     }