Use Existing Netty Allocator in Azure Repository (#93906)

Since we're now using the Netty transport exclusively,
we can reuse the Netty module in the Azure plugin.
As we're already aligning versions across the Netty transport
and the Azure plugin this shouldn't change any behavior and saves
code for Gradle exclusions.
Also, we can then just reuse the memory allocator we use in the transport
in the Azure repository to eliminate direct allocations.

Author: original-brownbear

modules/repository-azure/build.gradle

@@ -3,9 +3,6 @@ import org.elasticsearch.gradle.PropertyNormalization
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
 
-import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT
-import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
-
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
@@ -62,26 +59,20 @@ dependencies {
   api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}"
 
   // netty
-  api "io.netty:netty-buffer:${versions.netty}"
   api "io.netty:netty-codec-dns:${versions.netty}"
-  api "io.netty:netty-codec-http:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
-  api "io.netty:netty-codec:${versions.netty}"
-  api "io.netty:netty-common:${versions.netty}"
   api "io.netty:netty-handler-proxy:${versions.netty}"
-  api "io.netty:netty-handler:${versions.netty}"
-  api "io.netty:netty-resolver:${versions.netty}"
   api "io.netty:netty-resolver-dns:${versions.netty}"
-  api "io.netty:netty-transport:${versions.netty}"
-  api "io.netty:netty-transport-native-unix-common:${versions.netty}"
 
   // reactor
   api "io.projectreactor.netty:reactor-netty-core:${versions.reactorNetty}"
   api "io.projectreactor.netty:reactor-netty-http:${versions.reactorNetty}"
   api "io.projectreactor:reactor-core:${versions.reactorCore}"
   api "org.reactivestreams:reactive-streams:${versions.reactiveStreams}"
 
+  implementation project(":modules:transport-netty4")
+
   runtimeOnly("org.slf4j:slf4j-api:${versions.slf4j}")
   runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}")
 
@@ -116,14 +107,6 @@ tasks.named("dependencyLicenses").configure {
 
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
-    'com.aayushatharva.brotli4j.Brotli4jLoader',
-    'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
-    'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
-    'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
-    'com.aayushatharva.brotli4j.encoder.Encoder',
-    'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
-    'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
-
     'com.azure.storage.internal.avro.implementation.AvroObject',
     'com.azure.storage.internal.avro.implementation.AvroReader',
     'com.azure.storage.internal.avro.implementation.AvroReaderFactory',
@@ -162,103 +145,6 @@ tasks.named("thirdPartyAudit").configure {
     'kotlin.reflect.KDeclarationContainer',
     'kotlin.sequences.Sequence',
 
-    // from io.netty.handler.codec.protobuf.ProtobufDecoder (netty)
-    'com.google.protobuf.ExtensionRegistry',
-    'com.google.protobuf.MessageLite$Builder',
-    'com.google.protobuf.MessageLite',
-    'com.google.protobuf.Parser',
-    'com.google.protobuf.ExtensionRegistryLite',
-    'com.google.protobuf.MessageLiteOrBuilder',
-    'com.google.protobuf.nano.CodedOutputByteBufferNano',
-    'com.google.protobuf.nano.MessageNano',
-
-    // from io.netty.logging.CommonsLoggerFactory (netty)
-    'org.apache.commons.logging.Log',
-    'org.apache.commons.logging.LogFactory',
-
-    // from Log4j (deliberate, Netty will fallback to Log4j 2)
-    'org.apache.log4j.Level',
-    'org.apache.log4j.Logger',
-
-    // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
-    'org.bouncycastle.cert.X509v3CertificateBuilder',
-    'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
-    'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
-    'org.bouncycastle.openssl.PEMEncryptedKeyPair',
-    'org.bouncycastle.openssl.PEMParser',
-    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
-    'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
-    'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
-    'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
-
-    // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
-    'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
-    'org.eclipse.jetty.npn.NextProtoNego$ServerProvider',
-    'org.eclipse.jetty.npn.NextProtoNego',
-
-    // from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty)
-    'org.jboss.marshalling.ByteInput',
-
-    // from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty)
-    'org.jboss.marshalling.ByteOutput',
-
-    // from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty)
-    'org.jboss.marshalling.Marshaller',
-
-    // from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty)
-    'org.jboss.marshalling.MarshallerFactory',
-    'org.jboss.marshalling.MarshallingConfiguration',
-    'org.jboss.marshalling.Unmarshaller',
-
-    // from io.netty.handler.codec.spdy.SpdyHeaderBlockJZlibEncoder (netty-codec-http)
-    'com.jcraft.jzlib.Deflater',
-    'com.jcraft.jzlib.Inflater',
-    'com.jcraft.jzlib.JZlib$WrapperType',
-    'com.jcraft.jzlib.JZlib',
-
-    'com.github.luben.zstd.Zstd',
-
-    // from io.netty.handler.codec.compression.LzfDecoder
-    // from io.netty.handler.codec.compression.LzfEncoder (netty-codec)
-    'com.ning.compress.BufferRecycler',
-    'com.ning.compress.lzf.ChunkDecoder',
-    'com.ning.compress.lzf.ChunkEncoder',
-    'com.ning.compress.lzf.LZFChunk',
-    'com.ning.compress.lzf.LZFEncoder',
-    'com.ning.compress.lzf.util.ChunkDecoderFactory',
-    'com.ning.compress.lzf.util.ChunkEncoderFactory',
-
-    // from io.netty.handler.codec.compression.LzmaFrameEncoder (netty-codec)
-    'lzma.sdk.lzma.Encoder',
-
-    // from io.netty.handler.ssl.JettyAlpnSslEngin (netty-handler optional dependency)
-    'org.eclipse.jetty.alpn.ALPN$ClientProvider',
-    'org.eclipse.jetty.alpn.ALPN$ServerProvider',
-    'org.eclipse.jetty.alpn.ALPN',
-
-    // from io.netty.handler.ssl.ConscryptAlpnSslEngine (netty-handler optional dependency)
-    'org.conscrypt.AllocatedBuffer',
-    'org.conscrypt.BufferAllocator',
-    'org.conscrypt.Conscrypt',
-    'org.conscrypt.HandshakeListener',
-
-    // from io.netty.handler.ssl.OpenSslEngine (netty)
-    'io.netty.internal.tcnative.AsyncSSLPrivateKeyMethod',
-    'io.netty.internal.tcnative.AsyncTask',
-    'io.netty.internal.tcnative.Buffer',
-    'io.netty.internal.tcnative.Library',
-    'io.netty.internal.tcnative.SSL',
-    'io.netty.internal.tcnative.SSLContext',
-    'io.netty.internal.tcnative.SSLPrivateKeyMethod',
-    'io.netty.internal.tcnative.CertificateCallback',
-    'io.netty.internal.tcnative.CertificateCompressionAlgo',
-    'io.netty.internal.tcnative.CertificateVerifier',
-    'io.netty.internal.tcnative.ResultCallback',
-    'io.netty.internal.tcnative.SessionTicketKey',
-    'io.netty.internal.tcnative.SniHostNameMatcher',
-    'io.netty.internal.tcnative.SSLSession',
-    'io.netty.internal.tcnative.SSLSessionCache',
-
     // from io.netty.util.internal.Hidden (netty-common optional dependency)
     'reactor.blockhound.BlockHound$Builder',
     'reactor.blockhound.integration.BlockHoundIntegration',
@@ -294,26 +180,6 @@ tasks.named("thirdPartyAudit").configure {
   )
 
   ignoreViolations(
-    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
-
-    'io.netty.util.internal.PlatformDependent0',
-    'io.netty.util.internal.PlatformDependent0$1',
-    'io.netty.util.internal.PlatformDependent0$2',
-    'io.netty.util.internal.PlatformDependent0$3',
-    'io.netty.util.internal.PlatformDependent0$4',
-    'io.netty.util.internal.PlatformDependent0$6',
-    'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef',
-    'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef',
-    'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields',
-    'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields',
-    'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields',
-    'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode',
-    'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
-    'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
-    'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
-    'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
-    'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
-
     'javax.activation.MailcapCommandMap',
     'javax.activation.MimetypesFileTypeMap',
     'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException',

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java

@@ -9,7 +9,6 @@
 package org.elasticsearch.repositories.azure;
 
 import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.PooledByteBufAllocator;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.EventLoopGroup;
 import io.netty.channel.nio.NioEventLoopGroup;
@@ -45,6 +44,7 @@
 import org.elasticsearch.repositories.azure.executors.PrivilegedExecutor;
 import org.elasticsearch.repositories.azure.executors.ReactorScheduledExecutorService;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.netty4.NettyAllocator;
 
 import java.io.IOException;
 import java.net.URL;
@@ -141,22 +141,9 @@ static AzureClientProvider create(ThreadPool threadPool, Settings settings) {
             .maxIdleTime(Duration.ofMillis(maxIdleTime.millis()))
             .build();
 
-        ByteBufAllocator pooledByteBufAllocator = createByteBufAllocator();
-
         // Just to verify that this executor exists
         threadPool.executor(REPOSITORY_THREAD_POOL_NAME);
-        return new AzureClientProvider(threadPool, REPOSITORY_THREAD_POOL_NAME, eventLoopGroup, provider, pooledByteBufAllocator);
-    }
-
-    private static ByteBufAllocator createByteBufAllocator() {
-        int nHeapArena = 1;
-        int pageSize = PooledByteBufAllocator.defaultPageSize();
-        int maxOrder = PooledByteBufAllocator.defaultMaxOrder();
-        int tinyCacheSize = PooledByteBufAllocator.defaultTinyCacheSize();
-        int smallCacheSize = PooledByteBufAllocator.defaultSmallCacheSize();
-        int normalCacheSize = PooledByteBufAllocator.defaultNormalCacheSize();
-
-        return new PooledByteBufAllocator(false, nHeapArena, 0, pageSize, maxOrder, tinyCacheSize, smallCacheSize, normalCacheSize, false);
+        return new AzureClientProvider(threadPool, REPOSITORY_THREAD_POOL_NAME, eventLoopGroup, provider, NettyAllocator.getAllocator());
     }
 
     AzureBlobServiceClient createClient(

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java

@@ -60,11 +60,10 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, R
     // protected for testing
     final SetOnce<AzureStorageService> azureStoreService = new SetOnce<>();
     private final Settings settings;
-    private final Map<String, AzureStorageSettings> initialClientSettings;
 
     public AzureRepositoryPlugin(Settings settings) {
         // eagerly load client settings so that secure settings are read
-        this.initialClientSettings = AzureStorageSettings.load(settings);
+        AzureStorageSettings.load(settings);
         this.settings = settings;
     }