Getting socket time out on upload large volume stream (100GB) #98
Description
Steps to reproduce
below code working fine till 50GBand when did top on current running pod its still not releasing memory after job done as well
CPU(cores) MEMORY(bytes)
34m 45675Mi
Expected behavior
Actual behavior
Exception in thread "Thread-11" java.io.UncheckedIOException: javax.net.ssl.SSLException: java.net.SocketException: Connection reset
at software.amazon.awssdk.utils.async.StoringSubscriber$Event.runtimeError(StoringSubscriber.java:181)
at software.amazon.awssdk.utils.async.ByteBufferStoringSubscriber.transferTo(ByteBufferStoringSubscriber.java:112)
at software.amazon.awssdk.utils.async.ByteBufferStoringSubscriber.blockingTransferTo(ByteBufferStoringSubscriber.java:134)
at software.amazon.awssdk.services.s3.internal.crt.S3CrtRequestBodyStreamAdapter.sendRequestBody(S3CrtRequestBodyStreamAdapter.java:48)
Caused by: javax.net.ssl.SSLException: java.net.SocketException: Connection reset
at sun.security.ssl.Alert.createSSLException(Alert.java:127)
at sun.security.ssl.TransportContext.fatal(TransportContext.java:370)
at sun.security.ssl.TransportContext.fatal(TransportContext.java:313)
at sun.security.ssl.TransportContext.fatal(TransportContext.java:308)
at sun.security.ssl.SSLTransport.decode(SSLTransport.java:141)
at sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1293)
at sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1260)
at sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:75)
at sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:926)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.read(SessionInputBufferImpl.java:197)
at org.apache.http.impl.io.ContentLengthInputStream.read(ContentLengthInputStream.java:176)
at org.apache.http.conn.EofSensorInputStream.read(EofSensorInputStream.java:135)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.event.ProgressInputStream.read(ProgressInputStream.java:180)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.event.ProgressInputStream.read(ProgressInputStream.java:180)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.util.LengthCheckInputStream.read(LengthCheckInputStream.java:107)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.services.s3.internal.S3AbortableInputStream.read(S3AbortableInputStream.java:125)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at java.io.FilterInputStream.read(FilterInputStream.java:133)
at software.amazon.awssdk.core.internal.io.SdkLengthAwareInputStream.read(SdkLengthAwareInputStream.java:65)
at java.io.FilterInputStream.read(FilterInputStream.java:107)
at software.amazon.awssdk.utils.async.InputStreamConsumingPublisher.doBlockingWrite(InputStreamConsumingPublisher.java:55)
at software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody.writeInputStream(BlockingInputStreamAsyncRequestBody.java:76)
at software.amazon.awssdk.core.internal.async.InputStreamWithExecutorAsyncRequestBody.doBlockingWrite(InputStreamWithExecutorAsyncRequestBody.java:108)
at software.amazon.awssdk.core.internal.async.InputStreamWithExecutorAsyncRequestBody.lambda$subscribe$0(InputStreamWithExecutorAsyncRequestBody.java:81)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Suppressed: java.net.SocketException: Broken pipe (Write failed)
at java.net.SocketOutputStream.socketWrite0(Native Method)
at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
at sun.security.ssl.SSLSocketOutputRecord.encodeAlert(SSLSocketOutputRecord.java:83)
at sun.security.ssl.TransportContext.fatal(TransportContext.java:401)
... 34 more
Caused by: java.net.SocketException: Connection reset
at java.net.SocketInputStream.read(SocketInputStream.java:210)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:475)
at sun.security.ssl.SSLSocketInputRecord.readFully(SSLSocketInputRecord.java:458)
at sun.security.ssl.SSLSocketInputRecord.decodeInputRecord(SSLSocketInputRecord.java:242)
at sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:180)
at sun.security.ssl.SSLTransport.decode(SSLTransport.java:110)
... 31 more
and stack trace:
org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:793)\n\t\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)\n\t\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)\n\t\tat org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:89)\n\t\tat org.springframework.cloud.sleuth.instrument.async.TraceAsyncAspect.traceBackgroundThread(TraceAsyncAspect.java:64)\n\t\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\t\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\t\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\t\tat java.lang.reflect.Method.invoke(Method.java:498)\n\t\tat org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:634)\n\t\tat org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:624)\n\t\tat org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:72)\n\t\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)\n\t\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)\n\t\tat org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:97)\n\t\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)\n\t\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)\n\t\tat org.springframework.aop.interceptor.AsyncExecutionInterceptor.lambda$invoke$0(AsyncExecutionInterceptor.java:115)\n\t\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\t\tat org.springframework.cloud.sleuth.instrument.async.TraceRunnable.run(TraceRunnable.java:64)\n\t\tat net.chase.ccb.photon.core.concurrent.PhotonRunnable.run(PhotonRunnable.java:34)\n\t\tat net.chase.ccb.photon.core.concurrent.PhotonRunnable.run(PhotonRunnable.java:34)\n\t\tat java.lang.Thread.run(Thread.java:748)\n\tCaused by: java.util.concurrent.CompletionException: software.amazon.awssdk.core.exception.SdkClientException: Failed to send the request: A callback has reported failure.\n\t\tat software.amazon.awssdk.utils.CompletableFutureUtils.errorAsCompletionException(CompletableFutureUtils.java:65)\n\t\tat software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncExecutionFailureExceptionReportingStage.lambda$execute$0(AsyncExecutionFailureExceptionReportingStage.java:51)\n\t\tat java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:836)\n\t\tat java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:811)\n\t\tat java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)\n\t\tat java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)\n\t\tat software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)\n\t\tat java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774)\n\t\tat java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750)\n\t\tat java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)\n\t\tat java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)\n\t\tat software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.maybeAttemptExecute(AsyncRetryableStage.java:103)\n\t\tat software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.maybeRetryExecute(AsyncRetryableStage.java:184)\n\t\tat software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.lambda$attemptExecute$1(AsyncRetryableStage.java:159)\n\t\tat java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774)\n\t\tat java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750)\n\t\tat java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)\n\t\tat java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)\n\t\tat software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)\n\t\tat java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774)\n\t\tat java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750)\n\t\tat java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)\n\t\tat java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)\n\t\tat software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$null$0(MakeAsyncHttpRequestStage.java:103)\n\t\tat java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774)\n\t\tat java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750)\n\t\tat java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)\n\t\tat java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)\n\t\tat software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$executeHttpRequest$3(MakeAsyncHttpRequestStage.java:165)\n\t\tat java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774)\n\t\tat java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750)\n\t\tat java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456)\n\t\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\t\t... 1 common frames omitted\n\tCaused by: software.amazon.awssdk.core.exception.SdkClientException: Failed to send the request: A callback has reported failure.\n\t\tat software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)\n\t\tat software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:43)\n\t\tat software.amazon.awssdk.services.s3.internal.crt.S3CrtResponseHandlerAdapter.handleError(S3CrtResponseHandlerAdapter.java:127)\n\t\tat software.amazon.awssdk.services.s3.internal.crt.S3CrtResponseHandlerAdapter.onFinished(S3CrtResponseHandlerAdapter.java:93)\n\t\tat software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandlerNativeAdapter.onFinished(S3MetaRequestResponseHandlerNativeAdapter.java:24)\n"
System configuration
amazonS3Client = S3AsyncClient.crtBuilder()
.credentialsProvider(StaticCredentialsProvider
.create(AwsSessionCredentials
.create("AccessKey", "SecretKey", "sessionToken"())))
.region(Region.US_EAST_1)
.maxConcurrency(500).minimumPartSizeInBytes((long) 12514185)
.targetThroughputInGbps(20.0)
.httpConfiguration(S3CrtHttpConfiguration.builder().proxyConfiguration(S3CrtProxyConfiguration.builder()
.host("").port()).build()).build())
.build();
PutObjectRequest putObjectRequest = PutObjectRequest.builder().bucket(<Bucket>).key(filePath).build();
CompletableFuture<PutObjectResponse> putObjectResponse =
amazonS3Client.putObject(putObjectRequest,
AsyncRequestBody.fromInputStream(inputStream, fileSize, Executors.newFixedThreadPool( 10)));
putObjectResponse.join();