ShardBulkInferenceActionFilter requests that are rejected by InferenceWaitForAllocation should bubble up http code #123731
Description
Elasticsearch Version
current
Installed Plugins
No response
Java Version
bundled
OS Version
any
Problem Description
ShardBulkInferenceActionFilter should bubble up any ElasticsearchStatusException status provided by its underlying actions. An example of this is: InferenceWaitForAllocation, which will throw a 429, but ShardBulkInferenceActionFilter seemingly unwraps/rewraps to be a 500 instead of a 429
Steps to Reproduce
N/A
Logs (if relevant)
org.elasticsearch.ElasticsearchException: Exception when running inference id [.elser-2-elasticsearch] on field [text]
at org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter$AsyncBulkShardInferenceAction$2.onResponse(ShardBulkInferenceActionFilter.java:323)
at org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter$AsyncBulkShardInferenceAction$2.onResponse(ShardBulkInferenceActionFilter.java:310)
at org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker.sendFinalResponse(EmbeddingRequestChunker.java:191)
at org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker$DebatchingListener.onFailure(EmbeddingRequestChunker.java:177)
at org.elasticsearch.action.ActionListenerImplementations.safeAcceptException(ActionListenerImplementations.java:64)
at org.elasticsearch.action.ActionListenerImplementations.safeOnFailure(ActionListenerImplementations.java:75)
at org.elasticsearch.action.DelegatingActionListener.onFailure(DelegatingActionListener.java:32)
at org.elasticsearch.action.ActionListenerImplementations.safeAcceptException(ActionListenerImplementations.java:64)
at org.elasticsearch.action.ActionListenerImplementations.safeOnFailure(ActionListenerImplementations.java:75)
at org.elasticsearch.action.DelegatingActionListener.onFailure(DelegatingActionListener.java:32)
at org.elasticsearch.action.ActionListenerImplementations$RunAfterActionListener.onFailure(ActionListenerImplementations.java:307)
at org.elasticsearch.action.ActionListenerImplementations.safeAcceptException(ActionListenerImplementations.java:64)
at org.elasticsearch.action.ActionListenerImplementations.safeOnFailure(ActionListenerImplementations.java:75)
at org.elasticsearch.action.DelegatingActionListener.onFailure(DelegatingActionListener.java:32)
at org.elasticsearch.action.support.ContextPreservingActionListener.onFailure(ContextPreservingActionListener.java:40)
at org.elasticsearch.tasks.TaskManager$1.onFailure(TaskManager.java:215)
at org.elasticsearch.action.ActionListenerImplementations.safeAcceptException(ActionListenerImplementations.java:64)
at org.elasticsearch.action.ActionListenerImplementations.safeOnFailure(ActionListenerImplementations.java:75)
at org.elasticsearch.action.DelegatingActionListener.onFailure(DelegatingActionListener.java:32)
at org.elasticsearch.action.ActionListenerImplementations$RunBeforeActionListener.onFailure(ActionListenerImplementations.java:346)
at org.elasticsearch.action.ActionListenerImplementations.safeAcceptException(ActionListenerImplementations.java:64)
at org.elasticsearch.action.ActionListenerImplementations.safeOnFailure(ActionListenerImplementations.java:75)
at org.elasticsearch.action.DelegatingActionListener.onFailure(DelegatingActionListener.java:32)
at org.elasticsearch.action.support.ContextPreservingActionListener.onFailure(ContextPreservingActionListener.java:40)
at org.elasticsearch.action.ActionListenerImplementations.safeAcceptException(ActionListenerImplementations.java:64)
at org.elasticsearch.action.ActionListenerImplementations.safeOnFailure(ActionListenerImplementations.java:75)
at org.elasticsearch.action.DelegatingActionListener.onFailure(DelegatingActionListener.java:32)
at org.elasticsearch.xpack.ml.inference.InferenceWaitForAllocation.waitForAssignment(InferenceWaitForAllocation.java:89)
at org.elasticsearch.xpack.ml.action.TransportInternalInferModelAction.inferAgainstAllocatedModel(TransportInternalInferModelAction.java:285)
at org.elasticsearch.xpack.ml.action.TransportInternalInferModelAction.doInfer(TransportInternalInferModelAction.java:188)
at org.elasticsearch.xpack.ml.action.TransportInternalInferModelAction.doExecute(TransportInternalInferModelAction.java:142)
at org.elasticsearch.xpack.ml.action.TransportInternalInferModelAction.doExecute(TransportInternalInferModelAction.java:66)
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:135)
at org.elasticsearch.action.support.ActionFilter$Simple.apply(ActionFilter.java:54)
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:132)
at co.elastic.elasticsearch.metering.activitytracking.ActivityTrackerActionFilter.apply(ActivityTrackerActionFilter.java:54)
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:132)
at org.elasticsearch.action.support.ActionFilter$Simple.apply(ActionFilter.java:54)
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:132)
at org.elasticsearch.action.support.MappedActionFilters$MappedFilterChain.proceed(MappedActionFilters.java:71)
at org.elasticsearch.action.support.MappedActionFilters.apply(MappedActionFilters.java:49)
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:132)
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$4(SecurityActionFilter.java:182)
at org.elasticsearch.action.ActionListenerImplementations$DelegatingFailureActionListener.onResponse(ActionListenerImplementations.java:219)
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:462)
at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:1033)
at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:999)
at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$9(AuthorizationService.java:476)
at org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:247)
at org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:206)
at org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:466)
at org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:442)
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$3(AuthorizationService.java:329)
at org.elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:257)
at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
at org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$0(RBACEngine.java:171)
at org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:247)
at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRoles$4(CompositeRolesStore.java:202)
at org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:247)
at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRole(CompositeRolesStore.java:210)
at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:195)
at org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:167)
at org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:345)
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$5(SecurityActionFilter.java:178)
at org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:247)
at org.elasticsearch.action.ActionListenerImplementations$MappedActionListener.onResponse(ActionListenerImplementations.java:97)
at org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticate(AuthenticatorChain.java:93)
at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:264)
at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:173)
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.applyInternal(SecurityActionFilter.java:174)
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$apply$1(SecurityActionFilter.java:115)
at org.elasticsearch.xpack.core.security.SecurityContext.executeAsInternalUser(SecurityContext.java:170)
at org.elasticsearch.xpack.security.authz.AuthorizationUtils.switchUserBasedOnActionOriginAndExecute(AuthorizationUtils.java:168)
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:111)
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:132)
at org.elasticsearch.action.support.TransportAction.handleExecution(TransportAction.java:96)
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:59)
at org.elasticsearch.tasks.TaskManager.registerAndExecute(TaskManager.java:197)
at org.elasticsearch.client.internal.node.NodeClient.executeLocally(NodeClient.java:106)
at org.elasticsearch.client.internal.node.NodeClient.doExecute(NodeClient.java:84)
at org.elasticsearch.client.internal.support.AbstractClient.execute(AbstractClient.java:140)
at org.elasticsearch.client.internal.FilterClient.doExecute(FilterClient.java:56)
at org.elasticsearch.client.internal.OriginSettingClient.doExecute(OriginSettingClient.java:44)
at org.elasticsearch.client.internal.support.AbstractClient.execute(AbstractClient.java:140)
at org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService$BatchIterator.executeRequest(ElasticsearchInternalService.java:1123)
at org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService$BatchIterator.inferBatch(ElasticsearchInternalService.java:1093)
at org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService$BatchIterator.lambda$inferBatch$1(ElasticsearchInternalService.java:1097)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.lang.Thread.run(Thread.java:1575)
Caused by: org.elasticsearch.ElasticsearchStatusException: Rejected inference request waiting for an allocation of deployment [.elser-2-elasticsearch]. Too many pending requests
at org.elasticsearch.xpack.ml.inference.InferenceWaitForAllocation.waitForAssignment(InferenceWaitForAllocation.java:93)
... 64 more