Return 503 on hitting distributor client inflight request limit

Signed-off-by: Anna Tran <[email protected]>

Author: anna-tran

pkg/distributor/distributor.go

@@ -55,7 +55,6 @@ var (
 	// Distributor instance limits errors.
 	errTooManyInflightPushRequests    = errors.New("too many inflight push requests in distributor")
 	errMaxSamplesPushRateLimitReached = errors.New("distributor's samples push rate limit reached")
-	errTooManyInflightClientRequests  = errors.New("too many inflight ingester client requests in distributor")
 )
 
 const (
@@ -680,7 +679,7 @@ func (d *Distributor) Push(ctx context.Context, req *cortexpb.WriteRequest) (*co
 	// only reject requests at this stage to allow distributor to finish sending the current batch request to all ingesters
 	// even if we've exceeded the MaxInflightClientRequests in the `doBatch`
 	if d.cfg.InstanceLimits.MaxInflightClientRequests > 0 && d.inflightClientRequests.Load() > int64(d.cfg.InstanceLimits.MaxInflightClientRequests) {
-		return nil, errTooManyInflightClientRequests
+		return nil, httpgrpc.Errorf(http.StatusServiceUnavailable, "too many inflight ingester client requests in distributor")
 	}
 
 	removeReplica := false

pkg/distributor/distributor_test.go

@@ -863,7 +863,8 @@ func TestDistributor_PushInstanceLimits(t *testing.T) {
 			preInflightClient:   103,
 			inflightClientLimit: 101,
 			pushes: []testPush{
-				{samples: 100, expectedError: errTooManyInflightClientRequests},
+				{samples: 100, expectedError: httpgrpc.Errorf(http.StatusServiceUnavailable,
+					"too many inflight ingester client requests in distributor")},
 			},
 		},
 		"below ingestion rate limit": {