Set RCD for 100% drop_overload case as unconditional_drop_overload (envoyproxy#38408)

This is a follow up commit of drop_overload
support:envoyproxy#30827.

It is observed that in some situations, EDS sends drop_overload ratio as
100% for some clusters do not have healthy endpoints in the cluster. For
such cases, it is preferred that the RCD of the local reply set to
"unconditional drop overload" to make it a little bit different from the
normal "drop overload" case.

---------

Signed-off-by: Yanjun Xiang <[email protected]>

Author: yanjunxiang-google

api/envoy/config/accesslog/v3/accesslog.proto

@@ -260,6 +260,7 @@ message ResponseFlagFilter {
         in: "DF"
         in: "DO"
         in: "DR"
+        in: "UDO"
       }
     }
   }];

docs/root/configuration/observability/access_log/usage.rst

@@ -569,6 +569,7 @@ HTTP only
   **DnsResolutionFailed**, **DF**, The request was terminated due to DNS resolution failure.
   **DropOverload**, **DO**, The request was terminated in addition to 503 response code due to :ref:`drop_overloads<envoy_v3_api_field_config.endpoint.v3.ClusterLoadAssignment.Policy.drop_overloads>`.
   **DownstreamRemoteReset**, **DR**, The response details are ``http2.remote_reset`` or ``http2.remote_refuse``.
+  **UnconditionalDropOverload**, **UDO**, The request was terminated in addition to 503 response code due to :ref:`drop_overloads<envoy_v3_api_field_config.endpoint.v3.ClusterLoadAssignment.Policy.drop_overloads>` is set to 100%.
 
 UDP
   Not implemented ("-").

envoy/stream_info/stream_info.h

@@ -95,8 +95,10 @@ enum CoreResponseFlag : uint16_t {
   DropOverLoad,
   // Downstream remote codec level reset was received on the stream.
   DownstreamRemoteReset,
+  // Unconditionally drop all traffic due to drop_overload is set to 100%.
+  UnconditionalDropOverload,
   // ATTENTION: MAKE SURE THIS REMAINS EQUAL TO THE LAST FLAG.
-  LastFlag = DownstreamRemoteReset,
+  LastFlag = UnconditionalDropOverload,
 };
 
 class ResponseFlagUtils;
@@ -196,6 +198,9 @@ struct ResponseCodeDetailValues {
   const std::string MaintenanceMode = "maintenance_mode";
   // The request was rejected by the router filter because the DROP_OVERLOAD configuration.
   const std::string DropOverload = "drop_overload";
+  // The request was rejected by the router filter because the DROP_OVERLOAD configuration is set to
+  // 100%.
+  const std::string UnconditionalDropOverload = "unconditional_drop_overload";
   // The request was rejected by the router filter because there was no healthy upstream found.
   const std::string NoHealthyUpstream = "no_healthy_upstream";
   // The request was forwarded upstream but the response timed out.

source/common/http/headers.h

@@ -178,6 +178,8 @@ class HeaderValues {
   const LowerCaseString EnvoyOriginalPath{absl::StrCat(prefix(), "-original-path")};
   const LowerCaseString EnvoyOverloaded{absl::StrCat(prefix(), "-overloaded")};
   const LowerCaseString EnvoyDropOverload{absl::StrCat(prefix(), "-drop-overload")};
+  const LowerCaseString EnvoyUnconditionalDropOverload{
+      absl::StrCat(prefix(), "-unconditional-drop-overload")};
   const LowerCaseString EnvoyRateLimited{absl::StrCat(prefix(), "-ratelimited")};
   const LowerCaseString EnvoyRetryOn{absl::StrCat(prefix(), "-retry-on")};
   const LowerCaseString EnvoyRetryGrpcOn{absl::StrCat(prefix(), "-retry-grpc-on")};
@@ -286,6 +288,10 @@ class HeaderValues {
     const std::string True{"true"};
   } EnvoyDropOverloadValues;
 
+  struct {
+    const std::string True{"true"};
+  } EnvoyUnconditionalDropOverloadValues;
+
   struct {
     const std::string True{"true"};
   } EnvoyRateLimitedValues;

source/common/router/router.cc

@@ -2229,6 +2229,29 @@ bool Filter::checkDropOverload(Upstream::ThreadLocalCluster& cluster,
   if (cluster.dropOverload().value()) {
     ENVOY_STREAM_LOG(debug, "Router filter: cluster DROP_OVERLOAD configuration: {}", *callbacks_,
                      cluster.dropOverload().value());
+
+    if (cluster.dropOverload().value() == 1.0) {
+      ENVOY_STREAM_LOG(
+          debug, "The configured DROP_OVERLOAD ratio is 100%, drop everything unconditionally.",
+          *callbacks_);
+      callbacks_->streamInfo().setResponseFlag(
+          StreamInfo::CoreResponseFlag::UnconditionalDropOverload);
+      chargeUpstreamCode(Http::Code::ServiceUnavailable, {}, true);
+      callbacks_->sendLocalReply(
+          Http::Code::ServiceUnavailable, "unconditional drop overload",
+          [modify_headers, this](Http::ResponseHeaderMap& headers) {
+            if (!config_->suppress_envoy_headers_) {
+              headers.addReference(Http::Headers::get().EnvoyUnconditionalDropOverload,
+                                   Http::Headers::get().EnvoyUnconditionalDropOverloadValues.True);
+            }
+            modify_headers(headers);
+          },
+          absl::nullopt, StreamInfo::ResponseCodeDetails::get().UnconditionalDropOverload);
+
+      cluster.info()->loadReportStats().upstream_rq_drop_overload_.inc();
+      return true;
+    }
+
     if (config_->random_.bernoulli(cluster.dropOverload())) {
       ENVOY_STREAM_LOG(debug, "The request is dropped by DROP_OVERLOAD", *callbacks_);
       callbacks_->streamInfo().setResponseFlag(StreamInfo::CoreResponseFlag::DropOverLoad);

source/common/stream_info/utility.cc

@@ -86,7 +86,7 @@ ResponseFlag ResponseFlagUtils::registerCustomFlag(absl::string_view custom_flag
 
 const ResponseFlagUtils::ResponseFlagsVecType& ResponseFlagUtils::responseFlagsVec() {
   CONSTRUCT_ON_FIRST_USE(ResponseFlagsVecType, []() {
-    static_assert(CoreResponseFlag::LastFlag == 28,
+    static_assert(CoreResponseFlag::LastFlag == 29,
                   "A flag has been added. Add the new flag to CORE_RESPONSE_FLAGS.");
 
     responseFlagsVecInitialized() = true;

source/common/stream_info/utility.h

@@ -104,6 +104,7 @@ class ResponseFlagUtils {
   constexpr static absl::string_view DNS_FAIL = "DF";
   constexpr static absl::string_view DROP_OVERLOAD = "DO";
   constexpr static absl::string_view DOWNSTREAM_REMOTE_RESET = "DR";
+  constexpr static absl::string_view UNCONDITIONAL_DROP_OVERLOAD = "UDO";
 
   constexpr static absl::string_view DOWNSTREAM_CONNECTION_TERMINATION_LONG =
       "DownstreamConnectionTermination";
@@ -140,6 +141,7 @@ class ResponseFlagUtils {
   constexpr static absl::string_view DNS_FAIL_LONG = "DnsResolutionFailed";
   constexpr static absl::string_view DROP_OVERLOAD_LONG = "DropOverload";
   constexpr static absl::string_view DOWNSTREAM_REMOTE_RESET_LONG = "DownstreamRemoteReset";
+  constexpr static absl::string_view UNCONDITIONAL_DROP_OVERLOAD_LONG = "UnconditionalDropOverload";
 
   static constexpr std::array CORE_RESPONSE_FLAGS{
       FlagStrings{FAILED_LOCAL_HEALTH_CHECK, FAILED_LOCAL_HEALTH_CHECK_LONG,
@@ -189,6 +191,8 @@ class ResponseFlagUtils {
       FlagStrings{DROP_OVERLOAD, DROP_OVERLOAD_LONG, CoreResponseFlag::DropOverLoad},
       FlagStrings{DOWNSTREAM_REMOTE_RESET, DOWNSTREAM_REMOTE_RESET_LONG,
                   CoreResponseFlag::DownstreamRemoteReset},
+      FlagStrings{UNCONDITIONAL_DROP_OVERLOAD, UNCONDITIONAL_DROP_OVERLOAD_LONG,
+                  CoreResponseFlag::UnconditionalDropOverload},
   };
 
 private:

source/extensions/access_loggers/grpc/grpc_access_log_utils.cc

@@ -41,7 +41,7 @@ void Utility::responseFlagsToAccessLogResponseFlags(
     envoy::data::accesslog::v3::AccessLogCommon& common_access_log,
     const StreamInfo::StreamInfo& stream_info) {
 
-  static_assert(StreamInfo::CoreResponseFlag::LastFlag == 28,
+  static_assert(StreamInfo::CoreResponseFlag::LastFlag == 29,
                 "A flag has been added. Fix this code.");
 
   if (stream_info.hasResponseFlag(StreamInfo::CoreResponseFlag::FailedLocalHealthCheck)) {

test/common/access_log/access_log_impl_test.cc

@@ -1067,6 +1067,7 @@ name: accesslog
       - DF
       - DO
       - DR
+      - UDO
 typed_config:
   "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
   path: /dev/null

test/common/router/router_test.cc

@@ -859,6 +859,30 @@ TEST_F(RouterTest, DropOverloadDropped) {
   EXPECT_EQ(callbacks_.details(), "drop_overload");
 }
 
+// UnconditionalDropOverload drop test
+TEST_F(RouterTest, UnconditionalDropOverloadDropped) {
+  EXPECT_CALL(cm_.thread_local_cluster_, dropOverload()).WillRepeatedly(Return(UnitFloat(1.0)));
+  Http::TestResponseHeaderMapImpl response_headers{{":status", "503"},
+                                                   {"content-length", "27"},
+                                                   {"content-type", "text/plain"},
+                                                   {"x-envoy-unconditional-drop-overload", "true"}};
+  EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), false));
+  EXPECT_CALL(callbacks_, encodeData(_, true));
+  EXPECT_CALL(callbacks_.stream_info_,
+              setResponseFlag(StreamInfo::CoreResponseFlag::UnconditionalDropOverload));
+
+  Http::TestRequestHeaderMapImpl headers;
+  HttpTestUtility::addDefaultHeaders(headers);
+  router_->decodeHeaders(headers, true);
+  EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->load_report_stats_store_
+                    .counter("upstream_rq_drop_overload")
+                    .value());
+  EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->load_report_stats_store_
+                    .counter("upstream_rq_dropped")
+                    .value());
+  EXPECT_EQ(callbacks_.details(), "unconditional_drop_overload");
+}
+
 TEST_F(RouterTest, ResponseCodeDetailsSetByUpstream) {
   NiceMock<Http::MockRequestEncoder> encoder1;
   Http::ResponseDecoder* response_decoder = nullptr;