generic proxy: supported custom substitution format specifier for tracing custom tags (#41842)

wbpcode · web-flow · commit bdfd1e911684 · 2025-11-05T14:48:30.000+08:00
Commit Message: generic proxy: supported custom substitution format
specifier for tracing custom tags
Additional Description:

Supported custom substitution format specifier for tracing custom tags
in the generic proxy.

Risk Level: low.
Testing: unit.
Docs Changes: added.
Release Notes: n/a.
Platform Specific Features: n/a.

---------

Signed-off-by: WangBaiping &lt;wbphub@gmail.com&gt;
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
@@ -163,6 +163,13 @@ new_features:
     Added new :ref:`value <envoy_v3_api_field_type.tracing.v3.CustomTag.value>` field and
     the :ref:`substitution format specifier <config_access_log_format>` could be used to
     extract values from various parts of the request/response for custom tags.
+- area: generic_proxy
+  change: |
+    Added custom substitution format specifiers support in the tracing custom tags of the
+    :ref:`generic_proxy
+    filter <envoy_v3_api_msg_extensions.filters.network.generic_proxy.v3.GenericProxy>`.
+    Now the ``%REQUEST_PROPERTY%``, ``%RESPONSE_PROPERTY%`` etc. can be used in the
+    :ref:`value <envoy_v3_api_field_type.tracing.v3.CustomTag.value>` field for generic proxy.
 - area: lua
   change: |
     Added ``drainConnectionUponCompletion()`` to the Lua filter stream info API. This allows Lua scripts
diff --git a/docs/root/configuration/listeners/network_filters/generic_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/generic_proxy_filter.rst
@@ -153,6 +153,7 @@ are supported in the access log format. In addition, the generic proxy also supp
 * ``%RESPONSE_PROPERTY(X)%``: The response property of generic response. The ``X`` is the property name. The value value depends on the
   application codec implementation.
 
+The above custom substitution format specifiers could be used in the access log format and also the tracing custom tags value field.
 
 Generic proxy statistics
 ------------------------
diff --git a/envoy/formatter/substitution_formatter_base.h b/envoy/formatter/substitution_formatter_base.h
@@ -83,6 +83,7 @@ class CommandParser {
 };
 
 using CommandParserPtr = std::unique_ptr<CommandParser>;
+using CommandParserPtrVector = std::vector<CommandParserPtr>;
 
 class CommandParserFactory : public Config::TypedFactory {
 public:
diff --git a/source/common/tracing/custom_tag_impl.cc b/source/common/tracing/custom_tag_impl.cc
@@ -146,8 +146,10 @@ MetadataCustomTag::metadata(const CustomTagContext& ctx) const {
   }
 }
 
-FormatterCustomTag::FormatterCustomTag(absl::string_view tag, absl::string_view value) : tag_(tag) {
-  auto formatter_or = Formatter::FormatterImpl::create(value, true);
+FormatterCustomTag::FormatterCustomTag(absl::string_view tag, absl::string_view value,
+                                       const Formatter::CommandParserPtrVector& command_parsers)
+    : tag_(tag) {
+  auto formatter_or = Formatter::FormatterImpl::create(value, true, command_parsers);
   THROW_IF_NOT_OK_REF(formatter_or.status());
   formatter_ = std::move(formatter_or.value());
 }
@@ -171,7 +173,8 @@ void FormatterCustomTag::applyLog(envoy::data::accesslog::v3::AccessLogCommon& e
 }
 
 CustomTagConstSharedPtr
-CustomTagUtility::createCustomTag(const envoy::type::tracing::v3::CustomTag& tag) {
+CustomTagUtility::createCustomTag(const envoy::type::tracing::v3::CustomTag& tag,
+                                  const Formatter::CommandParserPtrVector& command_parsers) {
   switch (tag.type_case()) {
   case envoy::type::tracing::v3::CustomTag::TypeCase::kLiteral:
     return std::make_shared<const Tracing::LiteralCustomTag>(tag.tag(), tag.literal());
@@ -182,7 +185,8 @@ CustomTagUtility::createCustomTag(const envoy::type::tracing::v3::CustomTag& tag
   case envoy::type::tracing::v3::CustomTag::TypeCase::kMetadata:
     return std::make_shared<const Tracing::MetadataCustomTag>(tag.tag(), tag.metadata());
   case envoy::type::tracing::v3::CustomTag::TypeCase::kValue:
-    return std::make_shared<const Tracing::FormatterCustomTag>(tag.tag(), tag.value());
+    return std::make_shared<const Tracing::FormatterCustomTag>(tag.tag(), tag.value(),
+                                                               command_parsers);
   case envoy::type::tracing::v3::CustomTag::TypeCase::TYPE_NOT_SET:
     break; // Panic below.
   }
diff --git a/source/common/tracing/custom_tag_impl.h b/source/common/tracing/custom_tag_impl.h
@@ -78,7 +78,8 @@ class MetadataCustomTag : public CustomTagBase {
 
 class FormatterCustomTag : public CustomTag {
 public:
-  FormatterCustomTag(absl::string_view tag, absl::string_view value);
+  FormatterCustomTag(absl::string_view tag, absl::string_view value,
+                     const Formatter::CommandParserPtrVector& command_parsers = {});
 
   absl::string_view tag() const override { return tag_; }
   void applySpan(Span& span, const CustomTagContext& ctx) const override;
@@ -96,7 +97,9 @@ class CustomTagUtility {
    * Create a custom tag according to the configuration.
    * @param tag a tracing custom tag configuration.
    */
-  static CustomTagConstSharedPtr createCustomTag(const envoy::type::tracing::v3::CustomTag& tag);
+  static CustomTagConstSharedPtr
+  createCustomTag(const envoy::type::tracing::v3::CustomTag& tag,
+                  const Formatter::CommandParserPtrVector& command_parsers = {});
 };
 
 } // namespace Tracing
diff --git a/source/common/tracing/tracer_config_impl.h b/source/common/tracing/tracer_config_impl.h
@@ -33,7 +33,8 @@ class TracerFactoryContextImpl : public Server::Configuration::TracerFactoryCont
 class ConnectionManagerTracingConfig {
 public:
   ConnectionManagerTracingConfig(envoy::config::core::v3::TrafficDirection traffic_direction,
-                                 const ConnectionManagerTracingConfigProto& tracing_config) {
+                                 const ConnectionManagerTracingConfigProto& tracing_config,
+                                 const Formatter::CommandParserPtrVector& command_parsers = {}) {
 
     // Listener level traffic direction overrides the operation name
     switch (traffic_direction) {
@@ -54,7 +55,8 @@ class ConnectionManagerTracingConfig {
         PROTOBUF_GET_WRAPPED_OR_DEFAULT(tracing_config, spawn_upstream_span, false);
 
     for (const auto& tag : tracing_config.custom_tags()) {
-      custom_tags_.emplace(tag.tag(), Tracing::CustomTagUtility::createCustomTag(tag));
+      custom_tags_.emplace(tag.tag(),
+                           Tracing::CustomTagUtility::createCustomTag(tag, command_parsers));
     }
 
     client_sampling_.set_numerator(
diff --git a/source/extensions/filters/network/generic_proxy/config.cc b/source/extensions/filters/network/generic_proxy/config.cc
@@ -118,8 +118,10 @@ Factory::createFilterFactoryFromProtoTyped(const ProxyConfig& proto_config,
     if (proto_config.tracing().has_provider()) {
       tracer = tracer_manager->getOrCreateTracer(&proto_config.tracing().provider());
     }
+    std::vector<Formatter::CommandParserPtr> command_parsers;
+    command_parsers.push_back(createGenericProxyCommandParser());
     tracing_config = std::make_unique<Tracing::ConnectionManagerTracingConfig>(
-        context.listenerInfo().direction(), proto_config.tracing());
+        context.listenerInfo().direction(), proto_config.tracing(), command_parsers);
   }
 
   // Access log configuration.
diff --git a/source/extensions/filters/network/generic_proxy/proxy.cc b/source/extensions/filters/network/generic_proxy/proxy.cc
@@ -96,6 +96,7 @@ void ActiveStream::modifySpan(Tracing::Span& span) const {
   const FormatterContextExtension context_extension(request_header_frame_.get(),
                                                     response_header_frame_.get());
   Formatter::Context context;
+  context.setExtension(context_extension);
   const Tracing::CustomTagContext ctx{trace_context, stream_info_, context};
 
   for (const auto& it : conn_manager_tracing_config_->getCustomTags()) {
diff --git a/test/common/tracing/BUILD b/test/common/tracing/BUILD
@@ -87,6 +87,7 @@ envoy_cc_test(
     ],
     rbe_pool = "6gig",
     deps = [
+        "//source/common/formatter:formatter_extension_lib",
         "//source/common/tracing:tracer_config_lib",
     ],
 )
diff --git a/test/common/tracing/tracer_config_impl_test.cc b/test/common/tracing/tracer_config_impl_test.cc
@@ -44,13 +44,16 @@ TEST(ConnectionManagerTracingConfigImplTest, SimpleTest) {
     auto* custom_tag = tracing_config.add_custom_tags();
     custom_tag->set_tag("foo");
     custom_tag->mutable_literal()->set_value("bar");
+    auto* custom_tag2 = tracing_config.add_custom_tags();
+    custom_tag2->set_tag("dynamic_foo");
+    custom_tag2->set_value("%REQ(X-FOO)%");
 
     ConnectionManagerTracingConfig config(traffic_direction, tracing_config);
 
     EXPECT_EQ(Tracing::OperationName::Egress, config.operationName());
     EXPECT_EQ(true, config.verbose());
     EXPECT_EQ(256, config.maxPathTagLength());
-    EXPECT_EQ(1, config.getCustomTags().size());
+    EXPECT_EQ(2, config.getCustomTags().size());
 
     EXPECT_EQ(100, config.getClientSampling().numerator());
     EXPECT_EQ(10000, config.getRandomSampling().numerator());
diff --git a/test/extensions/filters/network/generic_proxy/config_test.cc b/test/extensions/filters/network/generic_proxy/config_test.cc
@@ -369,6 +369,9 @@ TEST(BasicFilterConfigTest, TestConfigurationWithTracing) {
       route_config_name: test_route
     tracing:
       max_path_tag_length: 128
+      custom_tags:
+      - tag: "trace-id"
+        value: "%REQUEST_PROPERTY(X-TRACE-ID)%"
       provider:
         name: zipkin
         typed_config:
diff --git a/test/extensions/filters/network/generic_proxy/proxy_test.cc b/test/extensions/filters/network/generic_proxy/proxy_test.cc
@@ -56,14 +56,19 @@ class FilterConfigTest : public testing::Test {
       const std::string tracing_config_yaml = R"EOF(
       max_path_tag_length: 256
       spawn_upstream_span: true
+      custom_tags:
+      - tag: "x-key"
+        value: "%REQUEST_PROPERTY(x-key)%"
       )EOF";
 
       Tracing::ConnectionManagerTracingConfigProto tracing_config;
 
       TestUtility::loadFromYaml(tracing_config_yaml, tracing_config);
 
+      std::vector<Formatter::CommandParserPtr> command_parsers;
+      command_parsers.push_back(createGenericProxyCommandParser());
       tracing_config_ = std::make_unique<Tracing::ConnectionManagerTracingConfig>(
-          envoy::config::core::v3::TrafficDirection::OUTBOUND, tracing_config);
+          envoy::config::core::v3::TrafficDirection::OUTBOUND, tracing_config, command_parsers);
     }
 
     std::vector<NamedFilterFactoryCb> factories;
@@ -74,6 +79,9 @@ class FilterConfigTest : public testing::Test {
           {"mock_default_decoder_filter", std::make_shared<NiceMock<MockDecoderFilter>>()});
     }
 
+    factories.reserve(mock_decoder_filters_.size() + mock_encoder_filters_.size() +
+                      mock_stream_filters_.size());
+
     for (const auto& filter : mock_stream_filters_) {
       factories.push_back({filter.first, [f = filter.second](FilterChainFactoryCallbacks& cb) {
                              cb.addFilter(f);
@@ -1814,6 +1822,7 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithTracing) {
   initializeFilter(true);
 
   auto request = std::make_unique<FakeStreamCodecFactory::FakeRequest>();
+  request->data_["x-key"] = "x-value"; // The custom tag will extract this key-value pair.
 
   auto* span = new NiceMock<Tracing::MockSpan>();
   EXPECT_CALL(*tracer_, startSpan_(_, _, _, _))
@@ -1841,7 +1850,12 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithTracing) {
 
   auto active_stream = filter_->activeStreamsForTest().begin()->get();
 
-  EXPECT_CALL(*span, setTag(_, _)).Times(testing::AnyNumber());
+  absl::flat_hash_map<std::string, std::string> final_tags;
+  EXPECT_CALL(*span, setTag(_, _))
+      .WillRepeatedly(
+          testing::Invoke([&final_tags](absl::string_view key, absl::string_view value) {
+            final_tags[key] = std::string(value);
+          }));
   EXPECT_CALL(*span, finishSpan());
 
   EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false));
@@ -1863,6 +1877,9 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithTracing) {
 
   auto response = std::make_unique<FakeStreamCodecFactory::FakeResponse>();
   active_stream->onResponseHeaderFrame(std::move(response));
+
+  // Check the tracing tags after the stream is completed.
+  EXPECT_EQ(final_tags["x-key"], "x-value");
 }
 
 TEST_F(FilterTest, NewStreamAndReplyNormallyWithTracingAndSamplingToTrue) {

Original file line number	Diff line number	Diff line change
`@@ -146,8 +146,10 @@ MetadataCustomTag::metadata(const CustomTagContext& ctx) const {`
`146`	`146`	`}`
`147`	`147`	`}`
`148`	`148`
`149`		`-FormatterCustomTag::FormatterCustomTag(absl::string_view tag, absl::string_view value) : tag_(tag) {`
`150`		`- auto formatter_or = Formatter::FormatterImpl::create(value, true);`
	`149`	`+FormatterCustomTag::FormatterCustomTag(absl::string_view tag, absl::string_view value,`
	`150`	`+ const Formatter::CommandParserPtrVector& command_parsers)`
	`151`	`+ : tag_(tag) {`
	`152`	`+ auto formatter_or = Formatter::FormatterImpl::create(value, true, command_parsers);`
`151`	`153`	`THROW_IF_NOT_OK_REF(formatter_or.status());`
`152`	`154`	`formatter_ = std::move(formatter_or.value());`
`153`	`155`	`}`
`@@ -171,7 +173,8 @@ void FormatterCustomTag::applyLog(envoy::data::accesslog::v3::AccessLogCommon& e`
`171`	`173`	`}`
`172`	`174`
`173`	`175`	`CustomTagConstSharedPtr`
`174`		`-CustomTagUtility::createCustomTag(const envoy::type::tracing::v3::CustomTag& tag) {`
	`176`	`+CustomTagUtility::createCustomTag(const envoy::type::tracing::v3::CustomTag& tag,`
	`177`	`+ const Formatter::CommandParserPtrVector& command_parsers) {`
`175`	`178`	`switch (tag.type_case()) {`
`176`	`179`	`case envoy::type::tracing::v3::CustomTag::TypeCase::kLiteral:`
`177`	`180`	`return std::make_shared<const Tracing::LiteralCustomTag>(tag.tag(), tag.literal());`
`@@ -182,7 +185,8 @@ CustomTagUtility::createCustomTag(const envoy::type::tracing::v3::CustomTag& tag`
`182`	`185`	`case envoy::type::tracing::v3::CustomTag::TypeCase::kMetadata:`
`183`	`186`	`return std::make_shared<const Tracing::MetadataCustomTag>(tag.tag(), tag.metadata());`
`184`	`187`	`case envoy::type::tracing::v3::CustomTag::TypeCase::kValue:`
`185`		`- return std::make_shared<const Tracing::FormatterCustomTag>(tag.tag(), tag.value());`
	`188`	`+ return std::make_shared<const Tracing::FormatterCustomTag>(tag.tag(), tag.value(),`
	`189`	`+ command_parsers);`
`186`	`190`	`case envoy::type::tracing::v3::CustomTag::TypeCase::TYPE_NOT_SET:`
`187`	`191`	`break; // Panic below.`
`188`	`192`	`}`
Original file line number	Diff line number	Diff line change
`@@ -118,8 +118,10 @@ Factory::createFilterFactoryFromProtoTyped(const ProxyConfig& proto_config,`
`118`	`118`	`if (proto_config.tracing().has_provider()) {`
`119`	`119`	`tracer = tracer_manager->getOrCreateTracer(&proto_config.tracing().provider());`
`120`	`120`	`}`
	`121`	`+ std::vector<Formatter::CommandParserPtr> command_parsers;`
	`122`	`+ command_parsers.push_back(createGenericProxyCommandParser());`
`121`	`123`	`tracing_config = std::make_unique<Tracing::ConnectionManagerTracingConfig>(`
`122`		`- context.listenerInfo().direction(), proto_config.tracing());`
	`124`	`+ context.listenerInfo().direction(), proto_config.tracing(), command_parsers);`
`123`	`125`	`}`
`124`	`126`
`125`	`127`	`// Access log configuration.`
Original file line number	Diff line number	Diff line change
`@@ -87,6 +87,7 @@ envoy_cc_test(`
`87`	`87`	`],`
`88`	`88`	`rbe_pool = "6gig",`
`89`	`89`	`deps = [`
	`90`	`+ "//source/common/formatter:formatter_extension_lib",`
`90`	`91`	`"//source/common/tracing:tracer_config_lib",`
`91`	`92`	`],`
`92`	`93`	`)`