formatter: add query_params to log all params

Fixes: #40925

Signed-off-by: Christian Rohmann <christian.rohmann@inovex.de>

Author: frittentheke

changelogs/current.yaml

@@ -19,5 +19,9 @@ removed_config_or_runtime:
 # *Normally occurs at the end of the* :ref:`deprecation period <deprecated>`
 
 new_features:
+- area: formatter
+  change: |
+    Added ``QUERY_PARAMS`` support for substitution formatter to log all query params.
+    See :ref:`access log formatter <config_access_log_format>` for more details.
 
 deprecated:

docs/root/configuration/advanced/substitution_formatter.rst

@@ -1425,6 +1425,21 @@ Current supported substitution commands include:
   TCP/UDP
     Not implemented. It will appear as ``"-"`` in the access logs.
 
+``%QUERY_PARAMS(X):Z%``
+  HTTP
+    All of the query parameters. The parameter ``X`` is used to specify how the query parameters are presented.
+
+    The ``X`` parameter can be:
+
+    * ``ORIG``: The output will be original query params string part of the path with no treatment. If the ``X`` is
+      not present, ``WQ`` will be used.
+    * ``DECODED``: The query params will be URL decoded.
+
+    ``Z`` is an optional parameter denoting string truncation up to ``Z`` characters long.
+
+  TCP/UDP
+    Not implemented. It will appear as ``"-"`` in the access logs.
+
 ``%PATH(X:Y):Z%``
   HTTP
     The value of the request path. The parameter ``X`` is used to specify whether the output contains

source/common/formatter/http_specific_formatter.cc

@@ -304,6 +304,55 @@ QueryParameterFormatter::formatValue(const Context& context,
   return ValueUtil::optionalStringValue(format(context, stream_info));
 }
 
+QueryParametersFormatter::QueryParametersFormatter(absl::string_view decoding,
+                                                   absl::optional<size_t> max_length)
+    : decoding_(decoding), max_length_(max_length) {}
+
+// FormatterProvider
+absl::optional<std::string> QueryParametersFormatter::format(const Context& context,
+                                                             const StreamInfo::StreamInfo&) const {
+  const auto request_headers = context.requestHeaders();
+  if (!request_headers.has_value()) {
+    return absl::nullopt;
+  }
+
+  bool decode_bool = false;
+  if (decoding.empty()) {
+    decode_bool = false;
+  } else if (decoding == "ORIG") {
+    decode_bool = false;
+  } if (decoding == "DECODED") {
+    decode_bool = true;
+  } else {
+    return absl::InvalidArgumentError(
+        fmt::format("Invalid QUERY_PARAMS option: '{}', only 'ORIG'/'DECODED' are allowed", decoding));
+  }
+
+  // Gather query parameters substring from path
+  absl::string_view path_view = request_headers->getPathValue();
+  auto query_offset = path_view.find('?');
+  absl::optional<string_view> query_params;
+  if (query_offset != absl::string_view::npos) {
+    query_params = path_view.substr(query_offset+1);
+  }
+
+  // Apply query param percent decoding on the query params if requested
+  if (query_params.has_value() && decode_bool) {
+    query_params = Http::Utility::PercentEncoding::urlDecodeQueryParameter(query_params);
+  }
+
+  if (query_params.has_value() && max_length_.has_value()) {
+    SubstitutionFormatUtils::truncate(query_params.value(), max_length_.value());
+  }
+  return query_params;
+}
+
+Protobuf::Value
+QueryParametersFormatter::formatValue(const Context& context,
+                                      const StreamInfo::StreamInfo& stream_info) const {
+  return ValueUtil::optionalStringValue(format(context, stream_info));
+}
+
 absl::optional<std::string> PathFormatter::format(const Context& context,
                                                   const StreamInfo::StreamInfo&) const {
 
@@ -495,6 +544,11 @@ BuiltInHttpCommandParser::getKnownFormatters() {
          [](absl::string_view format, absl::optional<size_t> max_length) {
            return std::make_unique<QueryParameterFormatter>(std::string(format), max_length);
          }}},
+       {"QUERY_PARAMS",
+        {CommandSyntaxChecker::PARAMS_OPTIONAL | CommandSyntaxChecker::LENGTH_ALLOWED,
+         [](absl::string_view decoding, absl::optional<size_t> max_length) {
+           return std::make_unique<QueryParametersFormatter>(std::string(decoding), max_length);
+         }}},
        {"PATH",
         {CommandSyntaxChecker::PARAMS_OPTIONAL | CommandSyntaxChecker::LENGTH_ALLOWED,
          [](absl::string_view format, absl::optional<size_t> max_length) {

source/common/formatter/http_specific_formatter.h

@@ -180,6 +180,21 @@ class QueryParameterFormatter : public FormatterProvider {
   absl::optional<size_t> max_length_;
 };
 
+class QueryParametersFormatter : public FormatterProvider {
+public:
+  QueryParametersFormatter(absl::string_view decoding, absl::optional<size_t> max_length);
+
+  // FormatterProvider
+  absl::optional<std::string> decoding(const Context& context,
+                                     const StreamInfo::StreamInfo& stream_info) const override;
+  Protobuf::Value formatValue(const Context& context,
+                              const StreamInfo::StreamInfo& stream_info) const override;
+
+private:
+  const std::string decoding_;
+  absl::optional<size_t> max_length_;
+};
+
 class PathFormatter : public FormatterProvider {
 public:
   enum PathFormatterOption {

test/common/formatter/substitution_formatter_test.cc

@@ -2766,6 +2766,63 @@ TEST(SubstitutionFormatterTest, QueryParameterFormatter) {
   }
 }
 
+TEST(SubstitutionFormatterTest, QueryParametersFormatter) {
+  StreamInfo::MockStreamInfo stream_info;
+  Http::TestRequestHeaderMapImpl request_header{{":method", "GET"}, {":path",
+                                                                     "/path?x=xxxxxx&y=yyyyy&z=zzz&encoded=%26%"}};
+
+  Context formatter_context;
+  formatter_context.setRequestHeaders(request_header);
+
+  {
+    EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatParser::parse("%QUERY_PARAMS(A)%").IgnoreError(),
+                              EnvoyException,
+                              "Invalid PATH option: 'A', only 'ORIG'/'DECODED' are allowed");
+  }
+
+  {
+    QueryParametersFormatter formatter("", absl::optional<size_t>());
+    EXPECT_EQ("x=xxxxxx&y=yyyyy&z=zzz&encoded=%23%", formatter.format(formatter_context, stream_info));
+    EXPECT_THAT(formatter.formatValue(formatter_context, stream_info),
+                ProtoEq(ValueUtil::stringValue("x=xxxxxx&y=yyyyy&z=zzz&encoded=%23%")));
+  }
+
+  {
+    QueryParametersFormatter formatter("ORIG", absl::optional<size_t>());
+    EXPECT_EQ("x=xxxxxx&y=yyyyy&z=zzz&encoded=%23%", formatter.format(formatter_context, stream_info));
+    EXPECT_THAT(formatter.formatValue(formatter_context, stream_info),
+                ProtoEq(ValueUtil::stringValue("x=xxxxxx&y=yyyyy&z=zzz&encoded=%23%")));
+  }
+
+  {
+    QueryParametersFormatter formatter("DECODED",absl::optional<size_t>());
+    EXPECT_EQ("x=xxxxxx&y=yyyyy&z=zzz&encoded=#", formatter.format(formatter_context, stream_info));
+    EXPECT_THAT(formatter.formatValue(formatter_context, stream_info),
+                ProtoEq(ValueUtil::stringValue("x=xxxxxx&y=yyyyy&z=zzz&encoded=#")));
+  }
+
+  {
+    QueryParametersFormatter formatter("", absl::optional<size_t>(4));
+    EXPECT_EQ("x=xx", formatter.format(formatter_context, stream_info));
+    EXPECT_THAT(formatter.formatValue(formatter_context, stream_info),
+                ProtoEq(ValueUtil::stringValue("x=xx")));
+  }
+
+  {
+    QueryParametersFormatter formatter("ORIG", absl::optional<size_t>(4));
+    EXPECT_EQ("x=xx", formatter.format(formatter_context, stream_info));
+    EXPECT_THAT(formatter.formatValue(formatter_context, stream_info),
+                ProtoEq(ValueUtil::stringValue("x=xx")));
+  }
+
+  {
+    QueryParametersFormatter formatter("DECODED", absl::optional<size_t>(4));
+    EXPECT_EQ("x=x", formatter.format(formatter_context, stream_info));
+    EXPECT_THAT(formatter.formatValue(formatter_context, stream_info),
+                ProtoEq(ValueUtil::stringValue("x=xx")));
+  }
+}
+
 TEST(SubstitutionFormatterTest, headersByteSizeFormatter) {
   StreamInfo::MockStreamInfo stream_info;
   Http::TestRequestHeaderMapImpl request_header{{":method", "GET"}, {":path", "/"}};