[1223] Change FormattedMessage pattern heuristic

We change the order in which `FormattedMessage` checks the format of the
provided pattern: we first check for the presence of `{}` placeholders
and only then for `java.util.Format` specifiers.

This eliminates the need for a potentially exponential regular
expression evalutation, which was reported by Spotbugs (#1849).

The Javadoc and documentation were improved to clarify the heuristic
used by `FormattedMessage`.

Closes #1223.

Remark: since `FormattedMessage` used the **same** regular expression as
`java.util.Format`, if a message uses `java.util.Format` specifiers, it
is still vulnerable to a ReDOS.

Author: ppkarwasz

log4j-api/src/main/java/org/apache/logging/log4j/message/FormattedMessage.java

@@ -21,7 +21,6 @@
 import java.util.Arrays;
 import java.util.Locale;
 import java.util.Objects;
-import java.util.regex.Pattern;
 
 /**
  * Handles messages that contain a format String. This converts each message into a {@link MessageFormatMessage},
@@ -30,9 +29,6 @@
  */
 public class FormattedMessage implements Message {
 
-    private static final String FORMAT_SPECIFIER = "%(\\d+\\$)?([-#+ 0,(\\<]*)?(\\d+)?(\\.\\d+)?([tT])?([a-zA-Z%])";
-    private static final Pattern MSG_PATTERN = Pattern.compile(FORMAT_SPECIFIER);
-
     private final String messagePattern;
     private final Object[] argArray;
     private String formattedMessage;
@@ -168,7 +164,27 @@ public String getFormattedMessage() {
         return formattedMessage;
     }
 
+    /**
+     * Gets the message implementation to which formatting is delegated.
+     *
+     * <ul>
+     *     <li>if {@code msgPattern} contains {@link MessageFormat} format specifiers a {@link MessageFormatMessage}
+     * is returned,</li>
+     *     <li>if {@code msgPattern} contains {@code {}} placeholders a {@link ParameterizedMessage} is returned,</li>
+     *     <li>if {@code msgPattern} contains {@link Format} specifiers a {@link StringFormattedMessage} is returned
+     *    .</li>
+     * </ul>
+     * <p>
+     *     Mixing specifiers from multiple types is not supported.
+     * </p>
+     *
+     * @param msgPattern The message pattern.
+     * @param args       The parameters.
+     * @param aThrowable The throwable
+     * @return The message that performs formatting.
+     */
     protected Message getMessage(final String msgPattern, final Object[] args, final Throwable aThrowable) {
+        // Check for valid `{ ArgumentIndex [, FormatType [, FormatStyle]] }` format specifiers
         try {
             final MessageFormat format = new MessageFormat(msgPattern);
             final Format[] formats = format.getFormats();
@@ -178,14 +194,13 @@ protected Message getMessage(final String msgPattern, final Object[] args, final
         } catch (final Exception ignored) {
             // Obviously, the message is not a proper pattern for MessageFormat.
         }
-        try {
-            if (MSG_PATTERN.matcher(msgPattern).find()) {
-                return new StringFormattedMessage(locale, msgPattern, args);
-            }
-        } catch (final Exception ignored) {
-            // Also not properly formatted.
+        // Check for non-escaped `{}` format specifiers
+        // This case also includes patterns without any `java.util.Formatter` specifiers
+        if (ParameterFormatter.analyzePattern(msgPattern, 1).placeholderCount > 0 || msgPattern.indexOf('%') == -1) {
+            return new ParameterizedMessage(msgPattern, args, aThrowable);
         }
-        return new ParameterizedMessage(msgPattern, args, aThrowable);
+        // Interpret as `java.util.Formatter` format
+        return new StringFormattedMessage(locale, msgPattern, args);
     }
 
     /**

src/changelog/.3.x.x/1223_change_formatted_message_heuristic.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<entry xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns="http://logging.apache.org/log4j/changelog"
+       xsi:schemaLocation="http://logging.apache.org/log4j/changelog https://logging.apache.org/log4j/changelog-0.1.2.xsd"
+       type="changed">
+  <issue id="1223" link="https://github.com/apache/logging-log4j2/issues/1223"/>
+  <description format="asciidoc">
+    Change the order of evaluation of `FormattedMessage` formatters.
+    Messages are evaluated using `java.util.Format` only if they don't comply to the `java.text.MessageFormat` or `ParameterizedMessage` format.
+  </description>
+</entry>