8356441: IllegalStateException in RichDiagnosticFormatter after JDK-8355065

Reviewed-by: liach, mcimadamore

Author: cushon

src/jdk.compiler/share/classes/com/sun/tools/javac/util/RichDiagnosticFormatter.java

@@ -32,6 +32,7 @@
 import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.function.Supplier;
 
 import com.sun.tools.javac.code.Printer;
 import com.sun.tools.javac.code.Symbol;
@@ -102,17 +103,17 @@ public Map<Type, JCDiagnostic> get(WhereClauseKind kind) {
     /* map for keeping track of a where clause associated to a given type */
     WhereClauses whereClauses;
 
-    private void enter() {
-        if (nameSimplifier != null || whereClauses != null) {
-            throw new IllegalStateException();
+    private String enter(Supplier<String> r) {
+        ClassNameSimplifier nameSimplifier = this.nameSimplifier;
+        WhereClauses whereClauses = this.whereClauses;
+        try {
+            this.nameSimplifier = new ClassNameSimplifier();
+            this.whereClauses = new WhereClauses();
+            return r.get();
+        } finally {
+            this.nameSimplifier = nameSimplifier;
+            this.whereClauses = whereClauses;
         }
-        nameSimplifier = new ClassNameSimplifier();
-        whereClauses = new WhereClauses();
-    }
-
-    private void exit() {
-        nameSimplifier = null;
-        whereClauses = null;
     }
 
     /** Get the DiagnosticFormatter instance for this context. */
@@ -136,8 +137,7 @@ protected RichDiagnosticFormatter(Context context) {
 
     @Override
     public String format(JCDiagnostic diag, Locale l) {
-        enter();
-        try {
+        return enter(() -> {
             StringBuilder sb = new StringBuilder();
             preprocessDiagnostic(diag);
             sb.append(formatter.format(diag, l));
@@ -153,20 +153,15 @@ public String format(JCDiagnostic diag, Locale l) {
                 }
             }
             return sb.toString();
-        } finally {
-            exit();
-        }
+        });
     }
 
     @Override
     public String formatMessage(JCDiagnostic diag, Locale l) {
-        enter();
-        try {
+        return enter(() -> {
             preprocessDiagnostic(diag);
             return super.formatMessage(diag, l);
-        } finally {
-            exit();
-        }
+        });
     }
 
     /**

test/langtools/tools/javac/annotations/typeAnnotations/RichFormatterWithTypeAnnotationsReentrantTest.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2025, Google LLC. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8356441
+ * @summary Recursive formatting in RichDiagnosticFormatter
+ * @library /tools/lib
+ * @modules jdk.compiler/com.sun.tools.javac.main jdk.compiler/com.sun.tools.javac.api
+ * @build toolbox.ToolBox toolbox.JavacTask
+ * @run main RichFormatterWithTypeAnnotationsReentrantTest
+ */
+import toolbox.JavacTask;
+import toolbox.Task;
+import toolbox.TestRunner;
+import toolbox.ToolBox;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+
+public class RichFormatterWithTypeAnnotationsReentrantTest extends TestRunner {
+    ToolBox tb;
+
+    public RichFormatterWithTypeAnnotationsReentrantTest() {
+        super(System.err);
+        tb = new ToolBox();
+    }
+
+    public static void main(String[] args) throws Exception {
+        new RichFormatterWithTypeAnnotationsReentrantTest()
+                .runTests(m -> new Object[] {Paths.get(m.getName())});
+    }
+
+    @Test
+    public void test(Path base) throws Exception {
+        Path libClasses = base.resolve("libclasses");
+        Files.createDirectories(libClasses);
+        new JavacTask(tb)
+                .outdir(libClasses)
+                .sources(
+                        """
+                        package lib;
+                        enum Bar {
+                          BAZ
+                        }
+                        """,
+                        """
+                        package lib;
+                        import java.lang.annotation.ElementType;
+                        import java.lang.annotation.Retention;
+                        import java.lang.annotation.RetentionPolicy;
+                        import java.lang.annotation.Target;
+
+                        @Retention(RetentionPolicy.RUNTIME)
+                        @interface Foo {
+                          Bar value();
+                        }
+                        """,
+                        """
+                        package lib;
+                        import java.lang.annotation.ElementType;
+                        import java.lang.annotation.Retention;
+                        import java.lang.annotation.RetentionPolicy;
+                        import java.lang.annotation.Target;
+
+                        @Retention(RetentionPolicy.RUNTIME)
+                        @Target({ElementType.TYPE_USE, ElementType.PARAMETER})
+                        @Foo(Bar.BAZ)
+                        @interface A {}
+                        """,
+                        """
+                        package lib;
+                        public interface M {
+                          String f(@A String k, @A String v);
+                        }
+                        """)
+                .run()
+                .writeAll();
+        Files.delete(libClasses.resolve("lib").resolve("Bar.class"));
+        String code =
+                """
+                import lib.M;
+                class T implements M {
+                }
+                """;
+        // verify that the compilation fails wtih an error, and does not crash
+        new JavacTask(tb)
+                .classpath(libClasses)
+                .sources(code)
+                .run(Task.Expect.FAIL);
+    }
+}