Initial support for string templates

Fixes #981

PiperOrigin-RevId: 591982309

Author: cushon

core/src/main/java/com/google/googlejavaformat/java/JavaInput.java

@@ -387,7 +387,14 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
       final boolean isNumbered; // Is this tok numbered? (tokens and comments)
       String extraNewline = null; // Extra newline at end?
       List<String> strings = new ArrayList<>();
-      if (Character.isWhitespace(tokText0)) {
+      if (tokText.startsWith("'")
+          || tokText.startsWith("\"")
+          || JavacTokens.isStringFragment(t.kind())) {
+        // Perform this check first, STRINGFRAGMENT tokens can start with arbitrary characters.
+        isToken = true;
+        isNumbered = true;
+        strings.add(originalTokText);
+      } else if (Character.isWhitespace(tokText0)) {
         isToken = false;
         isNumbered = false;
         Iterator<String> it = Newlines.lineIterator(originalTokText);
@@ -404,10 +411,6 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
             strings.add(line);
           }
         }
-      } else if (tokText.startsWith("'") || tokText.startsWith("\"")) {
-        isToken = true;
-        isNumbered = true;
-        strings.add(originalTokText);
       } else if (tokText.startsWith("//") || tokText.startsWith("/*")) {
         // For compatibility with an earlier lexer, the newline after a // comment is its own tok.
         if (tokText.startsWith("//")

core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java

@@ -15,6 +15,7 @@
 package com.google.googlejavaformat.java;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Arrays.stream;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -27,6 +28,7 @@
 import com.sun.tools.javac.parser.Tokens.TokenKind;
 import com.sun.tools.javac.parser.UnicodeReader;
 import com.sun.tools.javac.util.Context;
+import java.util.Objects;
 import java.util.Set;
 
 /** A wrapper around javac's lexer. */
@@ -71,6 +73,16 @@ public String stringVal() {
     }
   }
 
+  private static final TokenKind STRINGFRAGMENT =
+      stream(TokenKind.values())
+          .filter(t -> t.name().contentEquals("STRINGFRAGMENT"))
+          .findFirst()
+          .orElse(null);
+
+  static boolean isStringFragment(TokenKind kind) {
+    return STRINGFRAGMENT != null && Objects.equals(kind, STRINGFRAGMENT);
+  }
+
   /** Lex the input and return a list of {@link RawTok}s. */
   public static ImmutableList<RawTok> getTokens(
       String source, Context context, Set<TokenKind> stopTokens) {
@@ -106,13 +118,39 @@ public static ImmutableList<RawTok> getTokens(
       if (last < t.pos) {
         tokens.add(new RawTok(null, null, last, t.pos));
       }
-      tokens.add(
-          new RawTok(
-              t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null,
-              t.kind,
-              t.pos,
-              t.endPos));
-      last = t.endPos;
+      int pos = t.pos;
+      int endPos = t.endPos;
+      if (isStringFragment(t.kind)) {
+        // A string template is tokenized as a series of STRINGFRAGMENT tokens containing the string
+        // literal values, followed by the tokens for the template arguments. For the formatter, we
+        // want the stream of tokens to appear in order by their start position, and also to have
+        // all the content from the original source text (including leading and trailing ", and the
+        // \ escapes from template arguments). This logic processes the token stream from javac to
+        // meet those requirements.
+        while (isStringFragment(t.kind)) {
+          endPos = t.endPos;
+          scanner.nextToken();
+          t = scanner.token();
+        }
+        // Read tokens for the string template arguments, until we read the end of the string
+        // template. The last token in a string template is always a trailing string fragment. Use
+        // lookahead to defer reading the token after the template until the next iteration of the
+        // outer loop.
+        while (scanner.token(/* lookahead= */ 1).endPos < endPos) {
+          scanner.nextToken();
+          t = scanner.token();
+        }
+        tokens.add(new RawTok(source.substring(pos, endPos), t.kind, pos, endPos));
+        last = endPos;
+      } else {
+        tokens.add(
+            new RawTok(
+                t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null,
+                t.kind,
+                t.pos,
+                t.endPos));
+        last = t.endPos;
+      }
     } while (scanner.token().kind != TokenKind.EOF);
     if (last < end) {
       tokens.add(new RawTok(null, null, last, end));

core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java

@@ -23,6 +23,7 @@
 import com.sun.source.tree.ExpressionTree;
 import com.sun.source.tree.PatternCaseLabelTree;
 import com.sun.source.tree.PatternTree;
+import com.sun.source.tree.StringTemplateTree;
 import javax.lang.model.element.Name;
 
 /**
@@ -60,6 +61,7 @@ public Void visitConstantCaseLabel(ConstantCaseLabelTree node, Void aVoid) {
 
   @Override
   public Void visitDeconstructionPattern(DeconstructionPatternTree node, Void unused) {
+    sync(node);
     scan(node.getDeconstructor(), null);
     builder.open(plusFour);
     token("(");
@@ -78,6 +80,16 @@ public Void visitDeconstructionPattern(DeconstructionPatternTree node, Void unus
     return null;
   }
 
+  @SuppressWarnings("preview")
+  @Override
+  public Void visitStringTemplate(StringTemplateTree node, Void aVoid) {
+    sync(node);
+    scan(node.getProcessor(), null);
+    token(".");
+    token(builder.peekToken().get());
+    return null;
+  }
+
   @Override
   protected void variableName(Name name) {
     if (name.isEmpty()) {

core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java

@@ -59,7 +59,8 @@ public class FormatterIntegrationTest {
               "SwitchDouble",
               "SwitchUnderscore",
               "I880",
-              "Unnamed")
+              "Unnamed",
+              "I981")
           .build();
 
   @Parameters(name = "{index}: {0}")

core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.input

@@ -0,0 +1,12 @@
+class Foo {
+    private static final int X = 42;
+    private static final String A = STR."\{X} = \{X}";
+    private static final String B = STR."";
+    private static final String C = STR."\{X}";
+    private static final String D = STR."\{X}\{X}";
+    private static final String E = STR."\{X}\{X}\{X}";
+    private static final String F = STR."     \{X}";
+    private static final String G = STR."\{X}    ";
+    private static final String H = STR."\{X} one long incredibly unbroken sentence moving from "+"topic to topic so that no-one had a chance to interrupt";
+    private static final String I = STR."\{X}  \uD83D\uDCA9  ";
+}

core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.output

@@ -0,0 +1,14 @@
+class Foo {
+  private static final int X = 42;
+  private static final String A = STR."\{X} = \{X}";
+  private static final String B = STR."";
+  private static final String C = STR."\{X}";
+  private static final String D = STR."\{X}\{X}";
+  private static final String E = STR."\{X}\{X}\{X}";
+  private static final String F = STR."     \{X}";
+  private static final String G = STR."\{X}    ";
+  private static final String H =
+      STR."\{X} one long incredibly unbroken sentence moving from "
+          + "topic to topic so that no-one had a chance to interrupt";
+  private static final String I = STR."\{X}  \uD83D\uDCA9  ";
+}