[clang-format] Set Change.TokenLength to ColumnWidth (#90378)

Fixes #37705.
Fixes #47333.
Fixes #47624.
Fixes #58850.
Fixes #75929.
Fixes #87885.
Fixes #89916.

Author: owenca

clang/lib/Format/WhitespaceManager.cpp

@@ -131,6 +131,7 @@ void WhitespaceManager::calculateLineBreakInformation() {
   for (unsigned I = 1, e = Changes.size(); I != e; ++I) {
     auto &C = Changes[I];
     auto &P = Changes[I - 1];
+    auto &PrevTokLength = P.TokenLength;
     SourceLocation OriginalWhitespaceStart =
         C.OriginalWhitespaceRange.getBegin();
     SourceLocation PreviousOriginalWhitespaceEnd =
@@ -169,21 +170,23 @@ void WhitespaceManager::calculateLineBreakInformation() {
     // line of the token.
     auto NewlinePos = Text.find_first_of('\n');
     if (NewlinePos == StringRef::npos) {
-      P.TokenLength = OriginalWhitespaceStartOffset -
+      PrevTokLength = OriginalWhitespaceStartOffset -
                       PreviousOriginalWhitespaceEndOffset +
                       C.PreviousLinePostfix.size() + P.CurrentLinePrefix.size();
+      if (!P.IsInsideToken)
+        PrevTokLength = std::min(PrevTokLength, P.Tok->ColumnWidth);
     } else {
-      P.TokenLength = NewlinePos + P.CurrentLinePrefix.size();
+      PrevTokLength = NewlinePos + P.CurrentLinePrefix.size();
     }
 
     // If there are multiple changes in this token, sum up all the changes until
     // the end of the line.
     if (P.IsInsideToken && P.NewlinesBefore == 0)
-      LastOutsideTokenChange->TokenLength += P.TokenLength + P.Spaces;
+      LastOutsideTokenChange->TokenLength += PrevTokLength + P.Spaces;
     else
       LastOutsideTokenChange = &P;
 
-    C.PreviousEndOfTokenColumn = P.StartOfTokenColumn + P.TokenLength;
+    C.PreviousEndOfTokenColumn = P.StartOfTokenColumn + PrevTokLength;
 
     P.IsTrailingComment =
         (C.NewlinesBefore > 0 || C.Tok->is(tok::eof) ||

clang/unittests/Format/FormatTest.cpp

@@ -27363,6 +27363,45 @@ TEST_F(FormatTest, BreakAdjacentStringLiterals) {
   verifyFormat(Code, Style);
 }
 
+TEST_F(FormatTest, AlignUTFCommentsAndStringLiterals) {
+  verifyFormat(
+      "int rus;      // А теперь комментарии, например, на русском, 2-байта\n"
+      "int long_rus; // Верхний коммент еще не превысил границу в 80, однако\n"
+      "              // уже отодвинут. Перенос, при этом, отрабатывает верно");
+
+  auto Style = getLLVMStyle();
+  Style.ColumnLimit = 15;
+  verifyNoChange("#define test  \\\n"
+                 "  /* 测试 */  \\\n"
+                 "  \"aa\"        \\\n"
+                 "  \"bb\"",
+                 Style);
+
+  Style.ColumnLimit = 25;
+  verifyFormat("struct foo {\n"
+               "  int iiiiii; ///< iiiiii\n"
+               "  int b;      ///< ыыы\n"
+               "  int c;      ///< ыыыы\n"
+               "};",
+               Style);
+
+  Style.ColumnLimit = 35;
+  verifyFormat("#define SENSOR_DESC_1             \\\n"
+               "  \"{\"                             \\\n"
+               "  \"unit_of_measurement: \\\"°C\\\",\"  \\\n"
+               "  \"}\"",
+               Style);
+
+  Style.ColumnLimit = 80;
+  Style.AlignArrayOfStructures = FormatStyle::AIAS_Left;
+  verifyFormat("Languages languages = {\n"
+               "    Language{{'e', 'n'}, U\"Test English\" },\n"
+               "    Language{{'l', 'v'}, U\"Test Latviešu\"},\n"
+               "    Language{{'r', 'u'}, U\"Test Русский\" },\n"
+               "};",
+               Style);
+}
+
 } // namespace
 } // namespace test
 } // namespace format