Fix a false negative assertion failure in LoopProgressCondition for unterminated nested string interpolation

If we have two nested, unterminated string interpolation segments, the lexer generates two empty `stringLiteral` tokens (one after each interpolation segment). When consuming the first empty string segment, we did actually make progress in the lexer by popping one nested string interpolation off the state stack. However, `LoopProgressCondition` did not consider this progress because it only looked at the top state in the state stack.

To fix this, consider the state stack size in `LoopProgressCondition` as well.

Fixes #2533
rdar://124168557

Author: ahoppen

Sources/SwiftParser/Lexer/Cursor.swift

@@ -52,7 +52,7 @@ extension Lexer.Cursor {
   ///  - A string interpolation inside is entered
   ///  - A regex literal is being lexed
   ///  - A narrow case for 'try?' and 'try!' to ensure correct regex lexing
-  enum State {
+  enum State: Equatable {
     /// Normal top-level lexing mode
     case normal
 
@@ -203,6 +203,11 @@ extension Lexer.Cursor {
         }
       }
     }
+
+    /// See `Lexer.Cursor.hasProgressed(comparedTo:)`.
+    fileprivate func hasProgressed(comparedTo other: StateStack) -> Bool {
+      return currentState != other.currentState || stateStack?.count != other.stateStack?.count
+    }
   }
 
   /// An error that was discovered in a lexeme while lexing it.
@@ -257,6 +262,16 @@ extension Lexer {
       self.position = Position(input: input, previous: previous)
     }
 
+    /// Returns `true` if this cursor is sufficiently different to `other` in a way that indicates that the lexer has
+    /// made progress.
+    ///
+    /// This is the case if the lexer advanced its position in the source file or if it has performed a state
+    /// transition.
+    func hasProgressed(comparedTo other: Cursor) -> Bool {
+      return position.input.baseAddress != other.position.input.baseAddress
+        || stateStack.hasProgressed(comparedTo: other.stateStack)
+    }
+
     var input: UnsafeBufferPointer<UInt8> { position.input }
     var previous: UInt8 { position.previous }
 

Sources/SwiftParser/LoopProgressCondition.swift

@@ -29,14 +29,7 @@ struct LoopProgressCondition {
     guard let previousToken = self.currentToken else {
       return true
     }
-    // The loop has made progress if either
-    //  - the parser is now pointing at a different location in the source file
-    //  - the parser is still pointing at the same position in the source file
-    //     but now has a different token kind (and thus consumed a zero-length
-    //     token like an empty string interpolation
-    let hasMadeProgress =
-      previousToken.tokenText.baseAddress != currentToken.tokenText.baseAddress
-      || (previousToken.byteLength == 0 && previousToken.rawTokenKind != currentToken.rawTokenKind)
+    let hasMadeProgress = currentToken.cursor.hasProgressed(comparedTo: previousToken.cursor)
     assert(hasMadeProgress, "Loop should always make progress")
     return hasMadeProgress
   }

Tests/SwiftParserTest/LexerTests.swift

@@ -1561,4 +1561,26 @@ class LexerTests: ParserTestCase {
       ]
     )
   }
+
+  func testNestedUnterminatedStringInterpolations() {
+    assertLexemes(
+      #"""
+      "\("\(
+
+      """#,
+      lexemes: [
+        LexemeSpec(.stringQuote, text: #"""#),
+        LexemeSpec(.stringSegment, text: ""),
+        LexemeSpec(.backslash, text: #"\"#),
+        LexemeSpec(.leftParen, text: "("),
+        LexemeSpec(.stringQuote, text: #"""#),
+        LexemeSpec(.stringSegment, text: ""),
+        LexemeSpec(.backslash, text: #"\"#),
+        LexemeSpec(.leftParen, text: "("),
+        LexemeSpec(.stringSegment, text: ""),
+        LexemeSpec(.stringSegment, text: ""),
+        LexemeSpec(.endOfFile, leading: "\n", text: "", flags: [.isAtStartOfLine]),
+      ]
+    )
+  }
 }

Tests/SwiftParserTest/translated/UnclosedStringInterpolationTests.swift

@@ -247,4 +247,38 @@ final class UnclosedStringInterpolationTests: ParserTestCase {
         """#
     )
   }
+
+  func testNestedUnterminatedStringInterpolation() {
+    assertParse(
+      #"""
+      1️⃣"\2️⃣(3️⃣"\(4️⃣
+
+      """#,
+      diagnostics: [
+        DiagnosticSpec(locationMarker: "4️⃣", message: "expected value and ')' in string literal", fixIts: ["insert value and ')'"]),
+        DiagnosticSpec(
+          locationMarker: "4️⃣",
+          message: #"expected '"' to end string literal"#,
+          notes: [NoteSpec(locationMarker: "3️⃣", message: #"to match this opening '"'"#)],
+          fixIts: [#"insert '"'"#]
+        ),
+        DiagnosticSpec(
+          locationMarker: "4️⃣",
+          message: "expected ')' in string literal",
+          notes: [NoteSpec(locationMarker: "2️⃣", message: "to match this opening '('")],
+          fixIts: ["insert ')'"]
+        ),
+        DiagnosticSpec(
+          locationMarker: "4️⃣",
+          message: #"expected '"' to end string literal"#,
+          notes: [NoteSpec(locationMarker: "1️⃣", message: #"to match this opening '"'"#)],
+          fixIts: [#"insert '"'"#]
+        ),
+      ],
+      fixedSource: #"""
+        "\("\(<#expression#>)")"
+
+        """#
+    )
+  }
 }