Embedded code accounts for concatenated strings and escape chars

Author: BoykoAlex

headless-services/commons/commons-language-server/src/main/java/org/springframework/ide/vscode/commons/languageserver/semantic/tokens/SemanticTokenData.java

@@ -13,27 +13,33 @@
 import java.util.Arrays;
 import java.util.Objects;
 
+import org.springframework.ide.vscode.commons.util.text.IRegion;
+import org.springframework.ide.vscode.commons.util.text.Region;
+
 public record SemanticTokenData(
-		int start,
-		int end,
+		IRegion range,
 		String type,
 		String[] modifiers
 	) implements Comparable<SemanticTokenData> {
+	
+	public SemanticTokenData(int start, int end, String type, String[] modifiers) {
+		this(new Region(start, end -start), type, modifiers);
+	}
 
 	@Override
 	public int compareTo(SemanticTokenData o) {
-		if (start == o.start) {
-			return end - o.end;
+		if (range.getOffset() == o.range().getOffset()) {
+			return range.getLength() - o.range().getLength();
 		}
-		return start - o.start;
+		return range.getOffset() - o.range().getOffset();
 	}
 
 	@Override
 	public int hashCode() {
 		final int prime = 31;
 		int result = 1;
 		result = prime * result + Arrays.hashCode(modifiers);
-		result = prime * result + Objects.hash(end, start, type);
+		result = prime * result + Objects.hash(range, type);
 		return result;
 	}
 
@@ -46,8 +52,16 @@ public boolean equals(Object obj) {
 		if (getClass() != obj.getClass())
 			return false;
 		SemanticTokenData other = (SemanticTokenData) obj;
-		return end == other.end && Arrays.equals(modifiers, other.modifiers) && start == other.start
+		return range.getLength() == other.range.getLength() && Arrays.equals(modifiers, other.modifiers) && range.getOffset() == other.range().getOffset()
 				&& Objects.equals(type, other.type);
 	}
 
+	public int getStart() {
+		return range.getStart();
+	}
+	
+	public int getEnd() {
+		return range.getEnd();
+	}
+	
 }

headless-services/commons/commons-language-server/src/main/java/org/springframework/ide/vscode/commons/languageserver/semantic/tokens/SemanticTokensDataProvider.java

@@ -17,6 +17,6 @@ public interface SemanticTokensDataProvider {
 
 	List<String> getTokenTypes();
 	default List<String> getTypeModifiers() { return Collections.emptyList(); }
-	List<SemanticTokenData> computeTokens(String text, int initialOffset);
+	List<SemanticTokenData> computeTokens(String text);
 
 }

headless-services/commons/commons-language-server/src/main/java/org/springframework/ide/vscode/commons/languageserver/semantic/tokens/SemanticTokensUtils.java

@@ -52,11 +52,11 @@ public static List<Integer> mapTokensDataToLsp(List<SemanticTokenData> tokensDat
 		int previousLine = 0;
 		int previousColumn = 0;
 		for (SemanticTokenData tokenData : tokensData) {
-			int currentLine = getLineNumber.apply(tokenData.start());
-			int currentColumn = getColumnNumber.apply(tokenData.start());
+			int currentLine = getLineNumber.apply(tokenData.getStart());
+			int currentColumn = getColumnNumber.apply(tokenData.getStart());
 			data.add(currentLine - previousLine);
 			data.add(currentLine == previousLine ? currentColumn - previousColumn : currentColumn);
-			data.add(tokenData.end() -  tokenData.start());
+			data.add(tokenData.getEnd() -  tokenData.getStart());
 			data.add(SemanticTokensUtils.getSemanticTokenTypeIndex(legend, tokenData.type()));
 			data.add(SemanticTokensUtils.getSemanticTokenModifiersFlags(legend, tokenData.modifiers()));
 			previousLine = currentLine;
@@ -79,11 +79,11 @@ public static List<Integer> mapTokensDataToLsp(TextDocument doc, SemanticTokensL
 		int previousColumn = 0;
 		for (SemanticTokenData tokenData : tokensData) {
 			try {
-				int currentLine = doc.getLineOfOffset(tokenData.start());
-				int currentColumn = tokenData.start() - doc.getLineOffset(currentLine);
+				int currentLine = doc.getLineOfOffset(tokenData.getStart());
+				int currentColumn = tokenData.getStart() - doc.getLineOffset(currentLine);
 				data.add(currentLine - previousLine);
 				data.add(currentLine == previousLine ? currentColumn - previousColumn : currentColumn);
-				data.add(tokenData.end() -  tokenData.start());
+				data.add(tokenData.getEnd() -  tokenData.getStart());
 				data.add(SemanticTokensUtils.getSemanticTokenTypeIndex(legend, tokenData.type()));
 				data.add(SemanticTokensUtils.getSemanticTokenModifiersFlags(legend, tokenData.modifiers()));
 				previousLine = currentLine;

headless-services/commons/commons-util/src/main/java/org/springframework/ide/vscode/commons/util/text/IRegion.java

@@ -18,5 +18,11 @@ public interface IRegion {
 
 	int getOffset();
 	int getLength();
+	default int getEnd() {
+		return getOffset() + getLength();
+	}
+	default int getStart() {
+		return getOffset();
+	}
 
 }

headless-services/commons/jpql/grammars/PostgreSqlLexer.g4

@@ -1489,7 +1489,7 @@ InvalidUnterminatedUnicodeQuotedIdentifier: 'U' '&' InvalidUnterminatedQuotedIde
 
 StringConstant: UnterminatedStringConstant '\'';
 
-UnterminatedStringConstant: '\'' ('\'\'' | ~ '\'' | '\\\\\'' | '\\\'')*;
+UnterminatedStringConstant: '\'' ('\'\'' | ~ '\'' | '\\\'')*;
 // String Constants with C-style Escapes (4.1.2.2)
 
 BeginEscapeStringConstant: 'E' '\'' -> more, pushMode (EscapeStringConstantMode);
@@ -1625,7 +1625,6 @@ fragment EscapeStringText options {
 }:
     (
         '\\\''
-        | '\'\'\''
         | '\\' (
             // two-digit hex escapes are still valid when treated as single-digit escapes
             'x' [0-9a-fA-F]