Add support for punctuation in generated textmate grammars [#3337]

Signed-off-by: iTob <itobdev@gmail.com>

Author: iTob

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/textmate/TextMateGrammar.java

@@ -16,13 +16,16 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.eclipse.xtext.Grammar;
 import org.eclipse.xtext.GrammarUtil;
 import org.eclipse.xtext.TerminalRule;
 
 import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
 import com.google.gson.annotations.Expose;
 
 /**
@@ -125,8 +128,8 @@ protected TextMateGrammar init(Grammar grammar) {
 		}
 		TextMateGrammar result = new TextMateGrammar();
 		result.setScopeName(scopeName);
-		TextMateRule keywords = getKeywordControlRule(grammar, ignoreCase);
-		result.addRule(keywords);
+		result.addRule(getKeywordControlRule(grammar, ignoreCase));
+		result.addRule(getPunctuationRule(grammar, ignoreCase));
 
 		Set<String> seenTerminalRules = new HashSet<>();
 		for(TextMateRule pattern: patterns) {
@@ -164,7 +167,7 @@ protected AutoRule newAutoRule() {
 	protected String getLanguageName(Grammar grammar) {
 		return GrammarUtil.getSimpleName(grammar).toLowerCase(Locale.ROOT);
 	}
-	
+
 	protected TextMateRule getKeywordControlRule(Grammar grammar, boolean ignoreCase) {
 		StringBuilder matchBuilder = new StringBuilder();
 		if (ignoreCase) {
@@ -173,15 +176,54 @@ protected TextMateRule getKeywordControlRule(Grammar grammar, boolean ignoreCase
 		matchBuilder.append("\\b(");
 		List<String> allKeywords = GrammarUtil.getAllKeywords(grammar)
 				.stream()
-				.filter(s->s.matches("\\w+"))
+				.filter(s -> s.matches("\\w+"))
 				.sorted(Comparator.naturalOrder())
 				.collect(Collectors.toList());
-		matchBuilder.append(Joiner.on("|").join(allKeywords));
+		Joiner.on("|").appendTo(matchBuilder, allKeywords);
 		matchBuilder.append(")\\b");
 		MatchRule result = new MatchRule();
 		result.setName("keyword.control." + getLanguageName(grammar));
 		result.setMatch(matchBuilder.toString());
 		return result;
 	}
 
+	protected TextMateRule getPunctuationRule(Grammar grammar, boolean ignoreCase) {
+		StringBuilder matchBuilder = new StringBuilder();
+		if (ignoreCase) {
+			matchBuilder.append("(?i)");
+		}
+		matchBuilder.append("(");
+		List<String> allKeywords = GrammarUtil.getAllKeywords(grammar)
+				.stream()
+				.filter(s -> !s.matches("\\w+"))
+				.sorted(Comparator.naturalOrder())
+				.collect(Collectors.toList());
+		Joiner.on("|").appendTo(matchBuilder, Iterables.transform(allKeywords, this::escapeAndAddWordBoundaries));
+		matchBuilder.append(")");
+		MatchRule result = new MatchRule();
+		result.setName("punctuation." + getLanguageName(grammar));
+		result.setMatch(matchBuilder.toString());
+		return result;
+	}
+
+	private static final Pattern START_IS_LETTER = Pattern.compile("^\\w");
+	private static final Pattern END_IS_LETTER = Pattern.compile("\\w$");
+	protected String escapeAndAddWordBoundaries(String token) {
+		StringBuilder result = new StringBuilder();
+		if (START_IS_LETTER.matcher(token).find()) {
+			result.append("\\b");
+		}
+		result.append(escapeForRegex(token));
+		if (END_IS_LETTER.matcher(token).find()) {
+			result.append("\\b");
+		}
+		return result.toString();
+	}
+
+	private static final Pattern REGEX_CONTROL_CHARS = Pattern.compile("[\\\\^$.*+?()\\[\\]{}|]");
+	private static String escapeForRegex(String input) {
+		Matcher matcher = REGEX_CONTROL_CHARS.matcher(input);
+		return matcher.replaceAll(match -> Matcher.quoteReplacement("\\" + match.group()));
+	}
+
 }