fix: invalid lookbehind when parsing julia textmate language

Author: sebthom

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java

@@ -117,6 +117,12 @@ private String rewritePatternIfRequired(final String pattern) {
 		if (pattern.startsWith(lookbehind3)) {
 			return "\\s*\\." + pattern.substring(lookbehind3.length());
 		}
+
+		// e.g. used in julia.tmLanguage.json
+		final var lookbehind4 = "(?<=\\S\\s+)";
+		if (pattern.startsWith(lookbehind4)) {
+			return "\\S\\s+" + pattern.substring(lookbehind4.length());
+		}
 		return pattern;
 	}
 

org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/parser/TMParserTest.java

@@ -36,6 +36,8 @@
 import org.eclipse.tm4e.core.internal.grammar.raw.RawGrammarReader;
 import org.eclipse.tm4e.core.internal.oniguruma.OnigRegExp;
 import org.eclipse.tm4e.core.internal.utils.ResourceUtils;
+import org.eclipse.tm4e.core.registry.IGrammarSource;
+import org.eclipse.tm4e.core.registry.Registry;
 import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
@@ -235,18 +237,28 @@ void testLanguagePackGrammars() throws IOException {
 		Files.walkFileTree(Paths.get("../org.eclipse.tm4e.language_pack"), new SimpleFileVisitor<Path>() {
 			@Override
 			public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
-				if (file.getFileName().toString().endsWith("tmLanguage.json")) {
-					try (var input = Files.newBufferedReader(file)) {
+				final var fileName = file.getFileName().toString().toLowerCase();
+				if (fileName.endsWith("tmlanguage.yaml") || fileName.endsWith("tmlanguage.json") || fileName.endsWith("plist")
+						|| fileName.endsWith("tmlanguage")) {
+					try {
 						System.out.println("Parsing [" + file + "]...");
-						final var grammar = TMParserJSON.INSTANCE.parse(input, RawGrammarReader.OBJECT_FACTORY);
+						final var rawGrammar = RawGrammarReader.readGrammar(IGrammarSource.fromFile(file));
 						count.incrementAndGet();
-						assertFalse(grammar.getScopeName().isBlank());
-						assertNotNull(grammar.getFileTypes());
-						assertNotNull(grammar.getRepository());
+						assertFalse(rawGrammar.getScopeName().isBlank());
+						assertNotNull(rawGrammar.getFileTypes());
+						assertNotNull(rawGrammar.getRepository());
 
-						final var patterns = castNonNull(grammar.getPatterns());
+						final var patterns = castNonNull(rawGrammar.getPatterns());
 						assertFalse(patterns.isEmpty());
 						assertParseablePatterns(patterns);
+
+						final var reg = new Registry();
+						final var grammar = reg.addGrammar(IGrammarSource.fromFile(file));
+						assertEquals(grammar.getName(), rawGrammar.getName());
+						assertEquals(grammar.getScopeName(), rawGrammar.getScopeName());
+						assertEquals(grammar.getFileTypes(), rawGrammar.getFileTypes());
+						grammar.tokenizeLine("");
+
 					} catch (final Exception ex) {
 						throw new RuntimeException(ex);
 					}