Merge whitespace into a single token, errors now report with 1 length

Author: jaredlll08

.run/tests_gradle.run.xml .run/ZenCode Tests - Gradle.run.xml.run/tests_gradle.run.xml renamed to .run/ZenCode Tests - Gradle.run.xml

@@ -10,7 +10,7 @@
       </option>
       <option name="taskNames">
         <list>
-          <option value=":test" />
+			<option value=":ScriptingExample:test" />
         </list>
       </option>
       <option name="vmOptions" />

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenType.java

@@ -4,10 +4,7 @@ public enum ZSTokenType implements TokenType {
 	T_COMMENT_SCRIPT("#[^\n]*[\n\\e]", true),
 	T_COMMENT_SINGLELINE("//[^\n]*", true),
 	T_COMMENT_MULTILINE("/\\*([^\\*]|(\\*+([^\\*/])))*\\*+/", true, true),
-	T_WHITESPACE_SPACE(true, " ", " "),
-	T_WHITESPACE_TAB(true, "\t", "\t"),
-	T_WHITESPACE_NEWLINE(true, "\n", "\n", true),
-	T_WHITESPACE_CARRIAGE_RETURN(true, "\r", "\r"),
+	T_WHITESPACE("[ \t\r\n]+", true, true),
 	T_IDENTIFIER("@?[a-zA-Z_][a-zA-Z_0-9]*"),
 	T_LOCAL_IDENTIFIER("$[a-zA-Z_][a-zA-Z_0-9]*"),
 	T_FLOAT("\\-?(0|[1-9][0-9]*)\\.[0-9]+([eE][\\+\\-]?[0-9]+)?[a-zA-Z_]*"),

Parser/src/main/java/org/openzen/zenscript/tree/Tree.java

@@ -1,14 +1,12 @@
 package org.openzen.zenscript.tree;
 
 import org.openzen.zencode.shared.SourceFile;
-import org.openzen.zenscript.lexer.ParseException;
 import org.openzen.zenscript.lexer.ZSToken;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.tree.lexer.DFALexer;
 import org.openzen.zenscript.tree.lexer.PositionedToken;
 import org.openzen.zenscript.tree.lexer.ZSPosTokenFactory;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -27,13 +25,13 @@ public Tree(TreeKind kind) {
 		this.children = new ArrayList<>();
 	}
 
-	public static Tree parse(SourceFile sourceFile) throws IOException, ParseException {
+	public static Tree parse(SourceFile sourceFile) {
 		DFALexer dfaLexer = new DFALexer(new ZSPosTokenFactory());
 		List<PositionedToken<ZSTokenType, ZSToken>> tokens = dfaLexer.tokenize(sourceFile);
 		return parse(tokens);
 	}
 
-	public static Tree parse(List<PositionedToken<ZSTokenType, ZSToken>> tokens) throws IOException, ParseException {
+	public static Tree parse(List<PositionedToken<ZSTokenType, ZSToken>> tokens) {
 		TreeParser p = new TreeParser(tokens);
 		p.file();
 		return p.buildTree();

Parser/src/main/java/org/openzen/zenscript/tree/TreeKind.java

@@ -164,9 +164,7 @@ public enum TreeKind {
 	T_COMMENT_SCRIPT,
 	T_COMMENT_SINGLELINE,
 	T_COMMENT_MULTILINE,
-	T_WHITESPACE_SPACE,
-	T_WHITESPACE_TAB,
-	T_WHITESPACE_NEWLINE,
+	T_WHITESPACE,
 	T_IDENTIFIER,
 	T_LOCAL_IDENTIFIER,
 	T_FLOAT,
@@ -310,9 +308,7 @@ public boolean isToken() {
 			case T_COMMENT_SCRIPT:
 			case T_COMMENT_SINGLELINE:
 			case T_COMMENT_MULTILINE:
-			case T_WHITESPACE_SPACE:
-			case T_WHITESPACE_TAB:
-			case T_WHITESPACE_NEWLINE:
+			case T_WHITESPACE:
 			case T_IDENTIFIER:
 			case T_LOCAL_IDENTIFIER:
 			case T_FLOAT:
@@ -464,13 +460,8 @@ public static TreeKind fromTokenType(ZSTokenType type) {
 				return TreeKind.T_COMMENT_SINGLELINE;
 			case T_COMMENT_MULTILINE:
 				return TreeKind.T_COMMENT_MULTILINE;
-			case T_WHITESPACE_SPACE:
-				return TreeKind.T_WHITESPACE_SPACE;
-			case T_WHITESPACE_TAB:
-				return TreeKind.T_WHITESPACE_TAB;
-			case T_WHITESPACE_NEWLINE:
-			case T_WHITESPACE_CARRIAGE_RETURN:
-				return TreeKind.T_WHITESPACE_NEWLINE;
+			case T_WHITESPACE:
+				return TreeKind.T_WHITESPACE;
 			case T_IDENTIFIER:
 				return TreeKind.T_IDENTIFIER;
 			case T_LOCAL_IDENTIFIER:

Parser/src/main/java/org/openzen/zenscript/tree/TreeParser.java

@@ -158,7 +158,11 @@ public void advance() {
 	}
 
 	public void error(String message) {
-		events.add(Event.error(message, this.tokens.get(pos - 1).position()));
+		int reportingPos = pos -1;
+//		while(this.tokens.get(reportingPos).getType().isWhitespace() && reportingPos > 0) {
+//			reportingPos--;
+//		}
+		events.add(Event.error(message, this.tokens.get(reportingPos).position().withLength(1)));
 	}
 
 	public void advanceWithError(String error) {

Parser/src/test/java/org/openzen/zenscript/test/framework/ParsingTestCase.java

@@ -21,24 +21,20 @@ public ParsingTestCase(Path zcFile, Path zastFile) {
 
 	public void execute() throws IOException {
 		// Read file content
-		try {
-			if (Files.notExists(zcFile)) {
-				Assertions.fail("File " + zcFile + " does not exist");
-			}
-			if (Files.notExists(zastFile)) {
-				Assertions.fail("File " + zastFile + " does not exist");
-			}
-			String zcContent = new String(Files.readAllBytes(zcFile), StandardCharsets.UTF_8).replace("\r\n", "\n");
-			String zastContent = new String(Files.readAllBytes(zastFile), StandardCharsets.UTF_8).replace("\r\n", "\n");
-			Tree tree = Tree.parse(new LiteralSourceFile(zcFile.getFileName().toString(), zcContent));
-
-			if (!zastContent.equals(tree.toString())) {
-				Assertions.fail(
-						"Error in \nfile://" + zcFile.toUri().getPath() + "\nfile://" + zastFile.toUri().getPath() + "\n Expected tree:\n'" + zastContent + "'\nbut got:\n'" + tree + "'\n"
-				);
-			}
-		} catch (ParseException e) {
-			Assertions.fail(e);
+		if (Files.notExists(zcFile)) {
+			Assertions.fail("File " + zcFile + " does not exist");
+		}
+		if (Files.notExists(zastFile)) {
+			Assertions.fail("File " + zastFile + " does not exist");
+		}
+		String zcContent = new String(Files.readAllBytes(zcFile), StandardCharsets.UTF_8).replace("\r\n", "\n");
+		String zastContent = new String(Files.readAllBytes(zastFile), StandardCharsets.UTF_8).replace("\r\n", "\n");
+		Tree tree = Tree.parse(new LiteralSourceFile(zcFile.getFileName().toString(), zcContent));
+
+		if (!zastContent.equals(tree.toString())) {
+			Assertions.fail(
+					"Error in \nfile://" + zcFile.toUri().getPath() + "\nfile://" + zastFile.toUri().getPath() + "\n Expected tree:\n'" + zastContent + "'\nbut got:\n'" + tree + "'\n"
+			);
 		}
 
 	}

Parser/src/test/zencode/tree/error/definition/alias.zast

@@ -1,20 +1,19 @@
 FILE
   DEF_ALIAS
     K_ALIAS "alias"
-    T_WHITESPACE_NEWLINE "\n"
-    T_WHITESPACE_NEWLINE "\n"
+    T_WHITESPACE "\n\n"
   DEF_ALIAS
     K_ALIAS "alias"
-    T_WHITESPACE_SPACE " "
+    T_WHITESPACE " "
     NAME
       T_IDENTIFIER "foo"
-    T_WHITESPACE_SPACE " "
+    T_WHITESPACE " "
     TYPE_DECLARATION
       K_AS "as"
-      T_WHITESPACE_SPACE " "
+      T_WHITESPACE " "
       TYPE
         NAME
           T_IDENTIFIER "Bar"
     T_SEMICOLON ";"
-(2:0->3:0)expected a name
-(2:0->3:0)expected as
+(1:5->1:6)expected a name
+(1:5->1:6)expected as

Parser/src/test/zencode/tree/error/definition/class.zast

@@ -1,16 +1,15 @@
 FILE
   DEF_CLASS
     K_CLASS "class"
-    T_WHITESPACE_NEWLINE "\n"
-    T_WHITESPACE_NEWLINE "\n"
+    T_WHITESPACE "\n\n"
   DEF_CLASS
     K_CLASS "class"
-    T_WHITESPACE_SPACE " "
+    T_WHITESPACE " "
     NAME
       T_IDENTIFIER "Foo"
-    T_WHITESPACE_SPACE " "
+    T_WHITESPACE " "
     MEMBERS
       T_AOPEN "{"
       T_ACLOSE "}"
-(2:0->3:0)expected a name
-(2:0->3:0)expected '{'
+(1:5->1:6)expected a name
+(1:5->1:6)expected '{'

Parser/src/test/zencode/tree/error/definition/enum.zast

@@ -1,15 +1,14 @@
 FILE
   DEF_ENUM
     K_ENUM "enum"
-    T_WHITESPACE_NEWLINE "\n"
-    T_WHITESPACE_NEWLINE "\n"
+    T_WHITESPACE "\n\n"
   DEF_ENUM
     K_ENUM "enum"
-    T_WHITESPACE_SPACE " "
+    T_WHITESPACE " "
     NAME
       T_IDENTIFIER "Foo"
-    T_WHITESPACE_SPACE " "
+    T_WHITESPACE " "
     T_AOPEN "{"
     T_ACLOSE "}"
-(2:0->3:0)expected a name
-(2:0->3:0)expected '{'
+(1:4->1:5)expected a name
+(1:4->1:5)expected '{'

Parser/src/test/zencode/tree/error/definition/expand.zast

@@ -1,16 +1,15 @@
 FILE
   DEF_EXPANSION
     K_EXPAND "expand"
-    T_WHITESPACE_NEWLINE "\n"
-    T_WHITESPACE_NEWLINE "\n"
+    T_WHITESPACE "\n\n"
   DEF_EXPANSION
     K_EXPAND "expand"
-    T_WHITESPACE_SPACE " "
+    T_WHITESPACE " "
     TYPE
       NAME
         T_IDENTIFIER "Foo"
-      T_WHITESPACE_SPACE " "
+      T_WHITESPACE " "
     MEMBERS
       T_AOPEN "{"
       T_ACLOSE "}"
-(2:0->3:0)expected type
+(1:6->1:7)expected type