[java20] New grammar, Java version 20 (#3659)

* Initial version of Java20.

* Add EOF-terminated rule.

* Fix list of targets that should work. Remove tests that fail.

* Fix lexer grammar for refactoring via transformGrammar.py.

Author: kaby76

java/java20/CSharp/JavaLexerBase.cs

@@ -0,0 +1,100 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2014 Terence Parr
+ *  Copyright (c) 2014 Sam Harwell
+ *  Copyright (c) 2017 Chan Chung Kwong
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using Antlr4.Runtime;
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+public abstract class JavaLexerBase : Lexer
+{
+    private readonly ICharStream _input;
+
+    protected JavaLexerBase(ICharStream input, TextWriter output, TextWriter errorOutput)
+        : base(input, output, errorOutput) {
+            _input = input;
+    }
+
+    private class Character
+    {
+        public static bool isJavaIdentifierPart(int c)
+        {
+            if (Char.IsLetter((char)c))
+                return true;
+            else if (c == (int)'$')
+                return true;
+            else if (c == (int)'_')
+                return true;
+            else if (Char.IsDigit((char)c))
+                return true;
+            else if (Char.IsNumber((char)c))
+                return true;
+            return false;
+        }
+
+        public static bool isJavaIdentifierStart(int c)
+        {
+            if (Char.IsLetter((char)c))
+                return true;
+            else if (c == (int)'$')
+                return true;
+            else if (c == (int)'_')
+                return true;
+            return false;
+        }
+
+        public static int toCodePoint(int high, int low)
+        {
+            return Char.ConvertToUtf32((char)high, (char)low);
+        }
+    }
+
+    public bool Check1()
+    {
+        return Character.isJavaIdentifierStart(_input.LA(-1));
+    }
+
+    public bool Check2()
+    {
+        return Character.isJavaIdentifierStart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)));
+    }
+
+    public bool Check3()
+    {
+        return Character.isJavaIdentifierPart(_input.LA(-1));
+    }
+
+    public bool Check4()
+    {
+        return Character.isJavaIdentifierPart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)));
+    }
+}

java/java20/Cpp/JavaLexerBase.cpp

@@ -0,0 +1,91 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2014 Terence Parr
+ *  Copyright (c) 2014 Sam Harwell
+ *  Copyright (c) 2017 Chan Chung Kwong
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "antlr4-runtime.h"
+#include "JavaLexerBase.h"
+
+JavaLexerBase::JavaLexerBase(antlr4::CharStream * input) : antlr4::Lexer(input)
+{
+    _input = input;
+}
+
+bool JavaLexerBase::Character::isJavaIdentifierPart(int c)
+{
+    if ((int)'a' <= c && c <= (int)'z' || (int)'A' <= c && c <= (int)'Z')
+        return true;
+    else if (c == (int)'$')
+        return true;
+    else if (c == (int)'_')
+        return true;
+    else if ((int)'0' <= c && c <= (int)'9')
+        return true;
+    return false;
+}
+
+bool JavaLexerBase::Character::isJavaIdentifierStart(int c)
+{
+    if ((int)'a' <= c && c <= (int)'z' || (int)'A' <= c && c <= (int)'Z')
+        return true;
+    else if (c == (int)'$')
+        return true;
+    else if (c == (int)'_')
+        return true;
+    return false;
+}
+
+char32_t surrogate_to_utf32(char16_t high, char16_t low) { 
+	return (high << 10) + low - 0x35fdc00; 
+}
+
+int JavaLexerBase::Character::toCodePoint(int high, int low)
+{
+    return surrogate_to_utf32(high, low);
+}
+
+bool JavaLexerBase::Check1()
+{
+    return JavaLexerBase::Character::isJavaIdentifierStart(_input->LA(-1));
+}
+
+bool JavaLexerBase::Check2()
+{
+    return JavaLexerBase::Character::isJavaIdentifierStart(JavaLexerBase::Character::toCodePoint((char)_input->LA(-2), (char)_input->LA(-1)));
+}
+
+bool JavaLexerBase::Check3()
+{
+    return JavaLexerBase::Character::isJavaIdentifierPart(_input->LA(-1));
+}
+
+bool JavaLexerBase::Check4()
+{
+    return JavaLexerBase::Character::isJavaIdentifierPart(JavaLexerBase::Character::toCodePoint((char)_input->LA(-2), (char)_input->LA(-1)));
+}

java/java20/Cpp/JavaLexerBase.h

@@ -0,0 +1,24 @@
+#pragma once
+#include "antlr4-runtime.h"
+
+class JavaLexerBase : public antlr4::Lexer
+{
+	private:
+		antlr4::CharStream * _input;
+
+	public:
+		JavaLexerBase(antlr4::CharStream * input);
+
+	class Character
+	{
+		public:
+		    static bool isJavaIdentifierPart(int c);
+		    static bool isJavaIdentifierStart(int c);
+		    static int toCodePoint(int high, int low);
+	};
+		
+	bool Check1();
+	bool Check2();
+	bool Check3();
+	bool Check4();
+};

java/java20/Cpp/transformGrammar.py

@@ -0,0 +1,34 @@
+import sys, os, re, shutil
+from glob import glob
+from pathlib import Path
+
+def main(argv):
+    for file in glob("./*.g4"):
+        fix(file)
+
+def fix(file_path):
+    print("Altering " + file_path)
+    if not os.path.exists(file_path):
+        print(f"Could not find file: {file_path}")
+        sys.exit(1)
+    parts = os.path.split(file_path)
+    file_name = parts[-1]
+    shutil.move(file_path, file_path + ".bak")
+    input_file = open(file_path + ".bak",'r')
+    output_file = open(file_path, 'w')
+    for x in input_file:
+        if '// Insert here @header for C++ lexer.' in x:
+            x = x.replace('// Insert here @header for C++ lexer.', '@header {#include "JavaLexerBase.h"}')
+        if '// Insert here @header for C++ parser.' in x:
+            x = x.replace('// Insert here @header for C++ parser.', '@header {#include "JavaParserBase.h"}')
+        if 'this.' in x:
+            x = x.replace('this.', 'this->')
+        output_file.write(x)
+        output_file.flush()
+
+    print("Writing ...")
+    input_file.close()
+    output_file.close()
+
+if __name__ == '__main__':
+    main(sys.argv)

java/java20/Dart/JavaLexerBase.dart

@@ -0,0 +1,102 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2014 Terence Parr
+ *  Copyright (c) 2014 Sam Harwell
+ *  Copyright (c) 2017 Chan Chung Kwong
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+import 'package:antlr4/antlr4.dart';
+import 'dart:io';
+import 'dart:convert';
+
+abstract class JavaLexerBase extends Lexer
+{
+    JavaLexerBase(CharStream input) : super(input)
+    {
+    }
+
+    bool Check1()
+    {
+        return Character.isJavaIdentifierStart(inputStream.LA(-1)!);
+    }
+
+    bool Check2()
+    {
+        return Character.isJavaIdentifierStart(Character.toCodePoint(inputStream.LA(-2)!, inputStream.LA(-1)!));
+    }
+
+    bool Check3()
+    {
+        return Character.isJavaIdentifierPart(inputStream.LA(-1)!);
+    }
+
+    bool Check4()
+    {
+        return Character.isJavaIdentifierPart(Character.toCodePoint(inputStream.LA(-2)!, inputStream.LA(-1)!));
+    }
+}
+
+class Character
+{
+    static bool isJavaIdentifierPart(int c)
+    {
+        if (('a'.codeUnitAt(0) <= c && c <= 'z'.codeUnitAt(0)) || ('A'.codeUnitAt(0) <= c && c <= 'Z'.codeUnitAt(0))) //Char.IsLetter((char)c))
+			return true;
+        else if (c == '\$'.codeUnitAt(0))
+			return true;
+        else if (c == '_'.codeUnitAt(0))
+			return true;
+        else if ('0'.codeUnitAt(0) <= c && c <= '9'.codeUnitAt(0)) //Char.IsDigit(c))
+			return true;
+        else if ('0'.codeUnitAt(0) <= c && c <= '9'.codeUnitAt(0)) //Char.IsNumber(c))
+			return true;
+        return false;
+    }
+
+    static bool isJavaIdentifierStart(int c)
+    {
+        if (('a'.codeUnitAt(0) <= c && c <= 'z'.codeUnitAt(0)) || ('A'.codeUnitAt(0) <= c && c <= 'Z'.codeUnitAt(0))) //Char.IsLetter((char)c))
+			return true;
+        else if (c == '\$'.codeUnitAt(0))
+			return true;
+        else if (c == '_'.codeUnitAt(0))
+			return true;
+        return false;
+    }
+
+    static int toCodePoint(int high, int low)
+    {
+		List<int> encoded = List.filled(2, 0, growable: false);
+		encoded.add(high);
+		encoded.add(low);
+		return base64.encode(encoded).codeUnitAt(0);
+
+//		return Utf16CodeUnitDecoder(encoded).codeUnitAt(0);
+//		return utf16.decode(encoded);
+//        return Char.ConvertToUtf32(high, low);
+    }
+}
+