Integrate parser generation into mx build

Author: msimacek

.gitignore

@@ -53,6 +53,7 @@ language
 /request_cache/
 /org.eclipse.jdt.core.prefs
 Python3.g4.stamp
+python.gram.stamp
 *.orig
 /*.diff
 /testenv

graalpython/com.oracle.graal.python.pegparser.generator/Makefile

@@ -0,0 +1,24 @@
+QUIETLY$(MX_VERBOSE) = @
+
+PARSER_PATH ?= ../com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser
+PYTHON ?= python3
+
+TARGET=${PARSER_PATH}/Parser.java
+
+GRAMMAR=${PARSER_PATH}/python.gram
+TOKENS=${PARSER_PATH}/Tokens
+
+PEGEN_FILES=$(shell find pegen pegjava -name '*.py')
+
+STAMP=${GRAMMAR}.stamp
+
+.PHONY: default clean
+default: ${STAMP}
+
+${STAMP}: ${GRAMMAR} ${TOKENS} ${PEGEN_FILES} main_parser_gen.py
+	$(QUIETLY) ${PYTHON} main_parser_gen.py ${GRAMMAR} ${TOKENS} ${TARGET}
+	$(QUIETLY) touch $@
+
+clean:
+	$(QUIETLY) rm -f ${TARGET}
+	$(QUIETLY) rm -f ${STAMP}

graalpython/com.oracle.graal.python.pegparser/src/python/diff_generator.py renamed to graalpython/com.oracle.graal.python.pegparser.generator/diff_generator.py

@@ -36,63 +36,42 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import argparse
 import os
 import sys
 import tokenize
 
-from os import path
-
 from pegen.build import generate_token_definitions
-from pegen.grammar import Grammar
 from pegen.grammar_parser import GeneratedParser as GrammarParser
 from pegen.tokenizer import Tokenizer
 from pegjava.java_generator import JavaParserGenerator
 
-verbose_tokenizer = False
-verbose_parser = False
 
 def main():
-    __dir__ = path.dirname(__file__)
-    grammar_file = path.relpath(path.join(__dir__, "pegjava", "python.gram"), os.getcwd())
-    print("Reading", grammar_file)
-    with open(grammar_file) as file:
-        tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=verbose_tokenizer)
-        parser = GrammarParser(tokenizer, verbose=verbose_parser)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("grammar_file")
+    parser.add_argument("tokens_file")
+    parser.add_argument("output_file")
+    parser.add_argument("--verbose", action="store_true")
+    parser.add_argument("--debug", action="store_true")
+
+    args = parser.parse_args()
+
+    with open(args.grammar_file) as file:
+        tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=args.verbose)
+        parser = GrammarParser(tokenizer, verbose=args.verbose)
         grammar = parser.start()
 
     if not grammar:
-        sys.exit("Fail")
+        sys.exit("Failed to generate grammar")
 
-    tokens_file = path.join(__dir__, "pegjava", "Tokens")
-    with open(tokens_file, "r") as tok_file:
+    with open(args.tokens_file, "r") as tok_file:
         all_tokens, exact_tokens, non_exact_tokens = generate_token_definitions(tok_file)
-#    print("all_tokens")
-#    print(all_tokens)
 
-#    print("exact_tokens")
-#    print(exact_tok)
+    with open(args.output_file, "w") as file:
+        gen = JavaParserGenerator(grammar, all_tokens, exact_tokens, non_exact_tokens, file, debug=args.debug)
+        gen.generate(os.path.relpath(args.grammar_file, os.path.dirname(args.output_file)))
 
-#    print("non_exact_tokens")
-#    print(non_exact_tok)
-    output_file = path.join(__dir__, "..", "java", "com", "oracle", "graal", "python", "pegparser", "Parser.java")
-    with open(output_file, "w") as file:
-        gen: ParserGenerator = JavaParserGenerator(grammar, all_tokens, exact_tokens, non_exact_tokens, file, debug=False)
-        gen.generate(grammar_file)
-#    print("[")
-#    for rule in rules:
-#        print(f"  {rule},")
-#    print("]")
-#    for rule in rules:
-#        print(rule.name, end=": ", file=sys.stderr)
-#        print(*(" ".join(alt) for alt in rule.alts), sep=" | ", file=sys.stderr)
-#
-#
-#
-#    outfile = "../src/genPythonParser/GenParser.java"
-#    print("Updating", outfile, file=sys.stderr)
-#    with open(outfile, "w") as stream:
-#        generate(rules, stream)
 
 if __name__ == '__main__':
     main()