bpo-40334: Support CO_FUTURE_BARRY_AS_BDFL in the new parser (GH-19721)

This commit also allows to pass flags to the new parser in all interfaces and fixes a bug in the parser generator that was causing to inline rules with actions, making them disappear.

Author: pablogsal

Grammar/python.gram

@@ -323,7 +323,8 @@ compare_op_bitwise_or_pair[CmpopExprPair*]:
     | isnot_bitwise_or
     | is_bitwise_or
 eq_bitwise_or[CmpopExprPair*]: '==' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Eq, a) }
-noteq_bitwise_or[CmpopExprPair*]: '!=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, NotEq, a) }
+noteq_bitwise_or[CmpopExprPair*]:
+    | (tok='!=' {_PyPegen_check_barry_as_flufl(p) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) }
 lte_bitwise_or[CmpopExprPair*]: '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) }
 lt_bitwise_or[CmpopExprPair*]: '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) }
 gte_bitwise_or[CmpopExprPair*]: '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) }

Include/internal/pegen_interface.h

@@ -11,21 +11,24 @@ extern "C" {
 #include "Python.h"
 #include "Python-ast.h"
 
-PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, int mode, PyArena *arena);
+PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, int mode, PyCompilerFlags*, PyArena *arena);
 PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, int mode, PyCompilerFlags *flags,
                                          PyArena *arena);
 PyAPI_FUNC(mod_ty) PyPegen_ASTFromStringObject(const char *str, PyObject* filename, int mode,
                                                PyCompilerFlags *flags, PyArena *arena);
 PyAPI_FUNC(mod_ty) PyPegen_ASTFromFileObject(FILE *fp, PyObject *filename_ob,
                                              int mode, const char *enc, const char *ps1,
-                                             const char *ps2, int *errcode, PyArena *arena);
-PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, int mode);
+                                             const char *ps2, PyCompilerFlags *flags,
+                                             int *errcode, PyArena *arena);
+PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, int mode, PyCompilerFlags *flags);
 PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, int mode,
                                                         PyCompilerFlags *flags);
 PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFileObject(FILE *, PyObject *filename_ob,
-                                                            int mode, const char *enc,
+                                                            int mode,
                                                             const char *ps1,
                                                             const char *ps2,
+                                                            PyCompilerFlags *flags,
+                                                            const char *enc,
                                                             int *errcode);
 
 #ifdef __cplusplus

Lib/test/test_flufl.py

@@ -4,7 +4,6 @@
 from test import support
 
 
-@support.skip_if_new_parser("Not supported by pegen yet")
 class FLUFLTests(unittest.TestCase):
 
     def test_barry_as_bdfl(self):
@@ -16,21 +15,26 @@ def test_barry_as_bdfl(self):
                     __future__.CO_FUTURE_BARRY_AS_BDFL)
         self.assertRegex(str(cm.exception),
                          "with Barry as BDFL, use '<>' instead of '!='")
-        self.assertEqual(cm.exception.text, '2 != 3\n')
+        self.assertIn('2 != 3', cm.exception.text)
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
-        self.assertEqual(cm.exception.lineno, 2)
-        self.assertEqual(cm.exception.offset, 4)
+
+        self.assertTrue(cm.exception.lineno, 2)
+        # The old parser reports the end of the token and the new
+        # parser reports the start of the token
+        self.assertEqual(cm.exception.offset, 4 if support.use_old_parser() else 3)
 
     def test_guido_as_bdfl(self):
         code = '2 {0} 3'
         compile(code.format('!='), '<BDFL test>', 'exec')
         with self.assertRaises(SyntaxError) as cm:
             compile(code.format('<>'), '<FLUFL test>', 'exec')
         self.assertRegex(str(cm.exception), "invalid syntax")
-        self.assertEqual(cm.exception.text, '2 <> 3\n')
+        self.assertIn('2 <> 3', cm.exception.text)
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
         self.assertEqual(cm.exception.lineno, 1)
-        self.assertEqual(cm.exception.offset, 4)
+        # The old parser reports the end of the token and the new
+        # parser reports the start of the token
+        self.assertEqual(cm.exception.offset, 4 if support.use_old_parser() else 3)
 
 
 if __name__ == '__main__':

Modules/_peg_parser.c

@@ -28,9 +28,10 @@ _Py_parse_file(PyObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
     PyObject *result = NULL;
 
-    mod_ty res = PyPegen_ASTFromFile(filename, mode, arena);
+    mod_ty res = PyPegen_ASTFromFile(filename, mode, &flags, arena);
     if (res == NULL) {
         goto error;
     }