bpo-40334: Rename PyConfig.use_peg to _use_peg_parser (GH-19670)

vstinner · web-flow · commit 1def7754b7a4 · 2020-04-23T03:03:24.000+02:00
* Rename PyConfig.use_peg to _use_peg_parser
* Document PyConfig._use_peg_parser and mark it a deprecated
* Mark -X oldparser option and PYTHONOLDPARSER env var as deprecated
  in the documentation.
* Add use_old_parser() and skip_if_new_parser() to test.support
* Remove sys.flags.use_peg: use_old_parser() uses
  _testinternalcapi.get_configs() instead.
* Enhance test_embed tests
* subprocess._args_from_interpreter_flags() copies -X oldparser
diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
@@ -686,6 +686,16 @@ PyConfig
 
       :data:`sys._xoptions`.
 
+   .. c:member:: int _use_peg_parser
+
+      Enable PEG parser? Default: 1.
+
+      Set to 0 by :option:`-X oldparser <-X>` and :envvar:`PYTHONOLDPARSER`.
+
+      See also :pep:`617`.
+
+      .. deprecated-removed:: 3.9 3.10
+
 If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same
 way the regular Python parses command line arguments, and Python
 arguments are stripped from ``argv``: see :ref:`Command Line Arguments
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
@@ -427,7 +427,7 @@ Miscellaneous options
 
    * ``-X faulthandler`` to enable :mod:`faulthandler`;
    * ``-X oldparser``: enable the traditional LL(1) parser.  See also
-     :envvar:`PYTHONOLDPARSER`.
+     :envvar:`PYTHONOLDPARSER` and :pep:`617`.
    * ``-X showrefcount`` to output the total reference count and number of used
      memory blocks when the program finishes or after each statement in the
      interactive interpreter. This only works on debug builds.
@@ -480,6 +480,9 @@ Miscellaneous options
 
       The ``-X showalloccount`` option has been removed.
 
+   .. deprecated-removed:: 3.9 3.10
+      The ``-X oldparser`` option.
+
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -578,8 +581,11 @@ conflict.
 
 .. envvar:: PYTHONOLDPARSER
 
-   If this is set it is equivalent to specifying the :option:`-X`
-   ``oldparser`` option.
+   If this is set to a non-empty string, enable the traditional LL(1) parser.
+
+   See also the :option:`-X` ``oldparser`` option and :pep:`617`.
+
+   .. deprecated-removed:: 3.9 3.10
 
 
 .. envvar:: PYTHONINSPECT
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
@@ -149,7 +149,7 @@ typedef struct {
 
     /* Enable PEG parser?
        1 by default, set to 0 by -X oldparser and PYTHONOLDPARSER */
-    int use_peg;
+    int _use_peg_parser;
 
     /* Enable tracemalloc?
        Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
@@ -326,7 +326,7 @@ def _args_from_interpreter_flags():
     if dev_mode:
         args.extend(('-X', 'dev'))
     for opt in ('faulthandler', 'tracemalloc', 'importtime',
-                'showrefcount', 'utf8'):
+                'showrefcount', 'utf8', 'oldparser'):
         if opt in xoptions:
             value = xoptions[opt]
             if value is True:
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
@@ -3454,3 +3454,13 @@ def wait_process(pid, *, exitcode, timeout=None):
     # sanity check: it should not fail in practice
     if pid2 != pid:
         raise AssertionError(f"pid {pid2} != pid {pid}")
+
+
+def use_old_parser():
+    import _testinternalcapi
+    config = _testinternalcapi.get_configs()
+    return (config['config']['_use_peg_parser'] == 0)
+
+
+def skip_if_new_parser(msg):
+    return unittest.skipIf(not use_old_parser(), msg)
diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
@@ -4,12 +4,12 @@
 """
 import sys
 import unittest
-from test.support import is_jython
+from test import support
 
 from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
 import io
 
-if is_jython:
+if support.is_jython:
 
     def unify_callables(d):
         for n,v in d.items():
@@ -21,7 +21,7 @@ class CodeopTests(unittest.TestCase):
 
     def assertValid(self, str, symbol='single'):
         '''succeed iff str is a valid piece of code'''
-        if is_jython:
+        if support.is_jython:
             code = compile_command(str, "<input>", symbol)
             self.assertTrue(code)
             if symbol == "single":
@@ -60,7 +60,7 @@ def test_valid(self):
         av = self.assertValid
 
         # special case
-        if not is_jython:
+        if not support.is_jython:
             self.assertEqual(compile_command(""),
                              compile("pass", "<input>", 'single',
                                      PyCF_DONT_IMPLY_DEDENT))
@@ -122,7 +122,7 @@ def test_valid(self):
         av("def f():\n pass\n#foo\n")
         av("@a.b.c\ndef f():\n pass\n")
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not support PyCF_DONT_INPLY_DEDENT yet")
+    @support.skip_if_new_parser("Pegen does not support PyCF_DONT_INPLY_DEDENT yet")
     def test_incomplete(self):
         ai = self.assertIncomplete
 
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
@@ -501,7 +501,7 @@ def test_single_statement(self):
         self.compile_single("if x:\n   f(x)\nelse:\n   g(x)")
         self.compile_single("class T:\n   pass")
 
-    @unittest.skipIf(sys.flags.use_peg, 'Pegen does not disallow multiline single stmts')
+    @support.skip_if_new_parser('Pegen does not disallow multiline single stmts')
     def test_bad_single_statement(self):
         self.assertInvalidSingle('1\n2')
         self.assertInvalidSingle('def f(): pass')
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
@@ -347,7 +347,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'isolated': 0,
         'use_environment': 1,
         'dev_mode': 0,
-        'use_peg': 1,
+        '_use_peg_parser': 1,
 
         'install_signal_handlers': 1,
         'use_hash_seed': 0,
@@ -729,7 +729,7 @@ def test_init_from_config(self):
             'import_time': 1,
             'show_ref_count': 1,
             'malloc_stats': 1,
-            'use_peg': 0,
+            '_use_peg_parser': 0,
 
             'stdio_encoding': 'iso8859-1',
             'stdio_errors': 'replace',
@@ -792,6 +792,7 @@ def test_init_compat_env(self):
             'user_site_directory': 0,
             'faulthandler': 1,
             'warnoptions': ['EnvVar'],
+            '_use_peg_parser': 0,
         }
         self.check_all_configs("test_init_compat_env", config, preconfig,
                                api=API_COMPAT)
@@ -819,6 +820,7 @@ def test_init_python_env(self):
             'user_site_directory': 0,
             'faulthandler': 1,
             'warnoptions': ['EnvVar'],
+            '_use_peg_parser': 0,
         }
         self.check_all_configs("test_init_python_env", config, preconfig,
                                api=API_PYTHON)
diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py
@@ -26,7 +26,7 @@ def test_EOFS(self):
         else:
             raise support.TestFailed
 
-    @unittest.skipIf(sys.flags.use_peg, "TODO for PEG -- fails with new parser")
+    @support.skip_if_new_parser("TODO for PEG -- fails with new parser")
     def test_line_continuation_EOF(self):
         """A continuation at the end of input must be an error; bpo2180."""
         expect = 'unexpected EOF while parsing (<string>, line 1)'
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
@@ -178,7 +178,7 @@ def ckmsg(src, msg, exception=SyntaxError):
         s = '''if True:\n        print()\n\texec "mixed tabs and spaces"'''
         ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError)
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen column offsets might be different")
+    @support.skip_if_new_parser("Pegen column offsets might be different")
     def testSyntaxErrorOffset(self):
         def check(src, lineno, offset, encoding='utf-8'):
             with self.assertRaises(SyntaxError) as cm:
diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py
@@ -1,9 +1,10 @@
 import __future__
 import unittest
 import sys
+from test import support
 
 
-@unittest.skipIf(sys.flags.use_peg, "Not supported by pegen yet")
+@support.skip_if_new_parser("Not supported by pegen yet")
 class FLUFLTests(unittest.TestCase):
 
     def test_barry_as_bdfl(self):
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
@@ -12,6 +12,7 @@
 import decimal
 import sys
 import unittest
+from test import support
 
 a_global = 'global variable'
 
@@ -206,7 +207,7 @@ def test_ast_line_numbers_nested(self):
         call = binop.right.values[1].value
         self.assertEqual(type(call), ast.Call)
         self.assertEqual(call.lineno, 3)
-        if not sys.flags.use_peg:
+        if support.use_old_parser():
             self.assertEqual(call.col_offset, 11)
 
     def test_ast_line_numbers_duplicate_expression(self):
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py
@@ -900,7 +900,7 @@ def test_deeply_nested_list(self):
         st = parser.expr(e)
         st.compile()
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not trigger memory error with this many parenthesis")
+    @support.skip_if_new_parser("Pegen does not trigger memory error with this many parenthesis")
     def test_trigger_memory_error(self):
         e = self._nested_expression(100)
         rc, out, err = assert_python_failure('-Xoldparser', '-c', e)
diff --git a/Lib/test/test_peg_parser.py b/Lib/test/test_peg_parser.py
@@ -6,6 +6,7 @@
 from pathlib import PurePath
 from typing import Any, Union, Iterable, Tuple
 from textwrap import dedent
+from test import support
 
 
 TEST_CASES = [
@@ -720,7 +721,7 @@ def test_incorrect_ast_generation_with_specialized_errors(self) -> None:
                 f"Actual error message does not match expexted for {source}"
             )
 
-    @unittest.skipIf(sys.flags.use_peg, "This tests nothing for now, since compile uses pegen as well")
+    @support.skip_if_new_parser("This tests nothing for now, since compile uses pegen as well")
     @unittest.expectedFailure
     def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
         for source in GOOD_BUT_FAIL_SOURCES:
diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py
@@ -5,7 +5,7 @@
 import unittest
 import sys
 
-from test.support import check_syntax_error
+from test.support import check_syntax_error, use_old_parser
 
 
 def global_pos_only_f(a, b, /):
@@ -24,7 +24,7 @@ def assertRaisesSyntaxError(self, codestr, regex="invalid syntax"):
             compile(codestr + "\n", "<test>", "single")
 
     def test_invalid_syntax_errors(self):
-        if not sys.flags.use_peg:
+        if use_old_parser():
             check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "non-default argument follows default argument")
@@ -47,7 +47,7 @@ def test_invalid_syntax_errors(self):
         check_syntax_error(self, "def f(a, *, c, /, d, e): pass")
 
     def test_invalid_syntax_errors_async(self):
-        if not sys.flags.use_peg:
+        if use_old_parser():
             check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "non-default argument follows default argument")
@@ -236,7 +236,7 @@ def test_lambdas(self):
         self.assertEqual(x(1, 2), 3)
 
     def test_invalid_syntax_lambda(self):
-        if not sys.flags.use_peg:
+        if use_old_parser():
             check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument")
             check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument")
             check_syntax_error(self, "lambda a = 5, b, /: None", "non-default argument follows default argument")
diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py
@@ -33,6 +33,7 @@
 import tempfile
 import unittest
 import warnings
+from test.support import check_syntax_warning, use_old_parser
 
 
 TEMPLATE = r"""# coding: %s
@@ -63,8 +64,6 @@ def byte(i):
 
 class TestLiterals(unittest.TestCase):
 
-    from test.support import check_syntax_warning
-
     def setUp(self):
         self.save_path = sys.path[:]
         self.tmpdir = tempfile.mkdtemp()
@@ -119,7 +118,7 @@ def test_eval_str_invalid_escape(self):
             eval("'''\n\\z'''")
         self.assertEqual(len(w), 1)
         self.assertEqual(w[0].filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(w[0].lineno, 1)
 
         with warnings.catch_warnings(record=True) as w:
@@ -129,7 +128,7 @@ def test_eval_str_invalid_escape(self):
             exc = cm.exception
         self.assertEqual(w, [])
         self.assertEqual(exc.filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(exc.lineno, 1)
 
     def test_eval_str_raw(self):
@@ -170,7 +169,7 @@ def test_eval_bytes_invalid_escape(self):
             eval("b'''\n\\z'''")
         self.assertEqual(len(w), 1)
         self.assertEqual(w[0].filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(w[0].lineno, 1)
 
         with warnings.catch_warnings(record=True) as w:
@@ -180,7 +179,7 @@ def test_eval_bytes_invalid_escape(self):
             exc = cm.exception
         self.assertEqual(w, [])
         self.assertEqual(exc.filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(exc.lineno, 1)
 
     def test_eval_bytes_raw(self):
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
@@ -678,8 +678,8 @@ def _check_error(self, code, errtext,
     def test_assign_call(self):
         self._check_error("f() = 1", "assign")
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not produce a specialized error "
-                                        "message yet")
+    @support.skip_if_new_parser("Pegen does not produce a specialized error "
+                                "message yet")
     def test_assign_del(self):
         self._check_error("del f()", "delete")
 
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
@@ -545,7 +545,7 @@ def __hash__(self):
     def test_sys_flags(self):
         self.assertTrue(sys.flags)
         attrs = ("debug",
-                 "inspect", "interactive", "optimize", "use_peg",
+                 "inspect", "interactive", "optimize",
                  "dont_write_bytecode", "no_user_site", "no_site",
                  "ignore_environment", "verbose", "bytes_warning", "quiet",
                  "hash_randomization", "isolated", "dev_mode", "utf8_mode")
diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
@@ -656,8 +656,7 @@ def outer_raise():
         self.assertIn('inner_raise() # Marker', blocks[2])
         self.check_zero_div(blocks[2])
 
-    @unittest.skipIf(sys.flags.use_peg,
-                     "Pegen is arguably better here, so no need to fix this")
+    @support.skip_if_new_parser("Pegen is arguably better here, so no need to fix this")
     def test_syntax_error_offset_at_eol(self):
         # See #10186.
         def e():
diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py
@@ -1,6 +1,7 @@
 import ast
 import sys
 import unittest
+from test import support
 
 
 funcdef = """\
@@ -218,7 +219,7 @@ def favk(
 """
 
 
-@unittest.skipIf(sys.flags.use_peg, "Pegen does not support type comments yet")
+@support.skip_if_new_parser("Pegen does not support type comments yet")
 class TypeCommentTests(unittest.TestCase):
 
     lowest = 4  # Lowest minor version supported
diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py
@@ -328,7 +328,7 @@ def test_constant_tuples(self):
             ast.Constant(value=(1, 2, 3), kind=None), "(1, 2, 3)"
         )
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not support type annotation yet")
+    @test.support.skip_if_new_parser("Pegen does not support type annotation yet")
     def test_function_type(self):
         for function_type in (
             "() -> int",
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
diff --git a/Python/initconfig.c b/Python/initconfig.c
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
