Support for new type parameters syntax in Bytecode DSL interpreter + update Bytecode DSL tags according to 3.12

Author: steve-s

graalpython/com.oracle.graal.python.pegparser.generator/asdl/asdl_java.py

@@ -108,6 +108,9 @@ def generate_sst_node(emitter: java_file.Emitter, c: model.ConcreteClass):
                     if not f.is_nullable and f.type.java not in ('int', 'boolean'):
                         emitter.println(f'assert {f.name.java} != null;')
                     emitter.println(f'this.{f.name.java} = {f.name.java};')
+            if 'typeParams' in [f.name.java for f in c.fields]:
+                with emitter.define('public boolean isGeneric()'):
+                    emitter.println('return typeParams != null && typeParams.length > 0;')
             # accept() method
             with emitter.define('public <T> T accept(SSTreeVisitor<T> visitor)', '@Override'):
                 emitter.println('return visitor.visit(this);')

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/scope/Scope.java

@@ -227,6 +227,10 @@ public boolean isFunction() {
         return type.isFunctionLike();
     }
 
+    public boolean isTypeParam() {
+        return type == ScopeType.TypeParam;
+    }
+
     public boolean isClass() {
         return type == ScopeType.Class;
     }

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/sst/StmtTy.java

@@ -75,6 +75,10 @@ public FunctionDef(String name, ArgumentsTy args, StmtTy[] body, ExprTy[] decora
             this.typeParams = typeParams;
         }
 
+        public boolean isGeneric() {
+            return typeParams != null && typeParams.length > 0;
+        }
+
         @Override
         public <T> T accept(SSTreeVisitor<T> visitor) {
             return visitor.visit(this);
@@ -103,6 +107,10 @@ public AsyncFunctionDef(String name, ArgumentsTy args, StmtTy[] body, ExprTy[] d
             this.typeParams = typeParams;
         }
 
+        public boolean isGeneric() {
+            return typeParams != null && typeParams.length > 0;
+        }
+
         @Override
         public <T> T accept(SSTreeVisitor<T> visitor) {
             return visitor.visit(this);
@@ -128,6 +136,10 @@ public ClassDef(String name, ExprTy[] bases, KeywordTy[] keywords, StmtTy[] body
             this.typeParams = typeParams;
         }
 
+        public boolean isGeneric() {
+            return typeParams != null && typeParams.length > 0;
+        }
+
         @Override
         public <T> T accept(SSTreeVisitor<T> visitor) {
             return visitor.visit(this);
@@ -195,6 +207,10 @@ public TypeAlias(ExprTy name, TypeParamTy[] typeParams, ExprTy value, SourceRang
             this.value = value;
         }
 
+        public boolean isGeneric() {
+            return typeParams != null && typeParams.length > 0;
+        }
+
         @Override
         public <T> T accept(SSTreeVisitor<T> visitor) {
             return visitor.visit(this);

graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

@@ -117,7 +117,10 @@ def get_setuptools(setuptools='setuptools==67.6.1'):
             py_executable = temp_env / 'Scripts' / 'python.exe'
         else:
             py_executable = temp_env / 'bin' / 'python3'
-        subprocess.run([py_executable, "-m", "pip", "install", "--target", str(setuptools_path), setuptools], check=True)
+        extra_args = []
+        if GRAALPYTHON and __graalpython__.is_bytecode_dsl_interpreter:
+            extra_args = ['--vm.Dpython.EnableBytecodeDSLInterpreter=true']
+        subprocess.run([py_executable, *extra_args, "-m", "pip", "install", "--target", str(setuptools_path), setuptools], check=True)
         print('setuptools is installed in %s' % setuptools_path)
         shutil.rmtree(temp_env)
 

graalpython/com.oracle.graal.python.test/src/tests/test_assign.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -210,3 +210,11 @@ def test_illegal_assignment(self):
 
         with self.assertRaisesRegex(SyntaxError, "assign to list comprehension|invalid syntax\. Maybe you meant '==' or ':=' instead of '='\?"):
             compile("[s for s in [1]], b, c = (1, 2, 3)", "<test>", "exec")
+
+
+class NotIllegaAssigmentTest(unittest.TestCase):
+    def test_not_syntax_error(self):
+        # If this changes in CPython, we just need to add the missing forbidden name checks to the bytecode compiler
+        with self.assertRaises(AttributeError):
+            x = object()
+            x.__debug__ += 1

graalpython/com.oracle.graal.python.test/src/tests/test_decorators.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -58,3 +58,54 @@ def myfunc(a, b, **kwargs):
     assert global_log[-1] == ["myfunc", (10, 20), {}]
     assert myfunc(10, 20, x=1, y=2) == (10, 20, {'x': 1, 'y': 2})
     assert global_log[-1] == ["myfunc", (10, 20), {'x': 1, 'y': 2}]
+
+
+def test_eval_order():
+    MyBase = None
+
+    def create_base(dummy_arg):
+        nonlocal MyBase
+        class MyBaseK:
+            pass
+        MyBase = MyBaseK
+        return lambda a: a
+
+    # this should just work: the decorator expression first
+    # creates MyBase and only then is the rest evaluated
+    @create_base('dummy')
+    class MyClass(MyBase):
+        pass
+
+    # dummy helper for following tests
+    def my_decorator(x):
+        return x
+
+    class assert_name_error:
+        def __init__(self, name):
+            self.name = name
+        def __enter__(self):
+            pass
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            assert exc_type == NameError, f"did not raise NameError with name '{self.name}'"
+            assert self.name in str(exc_val), f"not NameError for name '{self.name}'"
+            return True
+
+    with assert_name_error('my_decoratorr'):
+        @my_decoratorr
+        class ClassA(NonExistingBaseClass):
+            pass
+
+    with assert_name_error('NonExistingBase'):
+        @my_decorator
+        class ClassA(NonExistingBase):
+            pass
+
+    with assert_name_error('my_decoratorr'):
+        @my_decoratorr
+        def my_function(x=NonexistingName):
+            pass
+
+    with assert_name_error('NonexistingDefaultName'):
+        @my_decorator
+        def my_function(x=NonexistingDefaultName):
+            pass

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_eof.txt

@@ -2,5 +2,3 @@ test.test_eof.EOFTestCase.test_EOFS @ linux-x86_64
 test.test_eof.EOFTestCase.test_EOFS_with_file @ linux-x86_64
 test.test_eof.EOFTestCase.test_EOF_single_quote @ linux-x86_64
 test.test_eof.EOFTestCase.test_eof_with_line_continuation @ linux-x86_64
-test.test_eof.EOFTestCase.test_line_continuation_EOF @ linux-x86_64
-test.test_eof.EOFTestCase.test_line_continuation_EOF_from_file_bpo2180 @ linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_fstring.txt

@@ -3,20 +3,26 @@ test.test_fstring.TestCase.test_arguments @ linux-x86_64
 test.test_fstring.TestCase.test_assignment @ linux-x86_64
 test.test_fstring.TestCase.test_ast @ linux-x86_64
 test.test_fstring.TestCase.test_ast_compile_time_concat @ linux-x86_64
+test.test_fstring.TestCase.test_ast_fstring_empty_format_spec @ linux-x86_64
 test.test_fstring.TestCase.test_ast_line_numbers @ linux-x86_64
 test.test_fstring.TestCase.test_ast_line_numbers_duplicate_expression @ linux-x86_64
+test.test_fstring.TestCase.test_ast_line_numbers_multiline_fstring @ linux-x86_64
 test.test_fstring.TestCase.test_ast_line_numbers_multiple_formattedvalues @ linux-x86_64
 test.test_fstring.TestCase.test_ast_line_numbers_nested @ linux-x86_64
+test.test_fstring.TestCase.test_ast_line_numbers_with_parentheses @ linux-x86_64
 test.test_fstring.TestCase.test_ast_numbers_fstring_with_formatting @ linux-x86_64
 test.test_fstring.TestCase.test_backslash_char @ linux-x86_64
+test.test_fstring.TestCase.test_backslashes_in_expression_part @ linux-x86_64
 test.test_fstring.TestCase.test_backslashes_in_string_part @ linux-x86_64
 test.test_fstring.TestCase.test_call @ linux-x86_64
 test.test_fstring.TestCase.test_closure @ linux-x86_64
 test.test_fstring.TestCase.test_comments @ linux-x86_64
 test.test_fstring.TestCase.test_compile_time_concat @ linux-x86_64
 test.test_fstring.TestCase.test_compile_time_concat_errors @ linux-x86_64
 test.test_fstring.TestCase.test_conversions @ linux-x86_64
+test.test_fstring.TestCase.test_custom_format_specifier @ linux-x86_64
 test.test_fstring.TestCase.test_debug_conversion @ linux-x86_64
+test.test_fstring.TestCase.test_debug_in_file @ linux-x86_64
 test.test_fstring.TestCase.test_del @ linux-x86_64
 test.test_fstring.TestCase.test_dict @ linux-x86_64
 test.test_fstring.TestCase.test_docstring @ linux-x86_64
@@ -27,8 +33,14 @@ test.test_fstring.TestCase.test_errors @ linux-x86_64
 test.test_fstring.TestCase.test_expressions_with_triple_quoted_strings @ linux-x86_64
 test.test_fstring.TestCase.test_filename_in_syntaxerror @ linux-x86_64
 test.test_fstring.TestCase.test_format_specifier_expressions @ linux-x86_64
+test.test_fstring.TestCase.test_fstring_backslash_before_double_bracket @ linux-x86_64
+test.test_fstring.TestCase.test_fstring_backslash_before_double_bracket_warns_once @ linux-x86_64
+test.test_fstring.TestCase.test_fstring_backslash_prefix_raw @ linux-x86_64
+test.test_fstring.TestCase.test_fstring_format_spec_greedy_matching @ linux-x86_64
+test.test_fstring.TestCase.test_fstring_nested_too_deeply @ linux-x86_64
 test.test_fstring.TestCase.test_global @ linux-x86_64
 test.test_fstring.TestCase.test_if_conditional @ linux-x86_64
+test.test_fstring.TestCase.test_invalid_backslashes_inside_fstring_context @ linux-x86_64
 test.test_fstring.TestCase.test_invalid_string_prefixes @ linux-x86_64
 test.test_fstring.TestCase.test_invalid_syntax_error_message @ linux-x86_64
 test.test_fstring.TestCase.test_lambda @ linux-x86_64
@@ -52,11 +64,15 @@ test.test_fstring.TestCase.test_no_backslashes_in_expression_part @ linux-x86_64
 test.test_fstring.TestCase.test_no_escapes_for_braces @ linux-x86_64
 test.test_fstring.TestCase.test_not_equal @ linux-x86_64
 test.test_fstring.TestCase.test_parens_in_expressions @ linux-x86_64
+test.test_fstring.TestCase.test_roundtrip_raw_quotes @ linux-x86_64
 test.test_fstring.TestCase.test_shadowed_global @ linux-x86_64
 test.test_fstring.TestCase.test_side_effect_order @ linux-x86_64
 test.test_fstring.TestCase.test_str_format_differences @ linux-x86_64
+test.test_fstring.TestCase.test_syntax_error_after_debug @ linux-x86_64
 test.test_fstring.TestCase.test_syntax_error_for_starred_expressions @ linux-x86_64
+test.test_fstring.TestCase.test_syntax_error_in_nested_fstring @ linux-x86_64
 test.test_fstring.TestCase.test_unterminated_string @ linux-x86_64
+test.test_fstring.TestCase.test_valid_prefixes @ linux-x86_64
 test.test_fstring.TestCase.test_walrus @ linux-x86_64
 test.test_fstring.TestCase.test_with_a_commas_and_an_underscore_in_format_specifier @ linux-x86_64
 test.test_fstring.TestCase.test_with_an_underscore_and_a_comma_in_format_specifier @ linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_hashlib.txt

@@ -59,5 +59,3 @@ test.test_hashlib.HashLibTestCase.test_usedforsecurity_false @ linux-x86_64
 test.test_hashlib.HashLibTestCase.test_usedforsecurity_true @ linux-x86_64
 test.test_hashlib.KDFTests.test_file_digest @ linux-x86_64
 test.test_hashlib.KDFTests.test_normalized_name @ linux-x86_64
-test.test_hashlib.KDFTests.test_pbkdf2_hmac_c @ linux-x86_64
-test.test_hashlib.KDFTests.test_pbkdf2_hmac_py @ linux-x86_64