in token.c

Author: albertedwardson

Include/internal/pycore_global_objects_fini_generated.h

@@ -49,7 +49,6 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(PERCENTEQUAL, "%=")
         STRUCT_FOR_STR(PLUSEQUAL, "+=")
         STRUCT_FOR_STR(RARROW, "->")
-        STRUCT_FOR_STR(REPLSHIFT, ">>>")
         STRUCT_FOR_STR(RIGHTSHIFT, ">>")
         STRUCT_FOR_STR(RIGHTSHIFTEQUAL, ">>=")
         STRUCT_FOR_STR(SLASHEQUAL, "/=")

Include/internal/pycore_global_strings.h

@@ -261,7 +261,7 @@ def generate_global_strings(identifiers, strings):
             outfile.write('\n')
             with printer.block('struct', ' identifiers;'):
                 for name in sorted(identifiers):
-                    # assert name.isidentifier(), name
+                    assert name.isidentifier(), name
                     printer.write(f'STRUCT_FOR_ID({name})')
             with printer.block('struct', ' ascii[128];'):
                 printer.write("PyASCIIObject _ascii;")
@@ -324,7 +324,7 @@ def generate_runtime_init(identifiers, strings):
         printer.write('')
         with printer.block('#define _Py_str_identifiers_INIT', continuation=True):
             for name in sorted(identifiers):
-                # assert name.isidentifier(), name
+                assert name.isidentifier(), name
                 printer.write(f'INIT_ID({name}),')
                 immortal_objects.append(f'(PyObject *)&_Py_ID({name})')
         printer.write('')
@@ -453,33 +453,6 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]':
 
 def main() -> None:
     identifiers, strings = get_identifiers_and_strings()
-    strings.update([
-        ("==", "EQEQUAL"),
-        ("!=", "NOTEQUAL"),
-        ("<=", "LESSEQUAL"),
-        (">=", "GREATEREQUAL"),
-        ("<<", "LEFTSHIFT"),
-        (">>", "RIGHTSHIFT"),
-        ("**", "DOUBLESTAR"),
-        ("+=", "PLUSEQUAL"),
-        ("-=", "MINEQUAL"),
-        ("*=", "STAREQUAL"),
-        ("/=", "SLASHEQUAL"),
-        ("%=", "PERCENTEQUAL"),
-        ("&=", "AMPEREQUAL"),
-        ("|=", "VBAREQUAL"),
-        ("^=", "CIRCUMFLEXEQUAL"),
-        ("<<=", "LEFTSHIFTEQUAL"),
-        (">>=", "RIGHTSHIFTEQUAL"),
-        ("**=", "DOUBLESTAREQUAL"),
-        ("//", "DOUBLESLASH"),
-        ("//=", "DOUBLESLASHEQUAL"),
-        ("@=", "ATEQUAL"),
-        ("->", "RARROW"),
-        ("...", "ELLIPSIS"),
-        (":=", "COLONEQUAL"),
-        (">>>", "REPLSHIFT"),
-    ])
     generate_global_strings(identifiers, strings)
     generated_immortal_objects = generate_runtime_init(identifiers, strings)
     generate_static_strings_initializer(identifiers, strings)

Include/internal/pycore_runtime_init_generated.h

@@ -125,6 +125,12 @@ def make_h(infile, outfile='Include/internal/pycore_token.h'):
 
 #include "Python.h"
 #include "pycore_token.h"
+#include "pycore_global_strings.h"
+
+/* Interning tokens */
+
+%s\
+
 
 /* Token names */
 
@@ -176,10 +182,23 @@ def generate_chars_to_token(mapping, n=1):
     write('}\n')
     return ''.join(result)
 
+def declate_tokens(tok_name_to_string):
+    return "\n".join(
+        f'_Py_DECLARE_STR({name}, "{token}");'
+        for name, token in tok_name_to_string.items()
+    )
+
+
 def make_c(infile, outfile='Parser/token.c'):
     tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
-    string_to_tok['<>'] = string_to_tok['!=']
+    # string_to_tok['<>'] = string_to_tok['!=']
     chars_to_token = {}
+    tok_name_to_string = {
+        tok_names[name_idx]: token
+        for token, name_idx in string_to_tok.items()
+        if len(token) > 1
+    }
+
     for string, value in string_to_tok.items():
         assert 1 <= len(string) <= 3
         name = tok_names[value]
@@ -196,6 +215,7 @@ def make_c(infile, outfile='Parser/token.c'):
     names.append('    "<N_TOKENS>",\n')
 
     if update_file(outfile, token_c_template % (
+            declate_tokens(tok_name_to_string),
             ''.join(names),
             generate_chars_to_token(chars_to_token[1]),
             generate_chars_to_token(chars_to_token[2]),