gh-121404: split compile.c into compile.c and codegen.c

Include/internal/pycore_compile.h

@@ -8,6 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_ast.h"       // mod_ty
 #include "pycore_symtable.h"  // _Py_SourceLocation
 #include "pycore_instruction_sequence.h"
 
@@ -63,6 +64,115 @@ typedef struct {
     int u_firstlineno; /* the first lineno of the block */
 } _PyCompile_CodeUnitMetadata;
 
+struct compiler;
+
+typedef enum { OP_FAST, OP_GLOBAL, OP_DEREF, OP_NAME } compiler_optype;
+
+/* _PyCompile_FBlockInfo tracks the current frame block.
+ *
+ * A frame block is used to handle loops, try/except, and try/finally.
+ * It's called a frame block to distinguish it from a basic block in the
+ * compiler IR.
+ */
+
+enum _PyCompile_FBlockType {
+     COMPILER_FBLOCK_WHILE_LOOP,
+     COMPILER_FBLOCK_FOR_LOOP,
+     COMPILER_FBLOCK_TRY_EXCEPT,
+     COMPILER_FBLOCK_FINALLY_TRY,
+     COMPILER_FBLOCK_FINALLY_END,
+     COMPILER_FBLOCK_WITH,
+     COMPILER_FBLOCK_ASYNC_WITH,
+     COMPILER_FBLOCK_HANDLER_CLEANUP,
+     COMPILER_FBLOCK_POP_VALUE,
+     COMPILER_FBLOCK_EXCEPTION_HANDLER,
+     COMPILER_FBLOCK_EXCEPTION_GROUP_HANDLER,
+     COMPILER_FBLOCK_ASYNC_COMPREHENSION_GENERATOR,
+     COMPILER_FBLOCK_STOP_ITERATION,
+};
+
+typedef struct {
+    enum _PyCompile_FBlockType fb_type;
+    _PyJumpTargetLabel fb_block;
+    _Py_SourceLocation fb_loc;
+    /* (optional) type-specific exit or cleanup block */
+    _PyJumpTargetLabel fb_exit;
+    /* (optional) additional information required for unwinding */
+    void *fb_datum;
+} _PyCompile_FBlockInfo;
+
+
+int _PyCompiler_PushFBlock(struct compiler *c, _Py_SourceLocation loc,
+                           enum _PyCompile_FBlockType t,
+                           _PyJumpTargetLabel block_label,
+                           _PyJumpTargetLabel exit, void *datum);
+void _PyCompiler_PopFBlock(struct compiler *c, enum _PyCompile_FBlockType t,
+                           _PyJumpTargetLabel block_label);
+_PyCompile_FBlockInfo *_PyCompiler_TopFBlock(struct compiler *c);
+
+int _PyCompiler_EnterScope(struct compiler *c, identifier name, int scope_type,
+                           void *key, int lineno, PyObject *private,
+                           _PyCompile_CodeUnitMetadata *umd);
+void _PyCompiler_ExitScope(struct compiler *c);
+Py_ssize_t _PyCompiler_AddConst(struct compiler *c, PyObject *o);
+_PyInstructionSequence *_PyCompiler_InstrSequence(struct compiler *c);
+int _PyCompiler_FutureFeatures(struct compiler *c);
+PyObject *_PyCompiler_DeferredAnnotations(struct compiler *c);
+PyObject *_PyCompiler_Mangle(struct compiler *c, PyObject *name);
+PyObject *_PyCompiler_MaybeMangle(struct compiler *c, PyObject *name);
+int _PyCompiler_MaybeAddStaticAttributeToClass(struct compiler *c, expr_ty e);
+int _PyCompiler_GetRefType(struct compiler *c, PyObject *name);
+int _PyCompiler_LookupCellvar(struct compiler *c, PyObject *name);
+int _PyCompiler_ResolveNameop(struct compiler *c, PyObject *mangled, int scope,
+                              compiler_optype *optype, Py_ssize_t *arg);
+
+int _PyCompiler_IsInteractive(struct compiler *c);
+int _PyCompiler_IsNestedScope(struct compiler *c);
+int _PyCompiler_IsInInlinedComp(struct compiler *c);
+int _PyCompiler_ScopeType(struct compiler *c);
+int _PyCompiler_OptimizationLevel(struct compiler *c);
+PyArena *_PyCompiler_Arena(struct compiler *c);
+int _PyCompiler_LookupArg(struct compiler *c, PyCodeObject *co, PyObject *name);
+PyObject *_PyCompiler_Qualname(struct compiler *c);
+_PyCompile_CodeUnitMetadata *_PyCompiler_Metadata(struct compiler *c);
+PyObject *_PyCompiler_StaticAttributesAsTuple(struct compiler *c);
+
+#ifndef NDEBUG
+int _PyCompiler_IsTopLevelAwait(struct compiler *c);
+#endif
+
+struct symtable *_PyCompiler_Symtable(struct compiler *c);
+PySTEntryObject *_PyCompiler_SymtableEntry(struct compiler *c);
+
+enum {
+    COMPILER_SCOPE_MODULE,
+    COMPILER_SCOPE_CLASS,
+    COMPILER_SCOPE_FUNCTION,
+    COMPILER_SCOPE_ASYNC_FUNCTION,
+    COMPILER_SCOPE_LAMBDA,
+    COMPILER_SCOPE_COMPREHENSION,
+    COMPILER_SCOPE_ANNOTATIONS,
+};
+
+
+typedef struct {
+    PyObject *pushed_locals;
+    PyObject *temp_symbols;
+    PyObject *fast_hidden;
+    _PyJumpTargetLabel cleanup;
+} _PyCompille_InlinedComprehensionState;
+
+int _PyCompiler_TweakInlinedComprehensionScopes(struct compiler *c, _Py_SourceLocation loc,
+                                                PySTEntryObject *entry,
+                                                _PyCompille_InlinedComprehensionState *state);
+int _PyCompiler_RevertInlinedComprehensionScopes(struct compiler *c, _Py_SourceLocation loc,
+                                                 _PyCompille_InlinedComprehensionState *state);
+int _PyCompiler_AddDeferredAnnotaion(struct compiler *c, stmt_ty s);
+
+int _PyCodegen_AddReturnAtEnd(struct compiler *c, int addNone);
+int _PyCodegen_EnterAnonymousScope(struct compiler* c, mod_ty mod);
+int _PyCodegen_Expression(struct compiler *c, expr_ty e);
+int _PyCodegen_Body(struct compiler *c, _Py_SourceLocation loc, asdl_stmt_seq *stmts);
 
 /* Utility for a number of growing arrays used in the compiler */
 int _PyCompile_EnsureArrayLargeEnough(
@@ -72,8 +182,13 @@ int _PyCompile_EnsureArrayLargeEnough(
         int default_alloc,
         size_t item_size);
 
-int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj);
+int _PyCompiler_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj);
+
+PyCodeObject *_PyCompile_OptimizeAndAssemble(struct compiler *c, int addNone);
 
+Py_ssize_t _PyCompile_DictAddObj(PyObject *dict, PyObject *o);
+int _PyCompiler_Error(struct compiler *c, _Py_SourceLocation loc, const char *format, ...);
+int _PyCompiler_Warn(struct compiler *c, _Py_SourceLocation loc, const char *format, ...);
 
 // Export for '_opcode' extension module
 PyAPI_FUNC(PyObject*) _PyCompile_GetUnaryIntrinsicName(int index);

Include/internal/pycore_instruction_sequence.h

@@ -51,6 +51,11 @@ typedef struct {
     int id;
 } _PyJumpTargetLabel;
 
+#define NO_LABEL ((const _PyJumpTargetLabel){-1})
+
+#define SAME_JUMP_TARGET_LABEL(L1, L2) ((L1).id == (L2).id)
+#define IS_JUMP_TARGET_LABEL(L) (!SAME_JUMP_TARGET_LABEL((L), (NO_LABEL)))
+
 PyAPI_FUNC(PyObject*)_PyInstructionSequence_New(void);
 
 int _PyInstructionSequence_UseLabel(_PyInstructionSequence *seq, int lbl);

Makefile.pre.in

@@ -429,6 +429,7 @@ PYTHON_OBJS=	\
 		Python/brc.o \
 		Python/ceval.o \
 		Python/codecs.o \
+                Python/codegen.o \
 		Python/compile.o \
 		Python/context.o \
 		Python/critical_section.o \
@@ -1873,7 +1874,7 @@ regen-sre:
 		$(srcdir)/Modules/_sre/sre_constants.h \
 		$(srcdir)/Modules/_sre/sre_targets.h
 
-Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h
+Python/compile.o Python/codegen.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
 		$(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
@@ -2009,7 +2010,7 @@ regen-uop-metadata:
 	    $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(srcdir)/Python/bytecodes.c
 	$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new
 
-Python/compile.o Python/assemble.o Python/flowgraph.o Python/instruction_sequence.o: \
+Python/compile.o Python/codegen.o Python/assemble.o Python/flowgraph.o Python/instruction_sequence.o: \
                 $(srcdir)/Include/internal/pycore_compile.h \
                 $(srcdir)/Include/internal/pycore_flowgraph.h \
                 $(srcdir)/Include/internal/pycore_instruction_sequence.h \

PCbuild/_freeze_module.vcxproj

@@ -194,6 +194,7 @@
     <ClCompile Include="..\Python\bootstrap_hash.c" />
     <ClCompile Include="..\Python\ceval.c" />
     <ClCompile Include="..\Python\codecs.c" />
+    <ClCompile Include="..\Python\codegen.c" />
     <ClCompile Include="..\Python\compile.c" />
     <ClCompile Include="..\Python\context.c" />
     <ClCompile Include="..\Python\critical_section.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -97,6 +97,9 @@
     <ClCompile Include="..\Python\perf_jit_trampoline.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\codegen.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\compile.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -581,6 +581,7 @@
     <ClCompile Include="..\Python\brc.c" />
     <ClCompile Include="..\Python\ceval.c" />
     <ClCompile Include="..\Python\codecs.c" />
+    <ClCompile Include="..\Python\codegen.c" />
     <ClCompile Include="..\Python\compile.c" />
     <ClCompile Include="..\Python\context.c" />
     <ClCompile Include="..\Python\critical_section.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -1298,6 +1298,9 @@
     <ClCompile Include="..\Python\codecs.c">
       <Filter>Python</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\codegen.c">
+      <Filter>Python</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\compile.c">
       <Filter>Python</Filter>
     </ClCompile>

Python/assemble.c

@@ -435,13 +435,13 @@ assemble_emit(struct assembler *a, instr_sequence *instrs,
     RETURN_IF_ERROR(assemble_exception_table(a, instrs));
 
     RETURN_IF_ERROR(_PyBytes_Resize(&a->a_except_table, a->a_except_table_off));
-    RETURN_IF_ERROR(_PyCompile_ConstCacheMergeOne(const_cache, &a->a_except_table));
+    RETURN_IF_ERROR(_PyCompiler_ConstCacheMergeOne(const_cache, &a->a_except_table));
 
     RETURN_IF_ERROR(_PyBytes_Resize(&a->a_linetable, a->a_location_off));
-    RETURN_IF_ERROR(_PyCompile_ConstCacheMergeOne(const_cache, &a->a_linetable));
+    RETURN_IF_ERROR(_PyCompiler_ConstCacheMergeOne(const_cache, &a->a_linetable));
 
     RETURN_IF_ERROR(_PyBytes_Resize(&a->a_bytecode, a->a_offset * sizeof(_Py_CODEUNIT)));
-    RETURN_IF_ERROR(_PyCompile_ConstCacheMergeOne(const_cache, &a->a_bytecode));
+    RETURN_IF_ERROR(_PyCompiler_ConstCacheMergeOne(const_cache, &a->a_bytecode));
     return SUCCESS;
 }
 
@@ -553,15 +553,15 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_
     if (!names) {
         goto error;
     }
-    if (_PyCompile_ConstCacheMergeOne(const_cache, &names) < 0) {
+    if (_PyCompiler_ConstCacheMergeOne(const_cache, &names) < 0) {
         goto error;
     }
 
     consts = PyList_AsTuple(constslist); /* PyCode_New requires a tuple */
     if (consts == NULL) {
         goto error;
     }
-    if (_PyCompile_ConstCacheMergeOne(const_cache, &consts) < 0) {
+    if (_PyCompiler_ConstCacheMergeOne(const_cache, &consts) < 0) {
         goto error;
     }
 
@@ -615,7 +615,7 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_
         goto error;
     }
 
-    if (_PyCompile_ConstCacheMergeOne(const_cache, &localsplusnames) < 0) {
+    if (_PyCompiler_ConstCacheMergeOne(const_cache, &localsplusnames) < 0) {
         goto error;
     }
     con.localsplusnames = localsplusnames;