diff --git a/.gitattributes b/.gitattributes index 823e3e975a23a8..58cd96c0d3290d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -97,6 +97,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/generated_tracer_cases.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index c32bf4fd63cc8f..48b2e943e818c8 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,9 +57,9 @@ jobs: fail-fast: false matrix: target: - - i686-pc-windows-msvc/msvc - - x86_64-pc-windows-msvc/msvc - - aarch64-pc-windows-msvc/msvc +# - i686-pc-windows-msvc/msvc +# - x86_64-pc-windows-msvc/msvc +# - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -70,15 +70,15 @@ jobs: llvm: - 19 include: - - target: i686-pc-windows-msvc/msvc - architecture: Win32 - runner: windows-2022 - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-2022 - - target: aarch64-pc-windows-msvc/msvc - architecture: ARM64 - runner: windows-11-arm +# - target: i686-pc-windows-msvc/msvc +# architecture: Win32 +# runner: windows-2022 +# - target: x86_64-pc-windows-msvc/msvc +# architecture: x64 +# runner: windows-2022 +# - target: aarch64-pc-windows-msvc/msvc +# architecture: ARM64 +# runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-15-intel diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index badc97808c6132..ac1bcdf74ae417 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -757,6 +757,23 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +typedef struct _PyJitTracerState { + bool dependencies_still_valid; + int code_max_size; + int code_curr_size; + int initial_stack_depth; + int initial_chain_depth; + _PyUOpInstruction *code_buffer; + _Py_CODEUNIT *insert_exec_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *last_specialized_instr; + PyCodeObject *initial_code; // Strong + PyFunctionObject *initial_func; // Strong + struct _PyExitData *previous_exit; + _PyInterpreterFrame *current_frame; + _PyBloomFilter dependencies; +} _PyJitTracerState; + /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. @@ -932,9 +949,9 @@ struct _is { struct types_state types; struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; + _PyJitTracerState jit_state; bool jit; bool compiling; - struct _PyUOpInstruction *jit_uop_buffer; struct _PyExecutorObject *executor_list_head; struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bd6b84ec7fd908..e8e0c59874a546 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1406,6 +1406,8 @@ _PyOpcode_macro_expansion[256] = { [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, + [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, @@ -1769,6 +1771,37 @@ const uint8_t _PyOpcode_Caches[256] = { }; #endif +extern const uint8_t _PyOpcode_NeedsGuardIp[256]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyOpcode_NeedsGuardIp[256] = { + [INTERPRETER_EXIT] = 1, + [RETURN_VALUE] = 1, + [YIELD_VALUE] = 1, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = 1, + [RETURN_GENERATOR] = 1, + [BINARY_OP_SUBSCR_GETITEM] = 1, + [INSTRUMENTED_RETURN_VALUE] = 1, + [SEND] = 1, + [SEND_GEN] = 1, + [INSTRUMENTED_YIELD_VALUE] = 1, + [LOAD_ATTR_PROPERTY] = 1, + [FOR_ITER_GEN] = 1, + [CALL] = 1, + [INSTRUMENTED_CALL] = 1, + [CALL_PY_GENERAL] = 1, + [CALL_BOUND_METHOD_GENERAL] = 1, + [CALL_BOUND_METHOD_EXACT_ARGS] = 1, + [CALL_PY_EXACT_ARGS] = 1, + [CALL_ALLOC_AND_ENTER_INIT] = 1, + [CALL_KW_PY] = 1, + [CALL_KW_BOUND_METHOD] = 1, + [CALL_KW] = 1, + [INSTRUMENTED_CALL_KW] = 1, + [CALL_FUNCTION_EX] = 1, + [INSTRUMENTED_CALL_FUNCTION_EX] = 1, +}; +#endif + extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 685c39dcd65fb9..9c3aaf1edaf550 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -21,14 +21,6 @@ typedef struct _PyExecutorLinkListNode { } _PyExecutorLinkListNode; -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 - -typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - typedef struct { uint8_t opcode; uint8_t oparg; @@ -95,7 +87,8 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #define TRACE_STACK_SIZE 5 -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, +int _Py_uop_analyze_and_optimize( + PyFunctionObject *initial_func, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); @@ -336,7 +329,7 @@ extern int _Py_uop_frame_pop(JitOptContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, PyThreadState *tstate); static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) { @@ -353,7 +346,8 @@ static inline int is_terminator(const _PyUOpInstruction *uop) int opcode = uop->opcode; return ( opcode == _EXIT_TRACE || - opcode == _JUMP_TO_TOP + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT ); } @@ -364,6 +358,27 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif +int +_PyJit_translate_single_bytecode_to_trace( + PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *this_instr, + _Py_CODEUNIT *next_instr, + PyCodeObject *code, + PyFunctionObject *func, + int old_stack_level, + int opcode, + int oparg, + int jump_taken); + +void +_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, + _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); + +void _PyJit_FinalizeTracing(PyThreadState *tstate); + +void _PyJit_Tracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 4abefd3b95d21a..bccc0b00a4d210 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -35,10 +35,18 @@ typedef struct _PyUOpInstruction{ #endif } _PyUOpInstruction; -// This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 1200 +// This is the length of the trace we translate initially. +#define UOP_MAX_TRACE_LENGTH 1500 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ff1d75c0cb1938..9f4ae71a313aa0 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -108,74 +108,76 @@ extern "C" { #define _DO_CALL 374 #define _DO_CALL_FUNCTION_EX 375 #define _DO_CALL_KW 376 +#define _DYNAMIC_EXIT 377 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 377 +#define _ERROR_POP_N 378 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 378 -#define _EXPAND_METHOD_KW 379 -#define _FATAL_ERROR 380 +#define _EXPAND_METHOD 379 +#define _EXPAND_METHOD_KW 380 +#define _FATAL_ERROR 381 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 381 -#define _FOR_ITER_GEN_FRAME 382 -#define _FOR_ITER_TIER_TWO 383 +#define _FOR_ITER 382 +#define _FOR_ITER_GEN_FRAME 383 +#define _FOR_ITER_TIER_TWO 384 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 384 -#define _GUARD_CALLABLE_ISINSTANCE 385 -#define _GUARD_CALLABLE_LEN 386 -#define _GUARD_CALLABLE_LIST_APPEND 387 -#define _GUARD_CALLABLE_STR_1 388 -#define _GUARD_CALLABLE_TUPLE_1 389 -#define _GUARD_CALLABLE_TYPE_1 390 -#define _GUARD_DORV_NO_DICT 391 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 392 -#define _GUARD_GLOBALS_VERSION 393 -#define _GUARD_IS_FALSE_POP 394 -#define _GUARD_IS_NONE_POP 395 -#define _GUARD_IS_NOT_NONE_POP 396 -#define _GUARD_IS_TRUE_POP 397 -#define _GUARD_KEYS_VERSION 398 -#define _GUARD_NOS_DICT 399 -#define _GUARD_NOS_FLOAT 400 -#define _GUARD_NOS_INT 401 -#define _GUARD_NOS_LIST 402 -#define _GUARD_NOS_NOT_NULL 403 -#define _GUARD_NOS_NULL 404 -#define _GUARD_NOS_OVERFLOWED 405 -#define _GUARD_NOS_TUPLE 406 -#define _GUARD_NOS_UNICODE 407 -#define _GUARD_NOT_EXHAUSTED_LIST 408 -#define _GUARD_NOT_EXHAUSTED_RANGE 409 -#define _GUARD_NOT_EXHAUSTED_TUPLE 410 -#define _GUARD_THIRD_NULL 411 -#define _GUARD_TOS_ANY_SET 412 -#define _GUARD_TOS_DICT 413 -#define _GUARD_TOS_FLOAT 414 -#define _GUARD_TOS_INT 415 -#define _GUARD_TOS_LIST 416 -#define _GUARD_TOS_OVERFLOWED 417 -#define _GUARD_TOS_SLICE 418 -#define _GUARD_TOS_TUPLE 419 -#define _GUARD_TOS_UNICODE 420 -#define _GUARD_TYPE_VERSION 421 -#define _GUARD_TYPE_VERSION_AND_LOCK 422 -#define _HANDLE_PENDING_AND_DEOPT 423 +#define _GUARD_BINARY_OP_EXTEND 385 +#define _GUARD_CALLABLE_ISINSTANCE 386 +#define _GUARD_CALLABLE_LEN 387 +#define _GUARD_CALLABLE_LIST_APPEND 388 +#define _GUARD_CALLABLE_STR_1 389 +#define _GUARD_CALLABLE_TUPLE_1 390 +#define _GUARD_CALLABLE_TYPE_1 391 +#define _GUARD_DORV_NO_DICT 392 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 393 +#define _GUARD_GLOBALS_VERSION 394 +#define _GUARD_IP 395 +#define _GUARD_IS_FALSE_POP 396 +#define _GUARD_IS_NONE_POP 397 +#define _GUARD_IS_NOT_NONE_POP 398 +#define _GUARD_IS_TRUE_POP 399 +#define _GUARD_KEYS_VERSION 400 +#define _GUARD_NOS_DICT 401 +#define _GUARD_NOS_FLOAT 402 +#define _GUARD_NOS_INT 403 +#define _GUARD_NOS_LIST 404 +#define _GUARD_NOS_NOT_NULL 405 +#define _GUARD_NOS_NULL 406 +#define _GUARD_NOS_OVERFLOWED 407 +#define _GUARD_NOS_TUPLE 408 +#define _GUARD_NOS_UNICODE 409 +#define _GUARD_NOT_EXHAUSTED_LIST 410 +#define _GUARD_NOT_EXHAUSTED_RANGE 411 +#define _GUARD_NOT_EXHAUSTED_TUPLE 412 +#define _GUARD_THIRD_NULL 413 +#define _GUARD_TOS_ANY_SET 414 +#define _GUARD_TOS_DICT 415 +#define _GUARD_TOS_FLOAT 416 +#define _GUARD_TOS_INT 417 +#define _GUARD_TOS_LIST 418 +#define _GUARD_TOS_OVERFLOWED 419 +#define _GUARD_TOS_SLICE 420 +#define _GUARD_TOS_TUPLE 421 +#define _GUARD_TOS_UNICODE 422 +#define _GUARD_TYPE_VERSION 423 +#define _GUARD_TYPE_VERSION_AND_LOCK 424 +#define _HANDLE_PENDING_AND_DEOPT 425 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 424 -#define _INIT_CALL_PY_EXACT_ARGS 425 -#define _INIT_CALL_PY_EXACT_ARGS_0 426 -#define _INIT_CALL_PY_EXACT_ARGS_1 427 -#define _INIT_CALL_PY_EXACT_ARGS_2 428 -#define _INIT_CALL_PY_EXACT_ARGS_3 429 -#define _INIT_CALL_PY_EXACT_ARGS_4 430 -#define _INSERT_NULL 431 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 426 +#define _INIT_CALL_PY_EXACT_ARGS 427 +#define _INIT_CALL_PY_EXACT_ARGS_0 428 +#define _INIT_CALL_PY_EXACT_ARGS_1 429 +#define _INIT_CALL_PY_EXACT_ARGS_2 430 +#define _INIT_CALL_PY_EXACT_ARGS_3 431 +#define _INIT_CALL_PY_EXACT_ARGS_4 432 +#define _INSERT_NULL 433 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -185,177 +187,178 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 432 +#define _IS_NONE 434 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 433 -#define _ITER_CHECK_RANGE 434 -#define _ITER_CHECK_TUPLE 435 -#define _ITER_JUMP_LIST 436 -#define _ITER_JUMP_RANGE 437 -#define _ITER_JUMP_TUPLE 438 -#define _ITER_NEXT_LIST 439 -#define _ITER_NEXT_LIST_TIER_TWO 440 -#define _ITER_NEXT_RANGE 441 -#define _ITER_NEXT_TUPLE 442 -#define _JUMP_TO_TOP 443 +#define _ITER_CHECK_LIST 435 +#define _ITER_CHECK_RANGE 436 +#define _ITER_CHECK_TUPLE 437 +#define _ITER_JUMP_LIST 438 +#define _ITER_JUMP_RANGE 439 +#define _ITER_JUMP_TUPLE 440 +#define _ITER_NEXT_LIST 441 +#define _ITER_NEXT_LIST_TIER_TWO 442 +#define _ITER_NEXT_RANGE 443 +#define _ITER_NEXT_TUPLE 444 +#define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT +#define _JUMP_TO_TOP 445 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 444 -#define _LOAD_ATTR_CLASS 445 +#define _LOAD_ATTR 446 +#define _LOAD_ATTR_CLASS 447 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 446 -#define _LOAD_ATTR_METHOD_LAZY_DICT 447 -#define _LOAD_ATTR_METHOD_NO_DICT 448 -#define _LOAD_ATTR_METHOD_WITH_VALUES 449 -#define _LOAD_ATTR_MODULE 450 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 451 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 452 -#define _LOAD_ATTR_PROPERTY_FRAME 453 -#define _LOAD_ATTR_SLOT 454 -#define _LOAD_ATTR_WITH_HINT 455 +#define _LOAD_ATTR_INSTANCE_VALUE 448 +#define _LOAD_ATTR_METHOD_LAZY_DICT 449 +#define _LOAD_ATTR_METHOD_NO_DICT 450 +#define _LOAD_ATTR_METHOD_WITH_VALUES 451 +#define _LOAD_ATTR_MODULE 452 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 453 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 454 +#define _LOAD_ATTR_PROPERTY_FRAME 455 +#define _LOAD_ATTR_SLOT 456 +#define _LOAD_ATTR_WITH_HINT 457 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 456 +#define _LOAD_BYTECODE 458 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 457 -#define _LOAD_CONST_INLINE_BORROW 458 -#define _LOAD_CONST_UNDER_INLINE 459 -#define _LOAD_CONST_UNDER_INLINE_BORROW 460 +#define _LOAD_CONST_INLINE 459 +#define _LOAD_CONST_INLINE_BORROW 460 +#define _LOAD_CONST_UNDER_INLINE 461 +#define _LOAD_CONST_UNDER_INLINE_BORROW 462 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 461 -#define _LOAD_FAST_0 462 -#define _LOAD_FAST_1 463 -#define _LOAD_FAST_2 464 -#define _LOAD_FAST_3 465 -#define _LOAD_FAST_4 466 -#define _LOAD_FAST_5 467 -#define _LOAD_FAST_6 468 -#define _LOAD_FAST_7 469 +#define _LOAD_FAST 463 +#define _LOAD_FAST_0 464 +#define _LOAD_FAST_1 465 +#define _LOAD_FAST_2 466 +#define _LOAD_FAST_3 467 +#define _LOAD_FAST_4 468 +#define _LOAD_FAST_5 469 +#define _LOAD_FAST_6 470 +#define _LOAD_FAST_7 471 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 470 -#define _LOAD_FAST_BORROW_0 471 -#define _LOAD_FAST_BORROW_1 472 -#define _LOAD_FAST_BORROW_2 473 -#define _LOAD_FAST_BORROW_3 474 -#define _LOAD_FAST_BORROW_4 475 -#define _LOAD_FAST_BORROW_5 476 -#define _LOAD_FAST_BORROW_6 477 -#define _LOAD_FAST_BORROW_7 478 +#define _LOAD_FAST_BORROW 472 +#define _LOAD_FAST_BORROW_0 473 +#define _LOAD_FAST_BORROW_1 474 +#define _LOAD_FAST_BORROW_2 475 +#define _LOAD_FAST_BORROW_3 476 +#define _LOAD_FAST_BORROW_4 477 +#define _LOAD_FAST_BORROW_5 478 +#define _LOAD_FAST_BORROW_6 479 +#define _LOAD_FAST_BORROW_7 480 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 479 -#define _LOAD_GLOBAL_BUILTINS 480 -#define _LOAD_GLOBAL_MODULE 481 +#define _LOAD_GLOBAL 481 +#define _LOAD_GLOBAL_BUILTINS 482 +#define _LOAD_GLOBAL_MODULE 483 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 482 -#define _LOAD_SMALL_INT_0 483 -#define _LOAD_SMALL_INT_1 484 -#define _LOAD_SMALL_INT_2 485 -#define _LOAD_SMALL_INT_3 486 -#define _LOAD_SPECIAL 487 +#define _LOAD_SMALL_INT 484 +#define _LOAD_SMALL_INT_0 485 +#define _LOAD_SMALL_INT_1 486 +#define _LOAD_SMALL_INT_2 487 +#define _LOAD_SMALL_INT_3 488 +#define _LOAD_SPECIAL 489 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 488 +#define _MAKE_CALLARGS_A_TUPLE 490 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 489 +#define _MAKE_WARM 491 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 490 -#define _MAYBE_EXPAND_METHOD_KW 491 -#define _MONITOR_CALL 492 -#define _MONITOR_CALL_KW 493 -#define _MONITOR_JUMP_BACKWARD 494 -#define _MONITOR_RESUME 495 +#define _MAYBE_EXPAND_METHOD 492 +#define _MAYBE_EXPAND_METHOD_KW 493 +#define _MONITOR_CALL 494 +#define _MONITOR_CALL_KW 495 +#define _MONITOR_JUMP_BACKWARD 496 +#define _MONITOR_RESUME 497 #define _NOP NOP -#define _POP_CALL 496 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 497 -#define _POP_CALL_ONE 498 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 499 -#define _POP_CALL_TWO 500 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 501 +#define _POP_CALL 498 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 499 +#define _POP_CALL_ONE 500 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 501 +#define _POP_CALL_TWO 502 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 503 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 502 -#define _POP_JUMP_IF_TRUE 503 +#define _POP_JUMP_IF_FALSE 504 +#define _POP_JUMP_IF_TRUE 505 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 504 -#define _POP_TOP_INT 505 -#define _POP_TOP_LOAD_CONST_INLINE 506 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 507 -#define _POP_TOP_NOP 508 -#define _POP_TOP_UNICODE 509 -#define _POP_TWO 510 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 511 +#define _POP_TOP_FLOAT 506 +#define _POP_TOP_INT 507 +#define _POP_TOP_LOAD_CONST_INLINE 508 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 509 +#define _POP_TOP_NOP 510 +#define _POP_TOP_UNICODE 511 +#define _POP_TWO 512 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 513 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 512 +#define _PUSH_FRAME 514 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 513 -#define _PY_FRAME_GENERAL 514 -#define _PY_FRAME_KW 515 -#define _QUICKEN_RESUME 516 -#define _REPLACE_WITH_TRUE 517 +#define _PUSH_NULL_CONDITIONAL 515 +#define _PY_FRAME_GENERAL 516 +#define _PY_FRAME_KW 517 +#define _QUICKEN_RESUME 518 +#define _REPLACE_WITH_TRUE 519 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 518 -#define _SEND 519 -#define _SEND_GEN_FRAME 520 +#define _SAVE_RETURN_OFFSET 520 +#define _SEND 521 +#define _SEND_GEN_FRAME 522 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 521 -#define _STORE_ATTR 522 -#define _STORE_ATTR_INSTANCE_VALUE 523 -#define _STORE_ATTR_SLOT 524 -#define _STORE_ATTR_WITH_HINT 525 +#define _START_EXECUTOR 523 +#define _STORE_ATTR 524 +#define _STORE_ATTR_INSTANCE_VALUE 525 +#define _STORE_ATTR_SLOT 526 +#define _STORE_ATTR_WITH_HINT 527 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 526 -#define _STORE_FAST_0 527 -#define _STORE_FAST_1 528 -#define _STORE_FAST_2 529 -#define _STORE_FAST_3 530 -#define _STORE_FAST_4 531 -#define _STORE_FAST_5 532 -#define _STORE_FAST_6 533 -#define _STORE_FAST_7 534 +#define _STORE_FAST 528 +#define _STORE_FAST_0 529 +#define _STORE_FAST_1 530 +#define _STORE_FAST_2 531 +#define _STORE_FAST_3 532 +#define _STORE_FAST_4 533 +#define _STORE_FAST_5 534 +#define _STORE_FAST_6 535 +#define _STORE_FAST_7 536 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 535 -#define _STORE_SUBSCR 536 -#define _STORE_SUBSCR_DICT 537 -#define _STORE_SUBSCR_LIST_INT 538 -#define _SWAP 539 -#define _SWAP_2 540 -#define _SWAP_3 541 -#define _TIER2_RESUME_CHECK 542 -#define _TO_BOOL 543 +#define _STORE_SLICE 537 +#define _STORE_SUBSCR 538 +#define _STORE_SUBSCR_DICT 539 +#define _STORE_SUBSCR_LIST_INT 540 +#define _SWAP 541 +#define _SWAP_2 542 +#define _SWAP_3 543 +#define _TIER2_RESUME_CHECK 544 +#define _TO_BOOL 545 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 544 +#define _TO_BOOL_LIST 546 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 545 +#define _TO_BOOL_STR 547 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 546 -#define _UNPACK_SEQUENCE_LIST 547 -#define _UNPACK_SEQUENCE_TUPLE 548 -#define _UNPACK_SEQUENCE_TWO_TUPLE 549 +#define _UNPACK_SEQUENCE 548 +#define _UNPACK_SEQUENCE_LIST 549 +#define _UNPACK_SEQUENCE_TUPLE 550 +#define _UNPACK_SEQUENCE_TWO_TUPLE 551 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 549 +#define MAX_UOP_ID 551 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1248771996943b..82a0fa92652524 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -213,7 +213,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -336,7 +336,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = HAS_ESCAPES_FLAG, + [_COLD_EXIT] = 0, + [_GUARD_IP] = HAS_EXIT_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -443,6 +445,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", @@ -471,6 +474,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_IP] = "_GUARD_IP", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", @@ -1305,6 +1309,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _COLD_EXIT: return 0; + case _GUARD_IP: + return 0; + case _DYNAMIC_EXIT: + return 0; default: return -1; } diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f121f27174875e..9adbe75b205945 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -139,6 +139,7 @@ def testfunc(x): self.assertIn("_JUMP_TO_TOP", uops) self.assertIn("_LOAD_FAST_BORROW_0", uops) + @unittest.skip("gh-139109 WIP") def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" ns = {} @@ -422,32 +423,6 @@ def testfunc(n, m): uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) - def test_confidence_score(self): - def testfunc(n): - bits = 0 - for i in range(n): - if i & 0x01: - bits += 1 - if i & 0x02: - bits += 1 - if i&0x04: - bits += 1 - if i&0x08: - bits += 1 - if i&0x10: - bits += 1 - return bits - - x = testfunc(TIER2_THRESHOLD * 2) - - self.assertEqual(x, TIER2_THRESHOLD * 5) - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - ops = list(iter_opnames(ex)) - #Since branch is 50/50 the trace could go either way. - count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") - self.assertLessEqual(count, 2) - @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @@ -847,38 +822,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_unicode_count), 1) self.assertIn("_COMPARE_OP_STR", uops) - def test_type_inconsistency(self): - ns = {} - src = textwrap.dedent(""" - def testfunc(n): - for i in range(n): - x = _test_global + _test_global - """) - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - self.assertNotIn("_BINARY_OP_ADD_INT", uops) - self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) - # Try again, but between the runs, set the global to a float. - # This should result in no executor the second time. - ns = {} - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_sequential(self): def dummy12(x): return x - 1 @@ -907,6 +851,7 @@ def testfunc(n): largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_nested(self): def dummy12(x): return x + 3 @@ -937,6 +882,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls(self): def dummy12(x): return x + 3 @@ -972,6 +918,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls_different_order(self): # same as `several_calls` but with top-level calls reversed def dummy12(x): @@ -1008,6 +955,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_complex(self): def dummy0(x): return x @@ -1057,6 +1005,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_large_framesize(self): # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is # actually doing its job. Note that the resulting trace hits @@ -1118,6 +1067,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_recursion(self): def dummy15(x): while x > 0: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1198c6d35113c8..c5e50dffb13938 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2252,9 +2252,10 @@ def frame_2_jit(expected: bool) -> None: def frame_3_jit() -> None: # JITs just before the last loop: - for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + # 1 extra iteration for tracing. + for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i == _testinternalcapi.TIER2_THRESHOLD + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected diff --git a/Makefile.pre.in b/Makefile.pre.in index 19423c11545c19..2db96b74475705 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2141,7 +2141,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ regen-generated-cases regen-executor-cases regen-optimizer-cases \ - regen-opcode-metadata regen-uop-metadata + regen-opcode-metadata regen-uop-metadata regen-tracer-cases .PHONY: regen-opcode-ids regen-opcode-ids: @@ -2187,6 +2187,13 @@ regen-optimizer-cases: $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new +.PHONY: regen-tracer-cases +regen-tracer-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tracer_generator.py \ + -o $(srcdir)/Python/generated_tracer_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/generated_tracer_cases.c.h $(srcdir)/Python/generated_tracer_cases.c.h.new + + .PHONY: regen-opcode-metadata regen-opcode-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst new file mode 100644 index 00000000000000..01e2efa4df4590 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst @@ -0,0 +1 @@ +A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Brandt Bucher, Mark Shannon and Ken Jin. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c2647d405e25bc..9cb6695173b840 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2578,7 +2578,8 @@ module_exec(PyObject *module) } if (PyModule_Add(module, "TIER2_THRESHOLD", - PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 1)) < 0) { + // + 1 more due to one loop spent on tracing. + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 2)) < 0) { return 1; } diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d264a6e346f95..3a13cfaee2a5fc 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2415,6 +2415,7 @@ code_dealloc(PyObject *self) PyMem_Free(co_extra); } #ifdef _Py_TIER2 + _PyJit_Tracer_InvalidateDependency(tstate, self); if (co->co_executors != NULL) { clear_executors(co); } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 0cae3703d1d0c6..abed4547ffd053 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,6 +17,8 @@ #include "frameobject.h" // PyFrameLocalsProxyObject #include "opcode.h" // EXTENDED_ARG +#include "../Include/pytypedefs.h" +#include "pycore_optimizer.h" #include "clinic/frameobject.c.h" @@ -260,7 +262,10 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return -1; } +#if _Py_TIER2 _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), co); +#endif _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); _PyStackRef oldvalue = fast[i]; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 43198aaf8a7048..fcd79c7e4f4ea1 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -11,7 +11,7 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() - +#include "pycore_optimizer.h" // _PyJit_Tracer_InvalidateDependency static const char * func_event_name(PyFunction_WatchEvent event) { @@ -1151,6 +1151,9 @@ func_dealloc(PyObject *self) if (_PyObject_ResurrectEnd(self)) { return; } +#if _Py_TIER2 + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), self); +#endif _PyObject_GC_UNTRACK(op); FT_CLEAR_WEAKREFS(self, op->func_weakreflist); (void)func_clear((PyObject*)op); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f9f14322df0a5e..388da83dbc24bf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -45,6 +45,8 @@ #define USE_COMPUTED_GOTOS 0 #include "ceval_macros.h" +#include "../Include/internal/pycore_code.h" +#include "../Include/internal/pycore_stackref.h" /* Flow control macros */ @@ -1249,7 +1251,12 @@ dummy_func( frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); RELOAD_STACK(); + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif res = temp; LLTRACE_RESUME_FRAME(); } @@ -1371,6 +1378,7 @@ dummy_func( if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); + RECORD_DYNAMIC_JUMP_TAKEN(); } else { PyStackRef_CLOSE(v); @@ -1436,7 +1444,12 @@ dummy_func( _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif RELOAD_STACK(); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif + #if TIER_TWO + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); } @@ -2954,26 +2967,32 @@ dummy_func( tier1 op(_JIT, (--)) { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - while (oparg > 255) { - oparg >>= 8; - start--; - } - _PyExecutorObject *executor; - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - ERROR_IF(optimized < 0); + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + if (tstate->interp->jit_state.code_buffer == NULL) { + tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + if (tstate->interp->jit_state.code_buffer == NULL) { + // Don't error, just go to next instruction. + DISPATCH(); + } } - else { - this_instr[1].counter = initial_jump_backoff_counter(); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (!_is_sys_tracing) { + /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */ + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + ENTER_TRACING(); } + int _jump_taken = false; + PyCodeObject *old_code = _PyFrame_GetCode(frame); + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + int _old_stack_level = 0; + TRACING_DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -3038,6 +3057,9 @@ dummy_func( } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; + #if TRACING_JIT + RECORD_TRACE_NO_DISPATCH(); + #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -3079,7 +3101,7 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - tier1 inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + replaced inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3219,19 +3241,22 @@ dummy_func( } // Jump forward by oparg and skip the following END_FOR JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } next = item; } op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_FOR_ITER); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { ERROR_NO_POP(); } /* iterator ended normally */ - /* The translator sets the deopt target just past the matching END_FOR */ + /* This just sets the IP to what it expects */ + TIER2_STORE_IP(oparg + 1); EXIT_IF(true); } next = item; @@ -3278,6 +3303,7 @@ dummy_func( null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } #endif @@ -3355,6 +3381,7 @@ dummy_func( null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } } @@ -3399,6 +3426,7 @@ dummy_func( if (r->len <= 0) { // Jump over END_FOR instruction. JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } } @@ -4979,7 +5007,12 @@ dummy_func( _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif RELOAD_STACK(); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); @@ -5374,19 +5407,22 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); } tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { int err = _Py_HandlePending(tstate); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); } tier2 op(_ERROR_POP_N, (target/2 --)) { assert(oparg == 0); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; + // gh-140104: The exception handler expects frame->instr_ptr to be pointing to next_instr, not this_instr! + frame->instr_ptr = next_instr; SYNC_SP(); - GOTO_TIER_ONE(NULL); + GOTO_TIER_ONE(NULL, 0); } /* Progress is guaranteed if we DEOPT on the eval breaker, because @@ -5408,7 +5444,7 @@ dummy_func( _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); + GOTO_TIER_ONE(target, 0); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -5420,18 +5456,46 @@ dummy_func( _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } + _PyJit_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(target, 1); } assert(tstate->jit_exit == exit); exit->executor = executor; TIER2_TO_TIER2(exit->executor); } + tier2 op(_GUARD_IP, (ip/4 --)) { + EXIT_IF(frame->instr_ptr != (_Py_CODEUNIT *)ip); + } + + // Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors + // from a single exit! + tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr; +#if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + } +#endif + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; + tstate->jit_exit = NULL; + TIER2_TO_TIER2(executor); + } + else { + GOTO_TIER_ONE(target, 0); + } + } + label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/ceval.c b/Python/ceval.c index f48f412fab8335..3ca8378294998c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -984,6 +984,30 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +#if _Py_TIER2 +// 1 for trace full, 0 for successful write. +static inline int +add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, int old_stack_level, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) +{ + assert(frame != NULL); + assert(tstate->interp->jit_state.code_curr_size < UOP_MAX_TRACE_LENGTH); + return !_PyJit_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, old_stack_level, opcode, oparg, jump_taken); +} + + +// 0 for success, -1 for error. +static int +bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + int err = 0; + if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { + err = _PyOptimizer_Optimize(frame, tstate); + } + _PyJit_FinalizeTracing(tstate); + return err; +} +#endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ @@ -1001,6 +1025,7 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #if _Py_TAIL_CALL_INTERP #include "opcode_targets.h" #include "generated_cases.c.h" +#include "generated_tracer_cases.c.h" #endif #if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__) @@ -1057,11 +1082,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; _PyStackRef *stack_pointer; entry.stack[0] = PyStackRef_NULL; -#ifdef Py_STACKREF_DEBUG - entry.frame.f_funcobj = PyStackRef_None; -#elif defined(Py_DEBUG) + entry.frame.f_funcobj = PyStackRef_NULL; +#if defined(Py_DEBUG) /* Set these to invalid but identifiable values for debugging. */ - entry.frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; entry.frame.f_locals = (PyObject*)0xaaa1; entry.frame.frame_obj = (PyFrameObject*)0xaaa2; entry.frame.f_globals = (PyObject*)0xaaa3; @@ -1113,9 +1136,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0); # endif #else goto error; @@ -1124,13 +1147,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0); # endif #else goto start_frame; # include "generated_cases.c.h" + #include "generated_tracer_cases.c.h" #endif @@ -1197,8 +1221,28 @@ _PyTier2Interpreter( for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG - if (frame->lltrace >= 3) { - dump_stack(frame, stack_pointer); + if (frame->lltrace >= 4) { + if (next_uop->opcode != _YIELD_VALUE && + next_uop->opcode != _FOR_ITER_GEN_FRAME && + next_uop->opcode != _PUSH_FRAME && + next_uop->opcode != _PY_FRAME_KW && + next_uop->opcode != _SAVE_RETURN_OFFSET && + next_uop->opcode != _SAVE_RETURN_OFFSET) { + if (next_uop->opcode != _START_EXECUTOR) { + if (next_uop->format == UOP_FORMAT_TARGET) { + _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; + printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + } + else if (next_uop->format == UOP_FORMAT_JUMP) { + _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; + if (aim_uop->format == UOP_FORMAT_TARGET) { + _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; + printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + } + } + } + dump_stack(frame, stack_pointer); + } if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 4ed03b7fb01bdf..e19a614f28dec6 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -84,7 +84,12 @@ # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) +# define TRACING_TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_TRACING_##op(TAIL_CALL_PARAMS) + # define DISPATCH_GOTO() \ do { \ Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ @@ -106,8 +111,13 @@ # endif # define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS +# define DISPATCH_TABLE_VAR opcode_targets +# define DISPATCH_TABLE opcode_targets_table +# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table # define TARGET(op) TARGET_##op: +# define TRACING_TARGET(op) TARGET_TRACING_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] + # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: @@ -119,6 +129,38 @@ # define LABEL(name) name: #endif +#define TRACING_JUMP_TO_LABEL(label) \ + RECORD_DYNAMIC_JUMP_TAKEN() \ + RECORD_TRACE_NO_DISPATCH() \ + assert(!IS_JIT_TRACING()); \ + JUMP_TO_LABEL(label); + +#if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +// tstate->interp->jit_state.last_specialized_instr != this_instr is required to not get stuck in infinite +// specialization loops due to specialization failure. +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.last_specialized_instr != this_instr) +# define ENTER_TRACING() \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define LEAVE_TRACING() \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; +# define RECORD_TRACE_NO_DISPATCH() do { \ + int err = 0; \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + /* We need to check once more here in case it swapped out halfway. */ \ + if (IS_JIT_TRACING()) { \ + int full = add_to_code_trace(tstate, frame, old_code, old_func, _old_stack_level, this_instr, next_instr, opcode, oparg, _jump_taken); \ + if (full) { \ + LEAVE_TRACING(); \ + err = bail_tracing_and_jit(tstate, frame); \ + } \ + } \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (err < 0) { JUMP_TO_LABEL(error); } \ + } while (0); +#endif + + /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef Py_DEBUG #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ @@ -162,6 +204,14 @@ do { \ DISPATCH_GOTO(); \ } +#define TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() \ +{ \ + tstate->interp->jit_state.last_specialized_instr = this_instr; \ + opcode = next_instr->op.code; \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ +} + #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ @@ -172,6 +222,21 @@ do { \ JUMP_TO_LABEL(start_frame); \ } while (0) +#define TRACING_DISPATCH_INLINED(NEW_FRAME) \ + tstate->interp->jit_state.last_specialized_instr = this_instr; \ + RECORD_TRACE_NO_DISPATCH(); \ + DISPATCH_INLINED(NEW_FRAME); + +#define TRACING_DISPATCH() \ + { \ + tstate->interp->jit_state.last_specialized_instr = this_instr; \ + assert(frame->stackpointer == NULL); \ + RECORD_TRACE_NO_DISPATCH(); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ + } + /* Tuple access macros */ #ifndef Py_DEBUG @@ -201,6 +266,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { * and skipped instructions. */ #define JUMPBY(x) (next_instr += (x)) +#define TIER2_STORE_IP(x) (frame->instr_ptr += (x)) #define SKIP_OVER(x) (next_instr += (x)) #define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame))) @@ -271,9 +337,14 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ +#if _Py_TIER2 +// Force re-specialization when tracing a side exit to get good side exits. +#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ + backoff_counter_triggers(forge_backoff_counter((COUNTER))) || IS_JIT_TRACING_MAKING_PROGRESS() +#else #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ backoff_counter_triggers(forge_backoff_counter((COUNTER))) - +#endif #define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ do { \ (COUNTER) = advance_backoff_counter((COUNTER)); \ @@ -297,6 +368,8 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define RECORD_BRANCH_TAKEN(bitset, flag) #endif +#define RECORD_DYNAMIC_JUMP_TAKEN() _jump_taken = 1; + #define UNBOUNDLOCAL_ERROR_MSG \ "cannot access local variable '%s' where it is not associated with a value" #define UNBOUNDFREE_ERROR_MSG \ @@ -357,10 +430,17 @@ do { \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ if (next_instr == NULL) { \ next_instr = frame->instr_ptr; \ JUMP_TO_LABEL(error); \ } \ + if (keep_tracing_bit) { \ + assert(next_instr->op.code != ENTER_EXECUTOR); \ + assert(tstate->interp->jit_state.code_curr_size == 2); \ + ENTER_TRACING(); \ + } \ DISPATCH(); \ } while (0) @@ -371,13 +451,13 @@ do { \ goto tier2_start; \ } while (0) -#define GOTO_TIER_ONE(TARGET) \ +#define GOTO_TIER_ONE(TARGET, SHOULD_CONTINUE_TRACING) \ do \ { \ tstate->current_executor = NULL; \ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET)) | SHOULD_CONTINUE_TRACING); \ } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0e4d86463761a0..72927d8378c3bd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1930,7 +1930,12 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -2096,7 +2101,12 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif + #if TIER_TWO + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; @@ -4189,6 +4199,8 @@ break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 because it is replaced */ + case _GET_LEN: { _PyStackRef obj; _PyStackRef len; @@ -4383,8 +4395,10 @@ _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; + oparg = CURRENT_OPARG(); null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_FOR_ITER); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4392,6 +4406,7 @@ if (PyStackRef_IsError(item)) { JUMP_TO_ERROR(); } + TIER2_STORE_IP(oparg + 1); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -6748,7 +6763,12 @@ _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); @@ -7422,7 +7442,7 @@ } case _DEOPT: { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); break; } @@ -7430,7 +7450,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); break; } @@ -7438,8 +7458,10 @@ oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); assert(oparg == 0); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; - GOTO_TIER_ONE(NULL); + _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; + frame->instr_ptr = next_instr; + GOTO_TIER_ONE(NULL, 0); break; } @@ -7467,7 +7489,7 @@ _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); + GOTO_TIER_ONE(target, 0); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -7476,19 +7498,12 @@ Py_INCREF(executor); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - stack_pointer = _PyFrame_GetStackPointer(frame); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } + _PyJit_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(target, 1); } assert(tstate->jit_exit == exit); exit->executor = executor; @@ -7496,4 +7511,42 @@ break; } + case _GUARD_IP: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _DYNAMIC_EXIT: { + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; + tstate->jit_exit = NULL; + TIER2_TO_TIER2(executor); + } + else { + GOTO_TIER_ONE(target, 0); + } + break; + } + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 79328a7b725613..4fb61a06cdfe04 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5493,6 +5493,9 @@ } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; + #if TRACING_JIT + RECORD_TRACE_NO_DISPATCH(); + #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -7429,7 +7432,12 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif res = temp; LLTRACE_RESUME_FRAME(); } @@ -7497,7 +7505,12 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif + #if TIER_TWO + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); } @@ -7645,31 +7658,32 @@ { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; - while (oparg > 255) { - oparg >>= 8; - start--; - } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) { - JUMP_TO_LABEL(error); - } - } - else { + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + if (tstate->interp->jit_state.code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); + tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + if (tstate->interp->jit_state.code_buffer == NULL) { + DISPATCH(); + } } + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (!_is_sys_tracing) { + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + ENTER_TRACING(); + } + int _jump_taken = false; + PyCodeObject *old_code = _PyFrame_GetCode(frame); + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + int _old_stack_level = 0; + TRACING_DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -10370,7 +10384,12 @@ _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); @@ -10402,7 +10421,12 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -12090,7 +12114,12 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif + #if TIER_TWO + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h new file mode 100644 index 00000000000000..932a57a22e5110 --- /dev/null +++ b/Python/generated_tracer_cases.c.h @@ -0,0 +1,14556 @@ +// This file is generated by Tools/cases_generator/tracer_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + #define TIER_ONE 1 + #define TRACING_JIT 1 + #ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */ + + + TRACING_TARGET(BINARY_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED_TRACING_BINARY_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + opcode = BINARY_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_OPARG_LAST); + } + /* Skip 4 cache entries */ + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = lhs; + lhs = res; + stack_pointer[-2] = lhs; + PyStackRef_CLOSE(tmp); + tmp = rhs; + rhs = PyStackRef_NULL; + stack_pointer[-1] = rhs; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + opcode = BINARY_OP_ADD_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + opcode = BINARY_OP_ADD_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_UNICODE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + opcode = BINARY_OP_ADD_UNICODE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_EXTEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + opcode = BINARY_OP_EXTEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_INPLACE_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + opcode = BINARY_OP_INPLACE_ADD_UNICODE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = (int)CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyUnicode_Append(&temp, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*target_local)) { + TRACING_JUMP_TO_LABEL(error); + } + #if TIER_ONE + + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + opcode = BINARY_OP_MULTIPLY_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_MULTIPLY_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + opcode = BINARY_OP_MULTIPLY_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); + opcode = BINARY_OP_SUBSCR_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_DICT + { + sub_st = stack_pointer[-1]; + dict_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (rc <= 0) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_GETITEM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); + opcode = BINARY_OP_SUBSCR_GETITEM; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef new_frame; + /* Skip 5 cache entries */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _BINARY_OP_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_OP, hit); + } + // _BINARY_OP_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); + opcode = BINARY_OP_SUBSCR_LIST_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = list_st; + list_st = res; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); + opcode = BINARY_OP_SUBSCR_LIST_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_SLICE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_SLICE + { + sub_st = tos; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PySlice_Check(sub)); + assert(PyList_CheckExact(list)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_SliceSubscript(list, sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = list_st; + list_st = PyStackRef_NULL; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_STR_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); + opcode = BINARY_OP_SUBSCR_STR_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_STR_INT + { + sub_st = value; + str_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectBorrow(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_TUPLE_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); + opcode = BINARY_OP_SUBSCR_TUPLE_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_TUPLE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_TUPLE_INT + { + sub_st = value; + tuple_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = tuple_st; + tuple_st = res; + stack_pointer[-1] = tuple_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + opcode = BINARY_OP_SUBTRACT_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBTRACT_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + opcode = BINARY_OP_SUBTRACT_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + opcode = BINARY_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(container); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_INTERPOLATION) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_INTERPOLATION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_INTERPOLATION); + opcode = BUILD_INTERPOLATION; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef str; + _PyStackRef *format; + _PyStackRef interpolation; + format = &stack_pointer[-(oparg & 1)]; + str = stack_pointer[-1 - (oparg & 1)]; + value = stack_pointer[-2 - (oparg & 1)]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); + int conversion = oparg >> 2; + PyObject *format_o; + if (oparg & 1) { + format_o = PyStackRef_AsPyObjectBorrow(format[0]); + } + else { + format_o = &_Py_STR(empty); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (oparg & 1) { + stack_pointer += -(oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(format[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer += -(oparg & 1); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (interpolation_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); + stack_pointer[0] = interpolation; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + opcode = BUILD_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (list_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + list = PyStackRef_FromPyObjectStealMortal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_MAP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_MAP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + opcode = BUILD_MAP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + if (map_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + map = PyStackRef_FromPyObjectStealMortal(map_o); + stack_pointer[0] = map; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_SET) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + opcode = BUILD_SET; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + int err = 0; + for (Py_ssize_t i = 0; i < oparg; i++) { + _PyStackRef value = values[i]; + values[i] = PyStackRef_NULL; + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + if (err) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + set = PyStackRef_FromPyObjectStealMortal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + opcode = BUILD_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef *args; + _PyStackRef slice; + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + if (slice_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + slice = PyStackRef_FromPyObjectStealMortal(slice_o); + stack_pointer[0] = slice; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_STRING) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_STRING; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + opcode = BUILD_STRING; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + if (str_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[0] = str; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_TEMPLATE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TEMPLATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TEMPLATE); + opcode = BUILD_TEMPLATE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef strings; + _PyStackRef interpolations; + _PyStackRef template; + interpolations = stack_pointer[-1]; + strings = stack_pointer[-2]; + PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); + PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(interpolations); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(strings); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (template_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + template = PyStackRef_FromPyObjectSteal(template_o); + stack_pointer[0] = template; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + opcode = BUILD_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + tup = PyStackRef_FromPyObjectStealMortal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CACHE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CACHE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + opcode = CACHE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED_TRACING_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ALLOC_AND_ENTER_INIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + opcode = CALL_ALLOC_AND_ENTER_INIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef init; + _PyStackRef self; + _PyStackRef *args; + _PyStackRef init_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_AND_ALLOCATE_OBJECT + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + self_or_null = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = self_or_null; + init = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + shim->localsplus[0] = PyStackRef_DUP(self); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init, NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); + } + // _PUSH_FRAME + { + new_frame = init_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + opcode = CALL_BOUND_METHOD_EXACT_ARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + opcode = CALL_BOUND_METHOD_GENERAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + opcode = CALL_BUILTIN_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + opcode = CALL_BUILTIN_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_O) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + opcode = CALL_BUILTIN_O; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_FUNCTION_EX) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + opcode = CALL_FUNCTION_EX; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + TRACING_DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_INTRINSIC_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + opcode = CALL_INTRINSIC_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_INTRINSIC_2) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_2; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + opcode = CALL_INTRINSIC_2; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; + PyStackRef_CLOSE(tmp); + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_ISINSTANCE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ISINSTANCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + opcode = CALL_ISINSTANCE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef instance; + _PyStackRef cls; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_THIRD_NULL + { + null = stack_pointer[-3]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_ISINSTANCE + { + callable = stack_pointer[-4]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_ISINSTANCE + { + cls = stack_pointer[-1]; + instance = stack_pointer[-2]; + STAT_INC(CALL, hit); + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(inst_o, cls_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + TRACING_JUMP_TO_LABEL(error); + } + (void)null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(cls); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(instance); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED_TRACING_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL_KW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_BOUND_METHOD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + opcode = CALL_KW_BOUND_METHOD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_NON_PY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_NON_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + opcode = CALL_KW_NON_PY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_PY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + opcode = CALL_KW_PY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_LEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + opcode = CALL_LEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_LEN + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_LEN + { + arg = stack_pointer[-1]; + (void)null; + STAT_INC(CALL, hit); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_LIST_APPEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + opcode = CALL_LIST_APPEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef nos; + _PyStackRef self; + _PyStackRef arg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_LIST_APPEND + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_NOS_LIST + { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_LIST_APPEND + { + arg = stack_pointer[-1]; + self = nos; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!PyList_CheckExact(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + #if TIER_ONE + + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + opcode = CALL_METHOD_DESCRIPTOR_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + opcode = CALL_METHOD_DESCRIPTOR_O; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_NON_PY_GENERAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_NON_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + opcode = CALL_NON_PY_GENERAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_PY_EXACT_ARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + opcode = CALL_PY_EXACT_ARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_PY_GENERAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + opcode = CALL_PY_GENERAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_STR_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_STR_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + opcode = CALL_STR_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_STR_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)callable; + (void)null; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_TUPLE_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TUPLE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + opcode = CALL_TUPLE_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_TUPLE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)callable; + (void)null; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_TYPE_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TYPE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + opcode = CALL_TYPE_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_TYPE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_TYPE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + (void)callable; + (void)null; + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CHECK_EG_MATCH) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EG_MATCH; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + opcode = CHECK_EG_MATCH; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (!Py_IsNone(match_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CHECK_EXC_MATCH) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EXC_MATCH; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + opcode = CHECK_EXC_MATCH; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CLEANUP_THROW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CLEANUP_THROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + opcode = CLEANUP_THROW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef sub_iter; + _PyStackRef last_sent_val; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #if !_Py_TAIL_CALL_INTERP + assert(throwflag); + #endif + assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_iter; + sub_iter = value; + stack_pointer[-3] = sub_iter; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-1] = exc_value_st; + PyStackRef_CLOSE(tmp); + tmp = last_sent_val; + last_sent_val = PyStackRef_NULL; + stack_pointer[-2] = last_sent_val; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + none = PyStackRef_None; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + stack_pointer[0] = none; + stack_pointer[1] = value; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED_TRACING_COMPARE_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = COMPARE_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (oparg & 16) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) { + TRACING_JUMP_TO_LABEL(error); + } + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + opcode = COMPARE_OP_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + opcode = COMPARE_OP_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_STR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + opcode = COMPARE_OP_STR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED_TRACING_CONTAINS_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = CONTAINS_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + opcode = CONTAINS_OP_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_DICT + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_DICT + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP_SET) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + opcode = CONTAINS_OP_SET; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_ANY_SET + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_SET + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONVERT_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONVERT_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + opcode = CONVERT_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COPY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COPY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + opcode = COPY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COPY_FREE_VARS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COPY_FREE_VARS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + opcode = COPY_FREE_VARS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + opcode = DELETE_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + opcode = DELETE_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + opcode = DELETE_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_GLOBAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_GLOBAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + opcode = DELETE_GLOBAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + opcode = DELETE_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_SUBSCR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_SUBSCR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + opcode = DELETE_SUBSCR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DICT_MERGE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_MERGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + opcode = DICT_MERGE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DICT_UPDATE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_UPDATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + opcode = DICT_UPDATE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_ASYNC_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + opcode = END_ASYNC_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + opcode = END_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_SEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + opcode = END_SEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(ENTER_EXECUTOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = ENTER_EXECUTOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + opcode = ENTER_EXECUTOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->current_executor == NULL); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; + #if TRACING_JIT + RECORD_TRACE_NO_DISPATCH(); + #endif + TIER1_TO_TIER2(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + TRACING_DISPATCH(); + } + + TRACING_TARGET(EXIT_INIT_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = EXIT_INIT_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + opcode = EXIT_INIT_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(EXTENDED_ARG) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = EXTENDED_ARG; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + opcode = EXTENDED_ARG; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TRACING_TARGET(FORMAT_SIMPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_SIMPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + opcode = FORMAT_SIMPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + else { + res = value; + stack_pointer += -1; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FORMAT_WITH_SPEC) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_WITH_SPEC; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + opcode = FORMAT_WITH_SPEC; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED_TRACING_FOR_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = FOR_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _FOR_ITER + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + TRACING_JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + next = item; + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_GEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + opcode = FOR_ITER_GEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #endif + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + opcode = FOR_ITER_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #endif + } + // _ITER_JUMP_LIST + { + #ifdef Py_GIL_DISABLED + + #else + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + #endif + } + // _ITER_NEXT_LIST + { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); + #ifdef Py_GIL_DISABLED + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); + STAT_INC(FOR_ITER, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result < 0) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + if (result == 0) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + #else + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); + #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_RANGE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_RANGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + opcode = FOR_ITER_RANGE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-2]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #endif + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); + TRACING_DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + opcode = FOR_ITER_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + } + // _ITER_JUMP_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_AITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + opcode = GET_AITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_ANEXT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ANEXT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + opcode = GET_ANEXT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_AWAITABLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AWAITABLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + opcode = GET_AWAITABLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + opcode = GET_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef index_or_null; + iterable = stack_pointer[-1]; + #ifdef Py_STATS + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_GatherStats_GetIter(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_LEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + opcode = GET_LEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_YIELD_FROM_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_YIELD_FROM_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + opcode = GET_YIELD_FROM_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + iter = iterable; + } + else if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-1] = iter; + TRACING_DISPATCH(); + } + + TRACING_TARGET(IMPORT_FROM) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_FROM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + opcode = IMPORT_FROM; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(IMPORT_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + opcode = IMPORT_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + opcode = INSTRUMENTED_CALL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef *args; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = self_or_null; + func = callable; + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + TRACING_DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL_KW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_KW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + opcode = INSTRUMENTED_CALL_KW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null); + } + else if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } + PyObject *function = PyStackRef_AsPyObjectBorrow(callable); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); + opcode = INSTRUMENTED_END_ASYNC_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + // _MONITOR_END_ASYNC_FOR + { + assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); + INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + // _END_ASYNC_FOR + { + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + opcode = INSTRUMENTED_END_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-3]; + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_SEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + opcode = INSTRUMENTED_END_SEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_FOR_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_FOR_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + opcode = INSTRUMENTED_FOR_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + TRACING_JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_INSTRUCTION) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_INSTRUCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + opcode = INSTRUMENTED_INSTRUCTION; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) { + TRACING_JUMP_TO_LABEL(error); + } + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_BACKWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + opcode = INSTRUMENTED_JUMP_BACKWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + opcode = INSTRUMENTED_JUMP_FORWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_LINE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LINE; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + opcode = INSTRUMENTED_LINE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + TRACING_JUMP_TO_LABEL(error); + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + TRACING_DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + + TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + opcode = INSTRUMENTED_NOT_TAKEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + (void)this_instr; + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + opcode = INSTRUMENTED_POP_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = !PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_RESUME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RESUME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + opcode = INSTRUMENTED_RESUME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + next_instr = this_instr; + TRACING_DISPATCH(); + } + } + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + opcode = INSTRUMENTED_RETURN_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _RETURN_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE + LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + opcode = INSTRUMENTED_YIELD_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + } + // _YIELD_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif + #if TIER_TWO + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INTERPRETER_EXIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INTERPRETER_EXIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + opcode = INTERPRETER_EXIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(frame)); + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(executor); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + #endif + LLTRACE_RESUME_FRAME(); + return result; + } + + TRACING_TARGET(IS_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = IS_OP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + opcode = IS_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_TRACING_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = JUMP_BACKWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + /* Skip 1 cache entry */ + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION_FT + if (this_instr->op.code == JUMP_BACKWARD) { + uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); + next_instr = this_instr; + DISPATCH_SAME_OPARG() + } + #endif + } + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_JIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + opcode = JUMP_BACKWARD_JIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + // _JIT + { + #ifdef _Py_TIER2 + _Py_BackoffCounter counter = this_instr[1].counter; + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + if (tstate->interp->jit_state.code_buffer == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tstate->interp->jit_state.code_buffer == NULL) { + TRACING_DISPATCH(); + } + } + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (!_is_sys_tracing) { + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + ENTER_TRACING(); + } + int _jump_taken = false; + PyCodeObject *old_code = _PyFrame_GetCode(frame); + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + int _old_stack_level = 0; + TRACING_DISPATCH(); + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_INTERRUPT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + opcode = JUMP_BACKWARD_NO_INTERRUPT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_NO_JIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + opcode = JUMP_BACKWARD_NO_JIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_FORWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + opcode = JUMP_FORWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + JUMPBY(oparg); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LIST_APPEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + opcode = LIST_APPEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) { + TRACING_JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LIST_EXTEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + opcode = LIST_EXTEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + assert(Py_IsNone(none_val)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED_TRACING_LOAD_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + opcode = LOAD_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef owner; + _PyStackRef *attr; + _PyStackRef *self_or_null; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + attr = &stack_pointer[-1]; + self_or_null = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + *attr = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + assert(!PyStackRef_IsNull(*attr)); + self_or_null[0] = owner; + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*attr)) { + TRACING_JUMP_TO_LABEL(error); + } + self_or_null[0] = PyStackRef_NULL; + stack_pointer += 1; + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + *attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer += 1; + } + } + stack_pointer += (oparg&1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + opcode = LOAD_ATTR_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + new_frame->localsplus[0] = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10u ; + TRACING_DISPATCH_INLINED(new_frame); + } + + TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + opcode = LOAD_ATTR_INSTANCE_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_LAZY_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + opcode = LOAD_ATTR_METHOD_LAZY_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + opcode = LOAD_ATTR_METHOD_NO_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + opcode = LOAD_ATTR_METHOD_WITH_VALUES; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_MODULE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + opcode = LOAD_ATTR_MODULE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_ATTR_MODULE + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_PROPERTY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_PROPERTY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + opcode = LOAD_ATTR_PROPERTY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_SLOT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + opcode = LOAD_ATTR_SLOT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_WITH_HINT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + opcode = LOAD_ATTR_WITH_HINT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); + assert(PyDict_CheckExact((PyObject *)dict)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #endif + PyObject *attr_o; + if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (dk->dk_kind != DICT_KEYS_UNICODE) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; + if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); + if (attr_o == NULL) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_BUILD_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_BUILD_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + opcode = LOAD_BUILD_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_COMMON_CONSTANT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_COMMON_CONSTANT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + opcode = LOAD_COMMON_CONSTANT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_CONST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_CONST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + opcode = LOAD_CONST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + opcode = LOAD_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = _PyCell_GetStackRef(cell); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(value)) { + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + opcode = LOAD_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_AND_CLEAR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_AND_CLEAR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + opcode = LOAD_FAST_AND_CLEAR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_BORROW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW); + opcode = LOAD_FAST_BORROW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); + opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); + value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + opcode = LOAD_FAST_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_LOAD_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + opcode = LOAD_FAST_LOAD_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + opcode = LOAD_FROM_DICT_OR_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_GLOBALS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + opcode = LOAD_FROM_DICT_OR_GLOBALS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(mod_or_class_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + } + } + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED_TRACING_LOAD_GLOBAL:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + opcode = LOAD_GLOBAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef *res; + _PyStackRef *null; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL_BUILTIN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_BUILTIN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + opcode = LOAD_GLOBAL_BUILTIN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t version = read_u16(&this_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL_MODULE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + opcode = LOAD_GLOBAL_MODULE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _NOP + { + } + // _LOAD_GLOBAL_MODULE + { + uint16_t version = read_u16(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_LOCALS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_LOCALS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + opcode = LOAD_LOCALS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + opcode = LOAD_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SMALL_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SMALL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + opcode = LOAD_SMALL_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SPECIAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SPECIAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + opcode = LOAD_SPECIAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef self; + _PyStackRef *method_and_self; + // _INSERT_NULL + { + self = stack_pointer[-1]; + method_and_self = &stack_pointer[-1]; + method_and_self[1] = self; + method_and_self[0] = PyStackRef_NULL; + } + // _LOAD_SPECIAL + { + method_and_self = &stack_pointer[-1]; + PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err <= 0) { + if (err == 0) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) + ? _Py_SpecialMethods[oparg].error_suggestion + : _Py_SpecialMethods[oparg].error; + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); + assert(errfmt != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED_TRACING_LOAD_SUPER_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = LOAD_SUPER_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + opcode = LOAD_SUPER_ATTR_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attr == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[0] = attr_st; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + opcode = LOAD_SUPER_ATTR_METHOD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAKE_CELL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_CELL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + opcode = MAKE_CELL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAKE_FUNCTION) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_FUNCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + opcode = MAKE_FUNCTION; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(codeobj_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[0] = func; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAP_ADD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MAP_ADD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + opcode = MAP_ADD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + opcode = MATCH_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + TRACING_JUMP_TO_LABEL(error); + } + attrs = PyStackRef_None; + } + stack_pointer[0] = attrs; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_KEYS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_KEYS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + opcode = MATCH_KEYS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_MAPPING) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_MAPPING; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + opcode = MATCH_MAPPING; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_SEQUENCE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_SEQUENCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + opcode = MATCH_SEQUENCE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(NOP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = NOP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + opcode = NOP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + TRACING_DISPATCH(); + } + + TRACING_TARGET(NOT_TAKEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + opcode = NOT_TAKEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_EXCEPT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_EXCEPT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + opcode = POP_EXCEPT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + opcode = POP_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_FALSE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + opcode = POP_JUMP_IF_FALSE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + opcode = POP_JUMP_IF_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_NOT_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + opcode = POP_JUMP_IF_NOT_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_TRUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + opcode = POP_JUMP_IF_TRUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_TOP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_TOP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + opcode = POP_TOP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(PUSH_EXC_INFO) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_EXC_INFO; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + opcode = PUSH_EXC_INFO; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(PUSH_NULL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_NULL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + opcode = PUSH_NULL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RAISE_VARARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RAISE_VARARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + opcode = RAISE_VARARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + TRACING_JUMP_TO_LABEL(error); + } + + TRACING_TARGET(RERAISE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RERAISE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + opcode = RERAISE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); + } + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + + TRACING_TARGET(RESERVED) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RESERVED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + opcode = RESERVED; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RESUME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED_TRACING_RESUME:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + opcode = RESUME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + next_instr = this_instr; + TRACING_DISPATCH(); + } + } + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(RESUME_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + opcode = RESUME_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(TRACING_RESUME); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(TRACING_RESUME); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(TRACING_RESUME); + } + #endif + TRACING_DISPATCH(); + } + + TRACING_TARGET(RETURN_GENERATOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_GENERATOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + opcode = RETURN_GENERATOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + #if TIER_ONE + LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RETURN_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + opcode = RETURN_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE + LOAD_IP(frame->return_offset); + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED_TRACING_SEND:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = SEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + TRACING_DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + RECORD_DYNAMIC_JUMP_TAKEN(); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + retval = PyStackRef_FromPyObjectSteal(retval_o); + } + stack_pointer[0] = retval; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SEND_GEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + opcode = SEND_GEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(TRACING_SEND); + } + } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(TRACING_SEND); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(TRACING_SEND); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SETUP_ANNOTATIONS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SETUP_ANNOTATIONS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + opcode = SETUP_ANNOTATIONS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_ADD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SET_ADD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + opcode = SET_ADD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SET_FUNCTION_ATTRIBUTE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + opcode = SET_FUNCTION_ATTRIBUTE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_UPDATE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SET_UPDATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + opcode = SET_UPDATE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED_TRACING_STORE_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + opcode = STORE_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + opcode = STORE_ATTR_INSTANCE_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + } + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_SLOT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + opcode = STORE_ATTR_SLOT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_WITH_HINT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + opcode = STORE_ATTR_WITH_HINT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + STAT_INC(STORE_ATTR, hit); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + opcode = STORE_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + opcode = STORE_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST_LOAD_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + opcode = STORE_FAST_LOAD_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST_STORE_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_STORE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + opcode = STORE_FAST_STORE_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + tmp = GETLOCAL(oparg2); + GETLOCAL(oparg2) = value2; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_GLOBAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_GLOBAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + opcode = STORE_GLOBAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + opcode = STORE_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + opcode = STORE_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-4] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED_TRACING_STORE_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = STORE_SUBSCR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + opcode = STORE_SUBSCR_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_DICT + { + sub = stack_pointer[-1]; + dict_st = nos; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR_LIST_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + opcode = STORE_SUBSCR_LIST_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(list_st); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SWAP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SWAP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + opcode = SWAP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef bottom; + _PyStackRef top; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-2 - (oparg-2)] = bottom; + stack_pointer[-1] = top; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED_TRACING_TO_BOOL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = TO_BOOL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_ALWAYS_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + opcode = TO_BOOL_ALWAYS_TRUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + } + // _REPLACE_WITH_TRUE + { + value = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_BOOL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_BOOL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + opcode = TO_BOOL_BOOL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + opcode = TO_BOOL_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + opcode = TO_BOOL_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_LIST + { + value = tos; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + opcode = TO_BOOL_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_STR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + opcode = TO_BOOL_STR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_STR + { + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + assert(Py_SIZE(value_o)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + } + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_INVERT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_INVERT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + opcode = UNARY_INVERT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_NEGATIVE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NEGATIVE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + opcode = UNARY_NEGATIVE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_NOT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + opcode = UNARY_NOT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_EX) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + opcode = UNPACK_EX; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef seq; + _PyStackRef *top; + seq = stack_pointer[-1]; + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED_TRACING_UNPACK_SEQUENCE:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = UNPACK_SEQUENCE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef seq; + _PyStackRef *top; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + opcode = UNPACK_SEQUENCE_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_LIST + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyList_CheckExact(seq_o)); + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + opcode = UNPACK_SEQUENCE_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TUPLE + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TWO_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + opcode = UNPACK_SEQUENCE_TWO_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TWO_TUPLE + { + seq = tos; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(WITH_EXCEPT_START) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = WITH_EXCEPT_START; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + opcode = WITH_EXCEPT_START; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + PyObject *original_tb = tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + assert(PyStackRef_IsTaggedInt(lasti)); + (void)lasti; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + Py_XDECREF(original_tb); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(YIELD_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + opcode = YIELD_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif + #if TIER_TWO + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + #endif /* END TRACING INSTRUCTIONS */ + #undef TRACING_JIT + #undef TIER_ONE diff --git a/Python/instrumentation.c b/Python/instrumentation.c index b4b2bc5dc69f9d..81e46a331e0b9e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -18,6 +18,7 @@ #include "pycore_tuple.h" // _PyTuple_FromArraySteal() #include "opcode_ids.h" +#include "pycore_optimizer.h" /* Uncomment this to dump debugging output when assertions fail */ @@ -1785,6 +1786,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), code); #endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6dd443e1655ed0..66abd6d4623e1c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -257,8 +257,270 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, }; +#if _Py_TIER2 +static void *opcode_tracing_targets_table[256] = { + &&TARGET_TRACING_CACHE, + &&TARGET_TRACING_BINARY_SLICE, + &&TARGET_TRACING_BUILD_TEMPLATE, + &&TARGET_TRACING_BINARY_OP_INPLACE_ADD_UNICODE, + &&TARGET_TRACING_CALL_FUNCTION_EX, + &&TARGET_TRACING_CHECK_EG_MATCH, + &&TARGET_TRACING_CHECK_EXC_MATCH, + &&TARGET_TRACING_CLEANUP_THROW, + &&TARGET_TRACING_DELETE_SUBSCR, + &&TARGET_TRACING_END_FOR, + &&TARGET_TRACING_END_SEND, + &&TARGET_TRACING_EXIT_INIT_CHECK, + &&TARGET_TRACING_FORMAT_SIMPLE, + &&TARGET_TRACING_FORMAT_WITH_SPEC, + &&TARGET_TRACING_GET_AITER, + &&TARGET_TRACING_GET_ANEXT, + &&TARGET_TRACING_GET_ITER, + &&TARGET_TRACING_RESERVED, + &&TARGET_TRACING_GET_LEN, + &&TARGET_TRACING_GET_YIELD_FROM_ITER, + &&TARGET_TRACING_INTERPRETER_EXIT, + &&TARGET_TRACING_LOAD_BUILD_CLASS, + &&TARGET_TRACING_LOAD_LOCALS, + &&TARGET_TRACING_MAKE_FUNCTION, + &&TARGET_TRACING_MATCH_KEYS, + &&TARGET_TRACING_MATCH_MAPPING, + &&TARGET_TRACING_MATCH_SEQUENCE, + &&TARGET_TRACING_NOP, + &&TARGET_TRACING_NOT_TAKEN, + &&TARGET_TRACING_POP_EXCEPT, + &&TARGET_TRACING_POP_ITER, + &&TARGET_TRACING_POP_TOP, + &&TARGET_TRACING_PUSH_EXC_INFO, + &&TARGET_TRACING_PUSH_NULL, + &&TARGET_TRACING_RETURN_GENERATOR, + &&TARGET_TRACING_RETURN_VALUE, + &&TARGET_TRACING_SETUP_ANNOTATIONS, + &&TARGET_TRACING_STORE_SLICE, + &&TARGET_TRACING_STORE_SUBSCR, + &&TARGET_TRACING_TO_BOOL, + &&TARGET_TRACING_UNARY_INVERT, + &&TARGET_TRACING_UNARY_NEGATIVE, + &&TARGET_TRACING_UNARY_NOT, + &&TARGET_TRACING_WITH_EXCEPT_START, + &&TARGET_TRACING_BINARY_OP, + &&TARGET_TRACING_BUILD_INTERPOLATION, + &&TARGET_TRACING_BUILD_LIST, + &&TARGET_TRACING_BUILD_MAP, + &&TARGET_TRACING_BUILD_SET, + &&TARGET_TRACING_BUILD_SLICE, + &&TARGET_TRACING_BUILD_STRING, + &&TARGET_TRACING_BUILD_TUPLE, + &&TARGET_TRACING_CALL, + &&TARGET_TRACING_CALL_INTRINSIC_1, + &&TARGET_TRACING_CALL_INTRINSIC_2, + &&TARGET_TRACING_CALL_KW, + &&TARGET_TRACING_COMPARE_OP, + &&TARGET_TRACING_CONTAINS_OP, + &&TARGET_TRACING_CONVERT_VALUE, + &&TARGET_TRACING_COPY, + &&TARGET_TRACING_COPY_FREE_VARS, + &&TARGET_TRACING_DELETE_ATTR, + &&TARGET_TRACING_DELETE_DEREF, + &&TARGET_TRACING_DELETE_FAST, + &&TARGET_TRACING_DELETE_GLOBAL, + &&TARGET_TRACING_DELETE_NAME, + &&TARGET_TRACING_DICT_MERGE, + &&TARGET_TRACING_DICT_UPDATE, + &&TARGET_TRACING_END_ASYNC_FOR, + &&TARGET_TRACING_EXTENDED_ARG, + &&TARGET_TRACING_FOR_ITER, + &&TARGET_TRACING_GET_AWAITABLE, + &&TARGET_TRACING_IMPORT_FROM, + &&TARGET_TRACING_IMPORT_NAME, + &&TARGET_TRACING_IS_OP, + &&TARGET_TRACING_JUMP_BACKWARD, + &&TARGET_TRACING_JUMP_BACKWARD_NO_INTERRUPT, + &&TARGET_TRACING_JUMP_FORWARD, + &&TARGET_TRACING_LIST_APPEND, + &&TARGET_TRACING_LIST_EXTEND, + &&TARGET_TRACING_LOAD_ATTR, + &&TARGET_TRACING_LOAD_COMMON_CONSTANT, + &&TARGET_TRACING_LOAD_CONST, + &&TARGET_TRACING_LOAD_DEREF, + &&TARGET_TRACING_LOAD_FAST, + &&TARGET_TRACING_LOAD_FAST_AND_CLEAR, + &&TARGET_TRACING_LOAD_FAST_BORROW, + &&TARGET_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW, + &&TARGET_TRACING_LOAD_FAST_CHECK, + &&TARGET_TRACING_LOAD_FAST_LOAD_FAST, + &&TARGET_TRACING_LOAD_FROM_DICT_OR_DEREF, + &&TARGET_TRACING_LOAD_FROM_DICT_OR_GLOBALS, + &&TARGET_TRACING_LOAD_GLOBAL, + &&TARGET_TRACING_LOAD_NAME, + &&TARGET_TRACING_LOAD_SMALL_INT, + &&TARGET_TRACING_LOAD_SPECIAL, + &&TARGET_TRACING_LOAD_SUPER_ATTR, + &&TARGET_TRACING_MAKE_CELL, + &&TARGET_TRACING_MAP_ADD, + &&TARGET_TRACING_MATCH_CLASS, + &&TARGET_TRACING_POP_JUMP_IF_FALSE, + &&TARGET_TRACING_POP_JUMP_IF_NONE, + &&TARGET_TRACING_POP_JUMP_IF_NOT_NONE, + &&TARGET_TRACING_POP_JUMP_IF_TRUE, + &&TARGET_TRACING_RAISE_VARARGS, + &&TARGET_TRACING_RERAISE, + &&TARGET_TRACING_SEND, + &&TARGET_TRACING_SET_ADD, + &&TARGET_TRACING_SET_FUNCTION_ATTRIBUTE, + &&TARGET_TRACING_SET_UPDATE, + &&TARGET_TRACING_STORE_ATTR, + &&TARGET_TRACING_STORE_DEREF, + &&TARGET_TRACING_STORE_FAST, + &&TARGET_TRACING_STORE_FAST_LOAD_FAST, + &&TARGET_TRACING_STORE_FAST_STORE_FAST, + &&TARGET_TRACING_STORE_GLOBAL, + &&TARGET_TRACING_STORE_NAME, + &&TARGET_TRACING_SWAP, + &&TARGET_TRACING_UNPACK_EX, + &&TARGET_TRACING_UNPACK_SEQUENCE, + &&TARGET_TRACING_YIELD_VALUE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACING_RESUME, + &&TARGET_TRACING_BINARY_OP_ADD_FLOAT, + &&TARGET_TRACING_BINARY_OP_ADD_INT, + &&TARGET_TRACING_BINARY_OP_ADD_UNICODE, + &&TARGET_TRACING_BINARY_OP_EXTEND, + &&TARGET_TRACING_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_TRACING_BINARY_OP_MULTIPLY_INT, + &&TARGET_TRACING_BINARY_OP_SUBSCR_DICT, + &&TARGET_TRACING_BINARY_OP_SUBSCR_GETITEM, + &&TARGET_TRACING_BINARY_OP_SUBSCR_LIST_INT, + &&TARGET_TRACING_BINARY_OP_SUBSCR_LIST_SLICE, + &&TARGET_TRACING_BINARY_OP_SUBSCR_STR_INT, + &&TARGET_TRACING_BINARY_OP_SUBSCR_TUPLE_INT, + &&TARGET_TRACING_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_TRACING_BINARY_OP_SUBTRACT_INT, + &&TARGET_TRACING_CALL_ALLOC_AND_ENTER_INIT, + &&TARGET_TRACING_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_TRACING_CALL_BOUND_METHOD_GENERAL, + &&TARGET_TRACING_CALL_BUILTIN_CLASS, + &&TARGET_TRACING_CALL_BUILTIN_FAST, + &&TARGET_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_TRACING_CALL_BUILTIN_O, + &&TARGET_TRACING_CALL_ISINSTANCE, + &&TARGET_TRACING_CALL_KW_BOUND_METHOD, + &&TARGET_TRACING_CALL_KW_NON_PY, + &&TARGET_TRACING_CALL_KW_PY, + &&TARGET_TRACING_CALL_LEN, + &&TARGET_TRACING_CALL_LIST_APPEND, + &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_FAST, + &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_O, + &&TARGET_TRACING_CALL_NON_PY_GENERAL, + &&TARGET_TRACING_CALL_PY_EXACT_ARGS, + &&TARGET_TRACING_CALL_PY_GENERAL, + &&TARGET_TRACING_CALL_STR_1, + &&TARGET_TRACING_CALL_TUPLE_1, + &&TARGET_TRACING_CALL_TYPE_1, + &&TARGET_TRACING_COMPARE_OP_FLOAT, + &&TARGET_TRACING_COMPARE_OP_INT, + &&TARGET_TRACING_COMPARE_OP_STR, + &&TARGET_TRACING_CONTAINS_OP_DICT, + &&TARGET_TRACING_CONTAINS_OP_SET, + &&TARGET_TRACING_FOR_ITER_GEN, + &&TARGET_TRACING_FOR_ITER_LIST, + &&TARGET_TRACING_FOR_ITER_RANGE, + &&TARGET_TRACING_FOR_ITER_TUPLE, + &&TARGET_TRACING_JUMP_BACKWARD_JIT, + &&TARGET_TRACING_JUMP_BACKWARD_NO_JIT, + &&TARGET_TRACING_LOAD_ATTR_CLASS, + &&TARGET_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + &&TARGET_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_TRACING_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_TRACING_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_TRACING_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_TRACING_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_TRACING_LOAD_ATTR_MODULE, + &&TARGET_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + &&TARGET_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + &&TARGET_TRACING_LOAD_ATTR_PROPERTY, + &&TARGET_TRACING_LOAD_ATTR_SLOT, + &&TARGET_TRACING_LOAD_ATTR_WITH_HINT, + &&TARGET_TRACING_LOAD_GLOBAL_BUILTIN, + &&TARGET_TRACING_LOAD_GLOBAL_MODULE, + &&TARGET_TRACING_LOAD_SUPER_ATTR_ATTR, + &&TARGET_TRACING_LOAD_SUPER_ATTR_METHOD, + &&TARGET_TRACING_RESUME_CHECK, + &&TARGET_TRACING_SEND_GEN, + &&TARGET_TRACING_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_TRACING_STORE_ATTR_SLOT, + &&TARGET_TRACING_STORE_ATTR_WITH_HINT, + &&TARGET_TRACING_STORE_SUBSCR_DICT, + &&TARGET_TRACING_STORE_SUBSCR_LIST_INT, + &&TARGET_TRACING_TO_BOOL_ALWAYS_TRUE, + &&TARGET_TRACING_TO_BOOL_BOOL, + &&TARGET_TRACING_TO_BOOL_INT, + &&TARGET_TRACING_TO_BOOL_LIST, + &&TARGET_TRACING_TO_BOOL_NONE, + &&TARGET_TRACING_TO_BOOL_STR, + &&TARGET_TRACING_UNPACK_SEQUENCE_LIST, + &&TARGET_TRACING_UNPACK_SEQUENCE_TUPLE, + &&TARGET_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACING_INSTRUMENTED_END_FOR, + &&TARGET_TRACING_INSTRUMENTED_POP_ITER, + &&TARGET_TRACING_INSTRUMENTED_END_SEND, + &&TARGET_TRACING_INSTRUMENTED_FOR_ITER, + &&TARGET_TRACING_INSTRUMENTED_INSTRUCTION, + &&TARGET_TRACING_INSTRUMENTED_JUMP_FORWARD, + &&TARGET_TRACING_INSTRUMENTED_NOT_TAKEN, + &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE, + &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE, + &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE, + &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_TRACING_INSTRUMENTED_RESUME, + &&TARGET_TRACING_INSTRUMENTED_RETURN_VALUE, + &&TARGET_TRACING_INSTRUMENTED_YIELD_VALUE, + &&TARGET_TRACING_INSTRUMENTED_END_ASYNC_FOR, + &&TARGET_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR, + &&TARGET_TRACING_INSTRUMENTED_CALL, + &&TARGET_TRACING_INSTRUMENTED_CALL_KW, + &&TARGET_TRACING_INSTRUMENTED_CALL_FUNCTION_EX, + &&TARGET_TRACING_INSTRUMENTED_JUMP_BACKWARD, + &&TARGET_TRACING_INSTRUMENTED_LINE, + &&TARGET_TRACING_ENTER_EXECUTOR, +}; +#endif #else /* _Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr instruction_funcptr_table[256]; +static py_tail_call_funcptr instruction_funcptr_handler_table[256]; + +static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); @@ -268,230 +530,455 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_MAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_SET(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_STRING(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_TEMPLATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CACHE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_O(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_ISINSTANCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_NON_PY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_PY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_LEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_LIST_APPEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_PY_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_STR_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_TUPLE_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_TYPE_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CHECK_EG_MATCH(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CLEANUP_THROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_STR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP_SET(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONVERT_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COPY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COPY_FREE_VARS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_SUBSCR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DICT_MERGE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DICT_UPDATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_ASYNC_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_ENTER_EXECUTOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_EXTENDED_ARG(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FORMAT_SIMPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_GEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_RANGE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_AITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_ANEXT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_AWAITABLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_LEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IMPORT_FROM(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IMPORT_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INTERPRETER_EXIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IS_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_FORWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LIST_APPEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LIST_EXTEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_CONST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_LOCALS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SMALL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SPECIAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAKE_CELL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAKE_FUNCTION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAP_ADD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_KEYS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_MAPPING(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_SEQUENCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_NOP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_NOT_TAKEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_EXCEPT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_TOP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_PUSH_EXC_INFO(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_PUSH_NULL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RAISE_VARARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RERAISE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESERVED(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESUME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESUME_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RETURN_GENERATOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RETURN_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SEND_GEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_ADD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_UPDATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SWAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_STR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_INVERT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_NEGATIVE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_NOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_WITH_EXCEPT_START(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; @@ -503,7 +990,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr instruction_funcptr_table[256] = { +static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -761,4 +1248,262 @@ static py_tail_call_funcptr instruction_funcptr_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; +static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [BINARY_OP] = _TAIL_CALL_TRACING_BINARY_OP, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_ADD_FLOAT, + [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACING_BINARY_OP_ADD_INT, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACING_BINARY_OP_ADD_UNICODE, + [BINARY_OP_EXTEND] = _TAIL_CALL_TRACING_BINARY_OP_EXTEND, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACING_BINARY_OP_INPLACE_ADD_UNICODE, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_FLOAT, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_INT, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_DICT, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_GETITEM, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_INT, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_SLICE, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_STR_INT, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_TUPLE_INT, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_FLOAT, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_INT, + [BINARY_SLICE] = _TAIL_CALL_TRACING_BINARY_SLICE, + [BUILD_INTERPOLATION] = _TAIL_CALL_TRACING_BUILD_INTERPOLATION, + [BUILD_LIST] = _TAIL_CALL_TRACING_BUILD_LIST, + [BUILD_MAP] = _TAIL_CALL_TRACING_BUILD_MAP, + [BUILD_SET] = _TAIL_CALL_TRACING_BUILD_SET, + [BUILD_SLICE] = _TAIL_CALL_TRACING_BUILD_SLICE, + [BUILD_STRING] = _TAIL_CALL_TRACING_BUILD_STRING, + [BUILD_TEMPLATE] = _TAIL_CALL_TRACING_BUILD_TEMPLATE, + [BUILD_TUPLE] = _TAIL_CALL_TRACING_BUILD_TUPLE, + [CACHE] = _TAIL_CALL_TRACING_CACHE, + [CALL] = _TAIL_CALL_TRACING_CALL, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACING_CALL_ALLOC_AND_ENTER_INIT, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACING_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACING_CALL_BOUND_METHOD_GENERAL, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACING_CALL_BUILTIN_CLASS, + [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACING_CALL_BUILTIN_FAST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS, + [CALL_BUILTIN_O] = _TAIL_CALL_TRACING_CALL_BUILTIN_O, + [CALL_FUNCTION_EX] = _TAIL_CALL_TRACING_CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = _TAIL_CALL_TRACING_CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = _TAIL_CALL_TRACING_CALL_INTRINSIC_2, + [CALL_ISINSTANCE] = _TAIL_CALL_TRACING_CALL_ISINSTANCE, + [CALL_KW] = _TAIL_CALL_TRACING_CALL_KW, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACING_CALL_KW_BOUND_METHOD, + [CALL_KW_NON_PY] = _TAIL_CALL_TRACING_CALL_KW_NON_PY, + [CALL_KW_PY] = _TAIL_CALL_TRACING_CALL_KW_PY, + [CALL_LEN] = _TAIL_CALL_TRACING_CALL_LEN, + [CALL_LIST_APPEND] = _TAIL_CALL_TRACING_CALL_LIST_APPEND, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_O, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACING_CALL_NON_PY_GENERAL, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACING_CALL_PY_EXACT_ARGS, + [CALL_PY_GENERAL] = _TAIL_CALL_TRACING_CALL_PY_GENERAL, + [CALL_STR_1] = _TAIL_CALL_TRACING_CALL_STR_1, + [CALL_TUPLE_1] = _TAIL_CALL_TRACING_CALL_TUPLE_1, + [CALL_TYPE_1] = _TAIL_CALL_TRACING_CALL_TYPE_1, + [CHECK_EG_MATCH] = _TAIL_CALL_TRACING_CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = _TAIL_CALL_TRACING_CHECK_EXC_MATCH, + [CLEANUP_THROW] = _TAIL_CALL_TRACING_CLEANUP_THROW, + [COMPARE_OP] = _TAIL_CALL_TRACING_COMPARE_OP, + [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACING_COMPARE_OP_FLOAT, + [COMPARE_OP_INT] = _TAIL_CALL_TRACING_COMPARE_OP_INT, + [COMPARE_OP_STR] = _TAIL_CALL_TRACING_COMPARE_OP_STR, + [CONTAINS_OP] = _TAIL_CALL_TRACING_CONTAINS_OP, + [CONTAINS_OP_DICT] = _TAIL_CALL_TRACING_CONTAINS_OP_DICT, + [CONTAINS_OP_SET] = _TAIL_CALL_TRACING_CONTAINS_OP_SET, + [CONVERT_VALUE] = _TAIL_CALL_TRACING_CONVERT_VALUE, + [COPY] = _TAIL_CALL_TRACING_COPY, + [COPY_FREE_VARS] = _TAIL_CALL_TRACING_COPY_FREE_VARS, + [DELETE_ATTR] = _TAIL_CALL_TRACING_DELETE_ATTR, + [DELETE_DEREF] = _TAIL_CALL_TRACING_DELETE_DEREF, + [DELETE_FAST] = _TAIL_CALL_TRACING_DELETE_FAST, + [DELETE_GLOBAL] = _TAIL_CALL_TRACING_DELETE_GLOBAL, + [DELETE_NAME] = _TAIL_CALL_TRACING_DELETE_NAME, + [DELETE_SUBSCR] = _TAIL_CALL_TRACING_DELETE_SUBSCR, + [DICT_MERGE] = _TAIL_CALL_TRACING_DICT_MERGE, + [DICT_UPDATE] = _TAIL_CALL_TRACING_DICT_UPDATE, + [END_ASYNC_FOR] = _TAIL_CALL_TRACING_END_ASYNC_FOR, + [END_FOR] = _TAIL_CALL_TRACING_END_FOR, + [END_SEND] = _TAIL_CALL_TRACING_END_SEND, + [ENTER_EXECUTOR] = _TAIL_CALL_TRACING_ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = _TAIL_CALL_TRACING_EXIT_INIT_CHECK, + [EXTENDED_ARG] = _TAIL_CALL_TRACING_EXTENDED_ARG, + [FORMAT_SIMPLE] = _TAIL_CALL_TRACING_FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACING_FORMAT_WITH_SPEC, + [FOR_ITER] = _TAIL_CALL_TRACING_FOR_ITER, + [FOR_ITER_GEN] = _TAIL_CALL_TRACING_FOR_ITER_GEN, + [FOR_ITER_LIST] = _TAIL_CALL_TRACING_FOR_ITER_LIST, + [FOR_ITER_RANGE] = _TAIL_CALL_TRACING_FOR_ITER_RANGE, + [FOR_ITER_TUPLE] = _TAIL_CALL_TRACING_FOR_ITER_TUPLE, + [GET_AITER] = _TAIL_CALL_TRACING_GET_AITER, + [GET_ANEXT] = _TAIL_CALL_TRACING_GET_ANEXT, + [GET_AWAITABLE] = _TAIL_CALL_TRACING_GET_AWAITABLE, + [GET_ITER] = _TAIL_CALL_TRACING_GET_ITER, + [GET_LEN] = _TAIL_CALL_TRACING_GET_LEN, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACING_GET_YIELD_FROM_ITER, + [IMPORT_FROM] = _TAIL_CALL_TRACING_IMPORT_FROM, + [IMPORT_NAME] = _TAIL_CALL_TRACING_IMPORT_NAME, + [INSTRUMENTED_CALL] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL_KW, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACING_INSTRUMENTED_END_ASYNC_FOR, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACING_INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACING_INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACING_INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACING_INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACING_INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACING_INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = _TAIL_CALL_TRACING_INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACING_INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_ITER, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACING_INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACING_INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACING_INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = _TAIL_CALL_TRACING_INTERPRETER_EXIT, + [IS_OP] = _TAIL_CALL_TRACING_IS_OP, + [JUMP_BACKWARD] = _TAIL_CALL_TRACING_JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_JIT, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_NO_JIT, + [JUMP_FORWARD] = _TAIL_CALL_TRACING_JUMP_FORWARD, + [LIST_APPEND] = _TAIL_CALL_TRACING_LIST_APPEND, + [LIST_EXTEND] = _TAIL_CALL_TRACING_LIST_EXTEND, + [LOAD_ATTR] = _TAIL_CALL_TRACING_LOAD_ATTR, + [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACING_LOAD_ATTR_CLASS, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACING_LOAD_ATTR_INSTANCE_VALUE, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_LAZY_DICT, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_NO_DICT, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_WITH_VALUES, + [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACING_LOAD_ATTR_MODULE, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACING_LOAD_ATTR_PROPERTY, + [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACING_LOAD_ATTR_SLOT, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACING_LOAD_ATTR_WITH_HINT, + [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACING_LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACING_LOAD_COMMON_CONSTANT, + [LOAD_CONST] = _TAIL_CALL_TRACING_LOAD_CONST, + [LOAD_DEREF] = _TAIL_CALL_TRACING_LOAD_DEREF, + [LOAD_FAST] = _TAIL_CALL_TRACING_LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACING_LOAD_FAST_AND_CLEAR, + [LOAD_FAST_BORROW] = _TAIL_CALL_TRACING_LOAD_FAST_BORROW, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW, + [LOAD_FAST_CHECK] = _TAIL_CALL_TRACING_LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACING_LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = _TAIL_CALL_TRACING_LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACING_LOAD_GLOBAL_BUILTIN, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACING_LOAD_GLOBAL_MODULE, + [LOAD_LOCALS] = _TAIL_CALL_TRACING_LOAD_LOCALS, + [LOAD_NAME] = _TAIL_CALL_TRACING_LOAD_NAME, + [LOAD_SMALL_INT] = _TAIL_CALL_TRACING_LOAD_SMALL_INT, + [LOAD_SPECIAL] = _TAIL_CALL_TRACING_LOAD_SPECIAL, + [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR_ATTR, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR_METHOD, + [MAKE_CELL] = _TAIL_CALL_TRACING_MAKE_CELL, + [MAKE_FUNCTION] = _TAIL_CALL_TRACING_MAKE_FUNCTION, + [MAP_ADD] = _TAIL_CALL_TRACING_MAP_ADD, + [MATCH_CLASS] = _TAIL_CALL_TRACING_MATCH_CLASS, + [MATCH_KEYS] = _TAIL_CALL_TRACING_MATCH_KEYS, + [MATCH_MAPPING] = _TAIL_CALL_TRACING_MATCH_MAPPING, + [MATCH_SEQUENCE] = _TAIL_CALL_TRACING_MATCH_SEQUENCE, + [NOP] = _TAIL_CALL_TRACING_NOP, + [NOT_TAKEN] = _TAIL_CALL_TRACING_NOT_TAKEN, + [POP_EXCEPT] = _TAIL_CALL_TRACING_POP_EXCEPT, + [POP_ITER] = _TAIL_CALL_TRACING_POP_ITER, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACING_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACING_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACING_POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACING_POP_JUMP_IF_TRUE, + [POP_TOP] = _TAIL_CALL_TRACING_POP_TOP, + [PUSH_EXC_INFO] = _TAIL_CALL_TRACING_PUSH_EXC_INFO, + [PUSH_NULL] = _TAIL_CALL_TRACING_PUSH_NULL, + [RAISE_VARARGS] = _TAIL_CALL_TRACING_RAISE_VARARGS, + [RERAISE] = _TAIL_CALL_TRACING_RERAISE, + [RESERVED] = _TAIL_CALL_TRACING_RESERVED, + [RESUME] = _TAIL_CALL_TRACING_RESUME, + [RESUME_CHECK] = _TAIL_CALL_TRACING_RESUME_CHECK, + [RETURN_GENERATOR] = _TAIL_CALL_TRACING_RETURN_GENERATOR, + [RETURN_VALUE] = _TAIL_CALL_TRACING_RETURN_VALUE, + [SEND] = _TAIL_CALL_TRACING_SEND, + [SEND_GEN] = _TAIL_CALL_TRACING_SEND_GEN, + [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACING_SETUP_ANNOTATIONS, + [SET_ADD] = _TAIL_CALL_TRACING_SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACING_SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = _TAIL_CALL_TRACING_SET_UPDATE, + [STORE_ATTR] = _TAIL_CALL_TRACING_STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACING_STORE_ATTR_INSTANCE_VALUE, + [STORE_ATTR_SLOT] = _TAIL_CALL_TRACING_STORE_ATTR_SLOT, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACING_STORE_ATTR_WITH_HINT, + [STORE_DEREF] = _TAIL_CALL_TRACING_STORE_DEREF, + [STORE_FAST] = _TAIL_CALL_TRACING_STORE_FAST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACING_STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACING_STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = _TAIL_CALL_TRACING_STORE_GLOBAL, + [STORE_NAME] = _TAIL_CALL_TRACING_STORE_NAME, + [STORE_SLICE] = _TAIL_CALL_TRACING_STORE_SLICE, + [STORE_SUBSCR] = _TAIL_CALL_TRACING_STORE_SUBSCR, + [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACING_STORE_SUBSCR_DICT, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT, + [SWAP] = _TAIL_CALL_TRACING_SWAP, + [TO_BOOL] = _TAIL_CALL_TRACING_TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE, + [TO_BOOL_BOOL] = _TAIL_CALL_TRACING_TO_BOOL_BOOL, + [TO_BOOL_INT] = _TAIL_CALL_TRACING_TO_BOOL_INT, + [TO_BOOL_LIST] = _TAIL_CALL_TRACING_TO_BOOL_LIST, + [TO_BOOL_NONE] = _TAIL_CALL_TRACING_TO_BOOL_NONE, + [TO_BOOL_STR] = _TAIL_CALL_TRACING_TO_BOOL_STR, + [UNARY_INVERT] = _TAIL_CALL_TRACING_UNARY_INVERT, + [UNARY_NEGATIVE] = _TAIL_CALL_TRACING_UNARY_NEGATIVE, + [UNARY_NOT] = _TAIL_CALL_TRACING_UNARY_NOT, + [UNPACK_EX] = _TAIL_CALL_TRACING_UNPACK_EX, + [UNPACK_SEQUENCE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_LIST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TUPLE, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, + [WITH_EXCEPT_START] = _TAIL_CALL_TRACING_WITH_EXCEPT_START, + [YIELD_VALUE] = _TAIL_CALL_TRACING_YIELD_VALUE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [210] = _TAIL_CALL_UNKNOWN_OPCODE, + [211] = _TAIL_CALL_UNKNOWN_OPCODE, + [212] = _TAIL_CALL_UNKNOWN_OPCODE, + [213] = _TAIL_CALL_UNKNOWN_OPCODE, + [214] = _TAIL_CALL_UNKNOWN_OPCODE, + [215] = _TAIL_CALL_UNKNOWN_OPCODE, + [216] = _TAIL_CALL_UNKNOWN_OPCODE, + [217] = _TAIL_CALL_UNKNOWN_OPCODE, + [218] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, + [233] = _TAIL_CALL_UNKNOWN_OPCODE, +}; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer.c b/Python/optimizer.c index 6ad9124744859a..4d9e9e0ae4237f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -33,6 +33,9 @@ static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { + if (code == (PyCodeObject *)&_Py_InitCleanup) { + return false; + } if (instr->op.code == ENTER_EXECUTOR) { return true; } @@ -99,11 +102,11 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO } static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth); static int -uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, int curr_stackentries, +uop_optimize(_PyInterpreterFrame *frame, PyThreadState *tstate, + _PyExecutorObject **exec_ptr, bool progress_needed); /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. @@ -112,13 +115,13 @@ uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, // gh-137573: inlining this function causes stack overflows Py_NO_INLINE int _PyOptimizer_Optimize( - _PyInterpreterFrame *frame, _Py_CODEUNIT *start, - _PyExecutorObject **executor_ptr, int chain_depth) + _PyInterpreterFrame *frame, PyThreadState *tstate) { - _PyStackRef *stack_pointer = frame->stackpointer; PyInterpreterState *interp = _PyInterpreterState_GET(); + int chain_depth = tstate->interp->jit_state.initial_chain_depth; assert(interp->jit); assert(!interp->compiling); + assert(tstate->interp->jit_state.initial_stack_depth >= 0); #ifndef Py_GIL_DISABLED interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* @@ -127,18 +130,29 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = _PyFrame_GetCode(frame); - assert(PyCode_Check(code)); + PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_code; + _Py_CODEUNIT *start = tstate->interp->jit_state.insert_exec_instr; + // A recursive trace might've cleared the values. In that case, bail. + if (code == NULL) { + interp->compiling = false; + return 0; + } if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; } - int err = uop_optimize(frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)), progress_needed); + // One of our dependencies while tracing was invalidated. Not worth compiling. + if (!tstate->interp->jit_state.dependencies_still_valid) { + interp->compiling = false; + return 0; + } + _PyExecutorObject *executor; + int err = uop_optimize(frame, tstate, &executor, progress_needed); if (err <= 0) { interp->compiling = false; return err; } - assert(*executor_ptr != NULL); + assert(executor != NULL); if (progress_needed) { int index = get_index_for_executor(code, start); if (index < 0) { @@ -148,17 +162,22 @@ _PyOptimizer_Optimize( * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(*executor_ptr); + Py_DECREF(executor); interp->compiling = false; return 0; } - insert_executor(code, start, index, *executor_ptr); + insert_executor(code, start, index, executor); } else { - (*executor_ptr)->vm_data.code = NULL; + executor->vm_data.code = NULL; } - (*executor_ptr)->vm_data.chain_depth = chain_depth; - assert((*executor_ptr)->vm_data.valid); + if (chain_depth > 0) { + _PyExitData *prev_exit = tstate->interp->jit_state.previous_exit; + assert(prev_exit != NULL); + prev_exit->executor = executor;; + } + executor->vm_data.chain_depth = chain_depth; + assert(executor->vm_data.valid); interp->compiling = false; return 1; #else @@ -452,7 +471,6 @@ is_for_iter_test[MAX_UOP_ID + 1] = { [_GUARD_NOT_EXHAUSTED_RANGE] = 1, [_GUARD_NOT_EXHAUSTED_LIST] = 1, [_GUARD_NOT_EXHAUSTED_TUPLE] = 1, - [_FOR_ITER_TIER_TWO] = 1, }; static const uint16_t @@ -523,64 +541,40 @@ add_to_trace( DPRINTF(2, "No room for %s (need %d, got %d)\n", \ (opname), (n), max_length - trace_length); \ OPT_STAT_INC(trace_too_long); \ - goto done; \ + goto full; \ } // Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE #define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) -// Trace stack operations (used by _PUSH_FRAME, _RETURN_VALUE) -#define TRACE_STACK_PUSH() \ - if (trace_stack_depth >= TRACE_STACK_SIZE) { \ - DPRINTF(2, "Trace stack overflow\n"); \ - OPT_STAT_INC(trace_stack_overflow); \ - return 0; \ - } \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - trace_stack[trace_stack_depth].func = func; \ - trace_stack[trace_stack_depth].code = code; \ - trace_stack[trace_stack_depth].instr = instr; \ - trace_stack_depth++; -#define TRACE_STACK_POP() \ - if (trace_stack_depth <= 0) { \ - Py_FatalError("Trace stack underflow\n"); \ - } \ - trace_stack_depth--; \ - func = trace_stack[trace_stack_depth].func; \ - code = trace_stack[trace_stack_depth].code; \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - instr = trace_stack[trace_stack_depth].instr; - -/* Returns the length of the trace on success, - * 0 if it failed to produce a worthwhile trace, - * and -1 on an error. + +/* Returns 1 on success (added to trace), 0 on trace end. */ -static int -translate_bytecode_to_trace( +int +_PyJit_translate_single_bytecode_to_trace( + PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, - _PyUOpInstruction *trace, - int buffer_size, - _PyBloomFilter *dependencies, bool progress_needed) + _Py_CODEUNIT *this_instr, + _Py_CODEUNIT *next_instr, + PyCodeObject *old_code, + PyFunctionObject *func, + int old_stack_level, + int opcode, + int oparg, + int jump_taken) { - bool first = true; - PyCodeObject *code = _PyFrame_GetCode(frame); - PyFunctionObject *func = _PyFrame_GetFunction(frame); - assert(PyFunction_Check(func)); - PyCodeObject *initial_code = code; - _Py_BloomFilter_Add(dependencies, initial_code); - _Py_CODEUNIT *initial_instr = instr; - int trace_length = 0; - // Leave space for possible trailing _EXIT_TRACE - int max_length = buffer_size-2; - struct { - PyFunctionObject *func; - PyCodeObject *code; - _Py_CODEUNIT *instr; - } trace_stack[TRACE_STACK_SIZE]; - int trace_stack_depth = 0; - int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions - bool jump_seen = false; + + bool progress_needed = (tstate->interp->jit_state.initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; + _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; + _Py_BloomFilter_Add(dependencies, old_code); + int trace_length = tstate->interp->jit_state.code_curr_size; + _PyUOpInstruction *trace = tstate->interp->jit_state.code_buffer; + int max_length = tstate->interp->jit_state.code_max_size; + + int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (is_sys_tracing) { + goto full; + } #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -589,410 +583,356 @@ translate_bytecode_to_trace( lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif - - DPRINTF(2, - "Optimizing %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); - ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); - ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); + _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; - for (;;) { - target = INSTR_IP(instr, code); - // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length-=2; - uint32_t opcode = instr->op.code; - uint32_t oparg = instr->op.arg; - - if (!first && instr == initial_instr) { - // We have looped around to the start: - RESERVE(1); - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); - goto done; - } + target = INSTR_IP(target_instr, old_code); - DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + // Rewind EXTENDED_ARG so that we see the whole thing. + // We must point to the first EXTENDED_ARG when deopting. + int rewind_oparg = oparg; + while (rewind_oparg > 255) { + rewind_oparg >>= 8; + target--; + } +#ifdef Py_DEBUG + if (oparg > 255) { + assert(_Py_GetBaseCodeUnit(old_code, target).op.code == EXTENDED_ARG); + } +#endif - if (opcode == EXTENDED_ARG) { - instr++; - opcode = instr->op.code; - oparg = (oparg << 8) | instr->op.arg; - if (opcode == EXTENDED_ARG) { - instr--; - goto done; - } - } - if (opcode == ENTER_EXECUTOR) { - // We have a couple of options here. We *could* peek "underneath" - // this executor and continue tracing, which could give us a longer, - // more optimizeable trace (at the expense of lots of duplicated - // tier two code). Instead, we choose to just end here and stitch to - // the other trace, which allows a side-exit traces to rejoin the - // "main" trace periodically (and also helps protect us against - // pathological behavior where the amount of tier two code explodes - // for a medium-length, branchy code path). This seems to work - // better in practice, but in the future we could be smarter about - // what we do here: - goto done; - } - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - RESERVE_RAW(2, "_CHECK_VALIDITY"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - RESERVE_RAW(2, "_SET_IP"); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)instr, target); + if (!tstate->interp->jit_state.dependencies_still_valid) { + goto done; + } + + DPRINTF(2, "%p %d: %s(%d) %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, progress_needed); + + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; + + // Strange control-flow, unsupported opcode, etc. + if (jump_taken || + // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls + // If we haven't guarded the IP, then it's untraceable. + (frame != tstate->interp->jit_state.current_frame && !needs_guard_ip) || + (oparg > 0xFFFF) || + // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. + (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize)|| + // Exception stuff, could be handled in the future maybe? + opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || + frame->owner >= FRAME_OWNED_BY_INTERPRETER + ) { + unsupported: + { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 2) { + trace_length--; + curr = &trace[trace_length-1]; + } + assert(curr->opcode == _SET_IP || trace_length == 2); + if (curr->opcode == _SET_IP) { + int32_t old_target = (int32_t)uop_get_target(curr); + curr++; + trace_length++; + curr->opcode = _EXIT_TRACE; + curr->format = UOP_FORMAT_TARGET; + curr->target = old_target; + } + goto done; + } } - /* Special case the first instruction, - * so that we can guarantee forward progress */ - if (first && progress_needed) { - assert(first); - if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; - } - assert(!OPCODE_HAS_EXIT(opcode)); - assert(!OPCODE_HAS_DEOPT(opcode)); - } + tstate->interp->jit_state.current_frame = frame; - if (OPCODE_HAS_EXIT(opcode)) { - // Make space for side exit and final _EXIT_TRACE: - RESERVE_RAW(2, "_EXIT_TRACE"); - max_length--; - } - if (OPCODE_HAS_ERROR(opcode)) { - // Make space for error stub and final _EXIT_TRACE: - RESERVE_RAW(2, "_ERROR_POP_N"); - max_length--; - } - switch (opcode) { - case POP_JUMP_IF_NONE: - case POP_JUMP_IF_NOT_NONE: - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - { - RESERVE(1); - int counter = instr[1].cache; - int bitcount = _Py_popcount32(counter); - int jump_likely = bitcount > 8; - /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. - For values in between, adjust proportionally. */ - if (jump_likely) { - confidence = confidence * bitcount / 16; - } - else { - confidence = confidence * (16 - bitcount) / 16; - } - uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", - target, _PyOpcode_OpName[opcode], oparg, - counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); - if (confidence < CONFIDENCE_CUTOFF) { - DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF); - OPT_STAT_INC(low_confidence); - goto done; - } - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - _Py_CODEUNIT *target_instr = next_instr + oparg; - if (jump_likely) { - DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", - instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); - instr = target_instr; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); - goto top; - } - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); - break; - } + if (opcode == NOP) { + return 1; + } - case JUMP_BACKWARD: - case JUMP_BACKWARD_JIT: - ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); - _Py_FALLTHROUGH; - case JUMP_BACKWARD_NO_INTERRUPT: - { - instr += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] - (int)oparg; - if (jump_seen) { - OPT_STAT_INC(inner_loop); - DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); - goto done; - } - jump_seen = true; - goto top; - } + if (opcode == JUMP_FORWARD) { + return 1; + } - case JUMP_FORWARD: - { - RESERVE(0); - // This will emit two _SET_IP instructions; leave it to the optimizer - instr += oparg; - break; - } + if (opcode == EXTENDED_ARG) { + return 1; + } - case RESUME: - /* Use a special tier 2 version of RESUME_CHECK to allow traces to - * start with RESUME_CHECK */ - ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); - break; + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT + max_length -= 2; - default: - { - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (expansion->nuops > 0) { - // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) - int nuops = expansion->nuops; - RESERVE(nuops + 1); /* One extra for exit */ - int16_t last_op = expansion->uops[nuops-1].uop; - if (last_op == _RETURN_VALUE || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { - // Check for trace stack underflow now: - // We can't bail e.g. in the middle of - // LOAD_CONST + _RETURN_VALUE. - if (trace_stack_depth == 0) { - DPRINTF(2, "Trace stack underflow\n"); - OPT_STAT_INC(trace_stack_underflow); - return 0; - } - } - uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM - for (int i = 0; i < nuops; i++) { - oparg = orig_oparg; - uint32_t uop = expansion->uops[i].uop; - uint64_t operand = 0; - // Add one to account for the actual opcode/oparg pair: - int offset = expansion->uops[i].offset + 1; - switch (expansion->uops[i].size) { - case OPARG_SIMPLE: - assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); - break; - case OPARG_CACHE_1: - operand = read_u16(&instr[offset].cache); - break; - case OPARG_CACHE_2: - operand = read_u32(&instr[offset].cache); - break; - case OPARG_CACHE_4: - operand = read_u64(&instr[offset].cache); - break; - case OPARG_TOP: // First half of super-instr - oparg = orig_oparg >> 4; - break; - case OPARG_BOTTOM: // Second half of super-instr - oparg = orig_oparg & 0xF; - break; - case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset - oparg = offset; - assert(uop == _SAVE_RETURN_OFFSET); - break; - case OPARG_REPLACED: - uop = _PyUOp_Replacements[uop]; - assert(uop != 0); - uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); - if (uop == _TIER2_RESUME_CHECK) { - target = next_inst; - } -#ifdef Py_DEBUG - else { - uint32_t jump_target = next_inst + oparg; - assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); - } -#endif - break; - case OPERAND1_1: - assert(trace[trace_length-1].opcode == uop); - operand = read_u16(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_2: - assert(trace[trace_length-1].opcode == uop); - operand = read_u32(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_4: - assert(trace[trace_length-1].opcode == uop); - operand = read_u64(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - default: - fprintf(stderr, - "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", - opcode, oparg, nuops, i, - expansion->uops[i].size, - expansion->uops[i].offset); - Py_FatalError("garbled expansion"); - } + if (opcode == ENTER_EXECUTOR) { + goto full; + } - if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { - TRACE_STACK_POP(); - /* Set the operand to the function or code object returned to, - * to assist optimization passes. (See _PUSH_FRAME below.) - */ - if (func != NULL) { - operand = (uintptr_t)func; - } - else if (code != NULL) { - operand = (uintptr_t)code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - DPRINTF(2, - "Returning to %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (uop == _PUSH_FRAME) { - assert(i + 1 == nuops); - if (opcode == FOR_ITER_GEN || - opcode == LOAD_ATTR_PROPERTY || - opcode == BINARY_OP_SUBSCR_GETITEM || - opcode == SEND_GEN) - { - DPRINTF(2, "Bailing due to dynamic target\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); - int func_version_offset = - offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) - // Add one to account for the actual opcode/oparg pair: - + 1; - uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyCodeObject *new_code = NULL; - PyFunctionObject *new_func = - _PyFunction_LookupByVersion(func_version, (PyObject **) &new_code); - DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n", - (int)func_version, new_func, new_code); - if (new_code != NULL) { - if (new_code == code) { - // Recursive call, bail (we could be here forever). - DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", - PyUnicode_AsUTF8(new_code->co_qualname), - PyUnicode_AsUTF8(new_code->co_filename), - new_code->co_firstlineno); - OPT_STAT_INC(recursive_call); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - if (new_code->co_version != func_version) { - // func.__code__ was updated. - // Perhaps it may happen again, so don't bother tracing. - // TODO: Reason about this -- is it better to bail or not? - DPRINTF(2, "Bailing because co_version != func_version\n"); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - // Increment IP to the return address - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; - TRACE_STACK_PUSH(); - _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's function or code object, - * to assist optimization passes. - * We prefer setting it to the function - * but if that's not available but the code is available, - * use the code, setting the low bit so the optimizer knows. - */ - if (new_func != NULL) { - operand = (uintptr_t)new_func; - } - else if (new_code != NULL) { - operand = (uintptr_t)new_code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - code = new_code; - func = new_func; - instr = _PyCode_CODE(code); - DPRINTF(2, - "Continuing in %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - DPRINTF(2, "Bail, new_code == NULL\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { - assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - assert(next_instr->op.code == STORE_FAST); - operand = next_instr->op.arg; - // Skip the STORE_FAST: - instr++; - } + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + assert(!_PyErr_Occurred(tstate)); - // All other instructions - ADD_TO_TRACE(uop, oparg, operand, target); - } - break; - } - DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); - OPT_UNSUPPORTED_OPCODE(opcode); - goto done; // Break out of loop - } // End default + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + } + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed && tstate->interp->jit_state.code_curr_size <= 2) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } + + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for side exit and final _EXIT_TRACE: + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error stub and final _EXIT_TRACE: + max_length--; + } - } // End switch (opcode) + RESERVE_RAW(expansion->nuops + needs_guard_ip + 4, "uop and various checks"); - instr++; - // Add cache size for opcode - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - if (opcode == CALL_LIST_APPEND) { - assert(instr->op.code == POP_TOP); - instr++; + switch (opcode) { + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + { + _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); + _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; + assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); + int jump_happened = computed_jump_instr == next_instr; + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + break; + } + case JUMP_BACKWARD_JIT: + // This is possible as the JIT might have re-activated after it was disabled + case JUMP_BACKWARD_NO_JIT: + case JUMP_BACKWARD: + ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); + _Py_FALLTHROUGH; + case JUMP_BACKWARD_NO_INTERRUPT: + { + if ((next_instr != tstate->interp->jit_state.close_loop_instr) && + (next_instr != tstate->interp->jit_state.insert_exec_instr) && + tstate->interp->jit_state.code_curr_size > 5) { + // We encountered a JUMP_BACKWARD but not to the top of our own loop. + // We don't want to continue tracing as we might get stuck in the + // inner loop. Instead, end the trace where the executor of the + // inner loop might start and let the traces rejoin. + OPT_STAT_INC(inner_loop); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, tstate->interp->jit_state.close_loop_instr, tstate->interp->jit_state.insert_exec_instr); + goto done; + } + break; } - top: - // Jump here after _PUSH_FRAME or likely branches. - first = false; - } // End for (;;) -done: - while (trace_stack_depth > 0) { - TRACE_STACK_POP(); - } - assert(code == initial_code); - // Skip short traces where we can't even translate a single instruction: - if (first) { - OPT_STAT_INC(trace_too_short); - DPRINTF(2, - "No trace for %s (%s:%d) at byte offset %d (no progress)\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); - return 0; + case RESUME: + case RESUME_CHECK: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + + default: + { + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) + int nuops = expansion->nuops; + if (nuops == 0) { + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; + } + assert(nuops > 0); + RESERVE(nuops + 1); /* One extra for exit */ + uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM + uint32_t orig_target = target; + for (int i = 0; i < nuops; i++) { + oparg = orig_oparg; + target = orig_target; + uint32_t uop = expansion->uops[i].uop; + uint64_t operand = 0; + // Add one to account for the actual opcode/oparg pair: + int offset = expansion->uops[i].offset + 1; + switch (expansion->uops[i].size) { + case OPARG_SIMPLE: + assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); + break; + case OPARG_CACHE_1: + operand = read_u16(&this_instr[offset].cache); + break; + case OPARG_CACHE_2: + operand = read_u32(&this_instr[offset].cache); + break; + case OPARG_CACHE_4: + operand = read_u64(&this_instr[offset].cache); + break; + case OPARG_TOP: // First half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg >> 4; + break; + case OPARG_BOTTOM: // Second half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg & 0xF; + break; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); + break; + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); + + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; + } + else if (uop != _FOR_ITER_TIER_TWO) { + int extended_arg = orig_oparg > 255; + uint32_t jump_target = next_inst + orig_oparg + extended_arg; + assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); + if (is_for_iter_test[uop]) { + target = jump_target + 1; + } + } + break; + case OPERAND1_1: + assert(trace[trace_length-1].opcode == uop); + operand = read_u16(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_2: + assert(trace[trace_length-1].opcode == uop); + operand = read_u32(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_4: + assert(trace[trace_length-1].opcode == uop); + operand = read_u64(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + default: + fprintf(stderr, + "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", + opcode, oparg, nuops, i, + expansion->uops[i].size, + expansion->uops[i].offset); + Py_FatalError("garbled expansion"); + } + if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyFunctionObject *new_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (new_func != NULL) { + operand = (uintptr_t)new_func; + DPRINTF(2, "Adding %p func to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_func); + } + else if (new_code != NULL) { + operand = (uintptr_t)new_code | 1; + DPRINTF(2, "Adding %p code to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_code); + } + else { + operand = 0; + } + } + if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { + assert(i + 1 == nuops); + _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next_instr->op.code == STORE_FAST); + operand = next_instr->op.arg; + } + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); + } + break; + } // End default + + } // End switch (opcode) + + if (needs_guard_ip) { + ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)next_instr, 0); } - if (!is_terminator(&trace[trace_length-1])) { - /* Allow space for _EXIT_TRACE */ - max_length += 2; + // Loop back to the start + int is_first_instr = tstate->interp->jit_state.close_loop_instr == next_instr || tstate->interp->jit_state.insert_exec_instr == next_instr; + if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) { + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; + } + tstate->interp->jit_state.code_curr_size = trace_length; + tstate->interp->jit_state.code_max_size = max_length; + return 1; +done: + tstate->interp->jit_state.code_curr_size = trace_length; + tstate->interp->jit_state.code_max_size = max_length; + return 0; +full: + if (!is_terminator(&tstate->interp->jit_state.code_buffer[trace_length-1])) { + // Undo the last few instructions. + trace_length = tstate->interp->jit_state.code_curr_size; + max_length = tstate->interp->jit_state.code_max_size; + // We previously reversed one. + max_length += 1; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); } - DPRINTF(1, - "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code), - trace_length); - OPT_HIST(trace_length, trace_length_hist); - return trace_length; + tstate->interp->jit_state.code_curr_size = trace_length; + tstate->interp->jit_state.code_max_size = max_length; + return 0; +} + +void +_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit) +{ + PyCodeObject *code = _PyFrame_GetCode(frame); +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 0; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + DPRINTF(2, + "Tracing %s (%s:%d) at byte offset %d at chain depth %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(close_loop_instr, code), + chain_depth); +#endif + add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); + add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + tstate->interp->jit_state.code_curr_size = 2; + tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH; + tstate->interp->jit_state.insert_exec_instr = insert_exec_instr; + tstate->interp->jit_state.close_loop_instr = close_loop_instr; + tstate->interp->jit_state.initial_code = (PyCodeObject *)Py_NewRef(code); + tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); + tstate->interp->jit_state.previous_exit = exit; + _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); + tstate->interp->jit_state.initial_stack_depth = curr_stackdepth; + tstate->interp->jit_state.initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; + tstate->interp->jit_state.current_frame = frame; + tstate->interp->jit_state.dependencies_still_valid = true; + tstate->interp->jit_state.last_specialized_instr = NULL; +} + +void +_PyJit_FinalizeTracing(PyThreadState *tstate) +{ + Py_CLEAR(tstate->interp->jit_state.initial_code); + Py_CLEAR(tstate->interp->jit_state.initial_func); + tstate->interp->jit_state.code_curr_size = 2; + tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } + #undef RESERVE #undef RESERVE_RAW #undef INSTR_IP @@ -1011,7 +951,7 @@ count_exits(_PyUOpInstruction *buffer, int length) int exit_count = 0; for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { exit_count++; } } @@ -1057,6 +997,9 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; int opcode = inst->opcode; + if (inst->format != UOP_FORMAT_TARGET) { + fprintf(stdout, "I: %d\n", i); + } int32_t target = (int32_t)uop_get_target(inst); uint16_t exit_flags = _PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG | HAS_PERIODIC_FLAG); if (exit_flags) { @@ -1068,14 +1011,15 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) exit_op = _HANDLE_PENDING_AND_DEOPT; } int32_t jump_target = target; - if (is_for_iter_test[opcode]) { - /* Target the POP_TOP immediately after the END_FOR, - * leaving only the iterator on the stack. */ - int extended_arg = inst->oparg > 255; - int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; - jump_target = next_inst + inst->oparg + 1; + bool unique_target = false; + if (opcode == _FOR_ITER_TIER_TWO) { + exit_op = _DYNAMIC_EXIT; + } + else if (opcode == _GUARD_IP) { + exit_op = _DYNAMIC_EXIT; + unique_target = true; } - if (jump_target != current_jump_target || current_exit_op != exit_op) { + if (unique_target || jump_target != current_jump_target || current_exit_op != exit_op) { make_exit(&buffer[next_spare], exit_op, jump_target); current_exit_op = exit_op; current_jump_target = jump_target; @@ -1182,7 +1126,8 @@ sanity_check(_PyExecutorObject *executor) opcode == _DEOPT || opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || - opcode == _ERROR_POP_N); + opcode == _ERROR_POP_N || + opcode == _DYNAMIC_EXIT); } } @@ -1195,7 +1140,7 @@ sanity_check(_PyExecutorObject *executor) * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth) { int exit_count = count_exits(buffer, length); _PyExecutorObject *executor = allocate_executor(exit_count, length); @@ -1205,6 +1150,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil /* Initialize exits */ _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + cold->vm_data.chain_depth = chain_depth; for (int i = 0; i < exit_count; i++) { executor->exits[i].index = i; executor->exits[i].temperature = initial_temperature_backoff_counter(); @@ -1218,8 +1164,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil int opcode = buffer[i].opcode; dest--; *dest = buffer[i]; - assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; @@ -1284,38 +1229,32 @@ int effective_trace_length(_PyUOpInstruction *buffer, int length) static int uop_optimize( _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, + PyThreadState *tstate, _PyExecutorObject **exec_ptr, - int curr_stackentries, bool progress_needed) { - _PyBloomFilter dependencies; - _Py_BloomFilter_Init(&dependencies); + _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->jit_uop_buffer == NULL) { - interp->jit_uop_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - if (interp->jit_uop_buffer == NULL) { - return 0; - } - } - _PyUOpInstruction *buffer = interp->jit_uop_buffer; + _PyUOpInstruction *buffer = interp->jit_state.code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); - if (length <= 0) { - // Error or nothing translated - return length; + int curr_stackentries = tstate->interp->jit_state.initial_stack_depth; + int length = interp->jit_state.code_curr_size; + // Trace too short, don't bother. + if (length <= 5) { + return 0; } + assert(length > 0); assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(frame, buffer, + length = _Py_uop_analyze_and_optimize(tstate->interp->jit_state.initial_func, buffer, length, - curr_stackentries, &dependencies); + curr_stackentries, dependencies); if (length <= 0) { return length; } @@ -1338,7 +1277,7 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, dependencies, tstate->interp->jit_state.initial_chain_depth); if (executor == NULL) { return -1; } @@ -1633,6 +1572,18 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_Executors_InvalidateAll(interp, is_invalidation); } +void +_PyJit_Tracer_InvalidateDependency(PyThreadState *tstate, void *obj) +{ + _PyBloomFilter obj_filter; + _Py_BloomFilter_Init(&obj_filter); + _Py_BloomFilter_Add(&obj_filter, obj); + + if (bloom_filter_may_contain(&tstate->interp->jit_state.dependencies, &obj_filter)) + { + tstate->interp->jit_state.dependencies_still_valid = false; + } +} /* Invalidate all executors */ void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) @@ -1772,6 +1723,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) fprintf(out, "]\n\n"); /* Write all the outgoing edges */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); for (uint32_t i = 0; i < executor->code_size; i++) { _PyUOpInstruction const *inst = &executor->trace[i]; uint16_t flags = _PyUop_Flags[inst->opcode]; @@ -1782,10 +1734,10 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) else if (flags & HAS_EXIT_FLAG) { assert(inst->format == UOP_FORMAT_JUMP); _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; - assert(exit_inst->opcode == _EXIT_TRACE); + assert(exit_inst->opcode == _EXIT_TRACE || exit_inst->opcode == _DYNAMIC_EXIT); exit = (_PyExitData *)exit_inst->operand0; } - if (exit != NULL && exit->executor != NULL) { + if (exit != NULL && exit->executor != cold) { fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); } if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a6add301ccb26c..2d3f1d95d5ab14 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -493,7 +493,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } /* _PUSH_FRAME doesn't escape or error, but it * does need the IP for the return address */ - bool needs_ip = opcode == _PUSH_FRAME; + bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT); if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; @@ -507,6 +507,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } case _JUMP_TO_TOP: case _EXIT_TRACE: + case _DYNAMIC_EXIT: return pc + 1; } } @@ -518,7 +519,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) // > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( - _PyInterpreterFrame *frame, + PyFunctionObject *initial_func, _PyUOpInstruction *buffer, int length, int curr_stacklen, @@ -528,8 +529,8 @@ _Py_uop_analyze_and_optimize( OPT_STAT_INC(optimizer_attempts); length = optimize_uops( - _PyFrame_GetFunction(frame), buffer, - length, curr_stacklen, dependencies); + initial_func, buffer, + length, curr_stacklen, dependencies); if (length == 0) { return length; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index da3d3c96bc1d97..09c5ae764e5ac3 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -775,7 +775,9 @@ dummy_func(void) { SAVE_STACK(); PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + if (frame_pop(ctx)) { + break; + } stack_pointer = ctx->frame->stack_pointer; /* Stack space handling */ @@ -797,7 +799,6 @@ dummy_func(void) { frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - /* Stack space handling */ assert(corresponding_check_stack == NULL); assert(co != NULL); @@ -808,7 +809,14 @@ dummy_func(void) { } op(_YIELD_VALUE, (unused -- value)) { + // TODO (gh-139109): handle this properly in a future optimization. + // A possibility to handle underflows is to just restore the current frame information + // from whatever is stored in the trace we record at that point of time. + // E.g. we record at this YIELD_VALUE, func_obj=x , stack_level=4 + // We can restore it to there. value = sym_new_unknown(ctx); + ctx->done = true; + ctx->out_of_space = true; } op(_GET_ITER, (iterable -- iter, index_or_null)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b08099d8e2fc3b..002da75ea38481 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1118,7 +1118,9 @@ assert(WITHIN_STACK_BOUNDS()); PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + if (frame_pop(ctx)) { + break; + } stack_pointer = ctx->frame->stack_pointer; assert(corresponding_check_stack == NULL); assert(co != NULL); @@ -1169,6 +1171,8 @@ case _YIELD_VALUE: { JitOptRef value; value = sym_new_unknown(ctx); + ctx->done = true; + ctx->out_of_space = true; stack_pointer[-1] = value; break; } @@ -2103,6 +2107,8 @@ break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 */ + case _GET_LEN: { JitOptRef obj; JitOptRef len; @@ -3418,3 +3424,11 @@ break; } + case _GUARD_IP: { + break; + } + + case _DYNAMIC_EXIT: { + break; + } + diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 01cff0b014cc7b..b0997f56b98b94 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -817,7 +817,11 @@ _Py_uop_frame_new( JitOptRef *args, int arg_len) { - assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + if (ctx->curr_frame_depth >= MAX_ABSTRACT_FRAME_DEPTH) { + ctx->done = true; + ctx->out_of_space = true; + return NULL; + } _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; frame->stack_len = co->co_stacksize; @@ -906,7 +910,12 @@ _Py_uop_frame_pop(JitOptContext *ctx) _Py_UOpsAbstractFrame *frame = ctx->frame; ctx->n_consumed = frame->locals; ctx->curr_frame_depth--; - assert(ctx->curr_frame_depth >= 1); + // TODO gh-139109: Handle trace recording underflow + if (ctx->curr_frame_depth == 0) { + ctx->done = true; + ctx->out_of_space = true; + return 1; + } ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; return 0; diff --git a/Python/pystate.c b/Python/pystate.c index af7828d6a030ab..ebb37fc8400eb2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -556,8 +556,11 @@ init_interpreter(PyInterpreterState *interp, #endif #ifdef _Py_TIER2 - // Ensure the buffer is to be set as NULL. - interp->jit_uop_buffer = NULL; + interp->jit_state.code_buffer = NULL; + interp->jit_state.initial_stack_depth = -1; + interp->jit_state.initial_chain_depth = -1; + interp->jit_state.initial_code = NULL; + interp->jit_state.initial_func = NULL; #endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); @@ -808,9 +811,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #ifdef _Py_TIER2 _Py_ClearExecutorDeletionList(interp); - if (interp->jit_uop_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_uop_buffer, UOP_BUFFER_SIZE); - interp->jit_uop_buffer = NULL; + if (interp->jit_state.code_buffer != NULL) { + _PyObject_VirtualFree(interp->jit_state.code_buffer, UOP_BUFFER_SIZE); + interp->jit_state.code_buffer = NULL; } #endif _PyAST_Fini(interp); diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index fd198d7d06c96f..0f944d45a4fc88 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -78,6 +78,7 @@ def format_tsv_lines(lines): 'Python/deepfreeze/*.c', 'Python/frozen_modules/*.h', 'Python/generated_cases.c.h', + 'Python/generated_tracer_cases.c.h', 'Python/executor_cases.c.h', 'Python/optimizer_cases.c.h', 'Python/opcode_targets.h', diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 9dd7e5dbfbae7b..7a5e8786f2a804 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -34,6 +34,7 @@ class Properties: side_exit: bool pure: bool uses_opcode: bool + needs_guard_ip: bool tier: int | None = None const_oparg: int = -1 needs_prev: bool = False @@ -75,6 +76,7 @@ def from_list(properties: list["Properties"]) -> "Properties": pure=all(p.pure for p in properties), needs_prev=any(p.needs_prev for p in properties), no_save_ip=all(p.no_save_ip for p in properties), + needs_guard_ip=any(p.needs_guard_ip for p in properties) ) @property @@ -102,6 +104,7 @@ def infallible(self) -> bool: side_exit=False, pure=True, no_save_ip=False, + needs_guard_ip=False, ) @@ -692,6 +695,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Wrap", "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", + "_PyExecutor_FromExit", + "_PyJit_InitializeTracing", ) @@ -932,6 +937,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), + needs_guard_ip=variable_used(op, "TIER2_STORE_IP") or variable_used(op, "LLTRACE_RESUME_FRAME") or variable_used(op, "DISPATCH_INLINED"), ) def expand(items: list[StackItem], oparg: int) -> list[StackItem]: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 61e855eb003706..3edefb110834a9 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -107,8 +107,9 @@ class Emitter: labels: dict[str, Label] _replacers: dict[str, ReplacementFunctionType] cannot_escape: bool + jump_prefix: str - def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False, jump_prefix: str = ""): self._replacers = { "EXIT_IF": self.exit_if, "AT_END_EXIT_IF": self.exit_if_after, @@ -127,10 +128,12 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, + "RECORD_DYNAMIC_JUMP_TAKEN": self.record_dynamic_jump_taken, } self.out = out self.labels = labels self.cannot_escape = cannot_escape + self.jump_prefix = jump_prefix def dispatch( self, @@ -167,7 +170,7 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"JUMP_TO_PREDICTED({family_name});\n") + self.emit(f"JUMP_TO_PREDICTED({self.jump_prefix}{family_name});\n") self.emit("}\n") return not always_true(first_tkn) @@ -198,10 +201,10 @@ def exit_if_after( def goto_error(self, offset: int, storage: Storage) -> str: if offset > 0: - return f"JUMP_TO_LABEL(pop_{offset}_error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(pop_{offset}_error);" if offset < 0: storage.copy().flush(self.out) - return f"JUMP_TO_LABEL(error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(error);" def error_if( self, @@ -421,7 +424,7 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) self.out.start_line() - self.out.emit("JUMP_TO_LABEL(") + self.out.emit(f"{self.jump_prefix}JUMP_TO_LABEL(") self.out.emit(label) self.out.emit(")") @@ -473,6 +476,19 @@ def instruction_size(self, self.out.emit(f" {uop.instruction_size}u ") return True + def record_dynamic_jump_taken( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter); + next(tkn_iter); + next(tkn_iter); + return True + def _print_storage(self, reason:str, storage: Storage) -> None: if DEBUG: self.out.start_line() diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index b649b38123388d..d17a2431c75b37 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -184,6 +184,16 @@ def generate_cache_table(analysis: Analysis, out: CWriter) -> None: out.emit("#endif\n\n") +def generate_needs_guard_ip_table(analysis: Analysis, out: CWriter) -> None: + out.emit("extern const uint8_t _PyOpcode_NeedsGuardIp[256];\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const uint8_t _PyOpcode_NeedsGuardIp[256] = {\n") + for inst in analysis.instructions.values(): + if inst.properties.needs_guard_ip: + out.emit(f"[{inst.name}] = 1,\n") + out.emit("};\n") + out.emit("#endif\n\n") + def generate_name_table(analysis: Analysis, out: CWriter) -> None: table_size = 256 + len(analysis.pseudos) out.emit(f"extern const char *_PyOpcode_OpName[{table_size}];\n") @@ -382,6 +392,7 @@ def generate_opcode_metadata( generate_expansion_table(analysis, out) generate_name_table(analysis, out) generate_cache_table(analysis, out) + generate_needs_guard_ip_table(analysis, out) generate_deopt_table(analysis, out) generate_extra_cases(analysis, out) generate_pseudo_targets(analysis, out) diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 324ef2773abe28..8e5dc10f759a15 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -31,6 +31,16 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: for target in targets: out.emit(target) out.emit("};\n") + targets = ["&&_unknown_opcode,\n"] * 256 + for name, op in analysis.opmap.items(): + if op < 256: + targets[op] = f"&&TARGET_TRACING_{name},\n" + out.emit("#if _Py_TIER2\n") + out.emit("static void *opcode_tracing_targets_table[256] = {\n") + for target in targets: + out.emit(target) + out.emit("};\n") + out.emit(f"#endif\n") out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") def function_proto(name: str) -> str: @@ -38,7 +48,9 @@ def function_proto(name: str) -> str: def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256];\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") + out.emit("\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") out.emit("\n") # Emit function prototypes for labels. @@ -49,6 +61,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: # Emit function prototypes for opcode handlers. for name in sorted(analysis.instructions.keys()): out.emit(f"{function_proto(name)};\n") + out.emit(f"{function_proto('TRACING_' + name)};\n") out.emit("\n") # Emit unknown opcode handler. @@ -60,7 +73,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit the dispatch table. - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256] = {\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") named_values = analysis.opmap.values() @@ -68,6 +81,16 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: if rest not in named_values: out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") out.emit("};\n") + + # Emit the tracing dispatch table. + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_TRACING_{name},\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") + out.emit("};\n") outfile.write("#endif /* _Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 94ffb0118f0786..25a7442a0ef611 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -167,7 +167,9 @@ def generate_tier1( {INSTRUCTION_START_MARKER} """ ) - generate_tier1_cases(analysis, outfile, lines) + out = CWriter(outfile, 2, lines) + emitter = Emitter(out, analysis.labels) + generate_tier1_cases(analysis, out, emitter) outfile.write(f""" {INSTRUCTION_END_MARKER} #if !_Py_TAIL_CALL_INTERP @@ -215,10 +217,8 @@ def get_popped(inst: Instruction, analysis: Analysis) -> str: return (-stack.base_offset).to_c() def generate_tier1_cases( - analysis: Analysis, outfile: TextIO, lines: bool + analysis: Analysis, out: CWriter, emitter: Emitter ) -> None: - out = CWriter(outfile, 2, lines) - emitter = Emitter(out, analysis.labels) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): out.emit("\n") @@ -265,7 +265,7 @@ def generate_tier1_cases( out.start_line() if reachable: # type: ignore[possibly-undefined] stack.flush(out) - out.emit("DISPATCH();\n") + out.emit(f"DISPATCH();\n") out.start_line() out.emit("}") out.emit("\n") diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 1bb5f48658ddfc..87b26b1c732599 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,6 +63,8 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg + self._replacers["JUMPBY"] = self.jumpby + self._replacers["DISPATCH"] = self.dispatch def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -134,6 +136,38 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True + def jumpby( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.emit("TIER2_STORE_IP") + emit_to(self.out, tkn_iter, "SEMI") + self.emit(";\n") + return True + + def dispatch( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.emit("break;\n") + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + return False def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: locals: dict[str, Local] = {} diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py new file mode 100644 index 00000000000000..1b7d9a5acea6bc --- /dev/null +++ b/Tools/cases_generator/tracer_generator.py @@ -0,0 +1,222 @@ +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + Label, + CodeSection, + Part, + analyze_files, + Skip, + Flush, + analysis_error, + StackItem, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + type_and_null, + Emitter, + TokenIterator, + always_true, + emit_to, + ReplacementFunctionType, +) +from cwriter import CWriter +from typing import TextIO +from lexer import Token +from stack import Local, Stack, StackError, get_stack_effect, Storage +from tier1_generator import get_popped, declare_variables, write_uop + +DEFAULT_OUTPUT = ROOT / "Python/generated_tracer_cases.c.h" + +class TracerEmitter(Emitter): + out: CWriter + labels: dict[str, Label] + _replacers: dict[str, ReplacementFunctionType] + cannot_escape: bool + + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): + super().__init__(out, labels, cannot_escape, jump_prefix="TRACING_") + self._replacers = { + **self._replacers, + "DISPATCH": self.dispatch, + "DISPATCH_INLINED": self.dispatch_inlined, + "DISPATCH_SAME_OPARG": self.dispatch_same_oparg, + } + + def dispatch( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.out.start_line() + self.emit("TRACING_DISPATCH") + return False + + def dispatch_inlined( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.out.start_line() + self.emit("TRACING_DISPATCH_INLINED") + return False + + def dispatch_same_oparg( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.out.start_line() + if isinstance(uop, Uop) and "specializing" in uop.annotations: + self.emit("TRACING_SPECIALIZE_DISPATCH_SAME_OPARG") + else: + self.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + return False + + def record_dynamic_jump_taken( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + self.out.emit(";\n") + return True + +def generate_tier1_tracer_cases( + analysis: Analysis, out: CWriter, emitter: Emitter +) -> None: + out.emit("\n") + for name, inst in sorted(analysis.instructions.items()): + out.emit("\n") + out.emit(f"TRACING_TARGET({name}) {{\n") + out.emit(f"assert(IS_JIT_TRACING());\n") + # We need to ifdef it because this breaks platforms + # without computed gotos/tail calling. + out.emit(f"#if _Py_TAIL_CALL_INTERP\n") + out.emit(f"int opcode = {name};\n") + out.emit(f"(void)(opcode);\n") + out.emit(f"#endif\n") + unused_guard = "(void)this_instr;\n" + if inst.properties.needs_prev: + out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") + if not inst.is_target: + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") + out.emit(unused_guard) + if not inst.properties.no_save_ip: + out.emit(f"frame->instr_ptr = next_instr;\n") + + out.emit(f"next_instr += {inst.size};\n") + out.emit(f"INSTRUCTION_STATS({name});\n") + if inst.is_target: + out.emit(f"PREDICTED_TRACING_{name}:;\n") + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") + out.emit(unused_guard) + # This is required so that the predicted ops reflect the correct opcode. + out.emit(f"opcode = {name};\n") + out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") + out.emit(f"(void)old_code;\n") + out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") + out.emit(f"(void)old_func;\n") + out.emit(f"int _jump_taken = false;\n") + out.emit(f"(void)_jump_taken;\n") + out.emit(f"int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0;\n") + out.emit(f"(void)(_old_stack_level);\n") + if inst.family is not None: + out.emit( + f"static_assert({inst.family.size} == {inst.size-1}" + ', "incorrect cache size");\n' + ) + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for part in inst.parts: + # Only emit braces if more than one uop + insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 + reachable, offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) + out.start_line() + if reachable: # type: ignore[possibly-undefined] + stack.flush(out) + out.emit(f"TRACING_DISPATCH();\n") + out.start_line() + out.emit("}") + out.emit("\n") + + +def generate_tracer_cases( + analysis: Analysis, out: CWriter +) -> None: + out.emit(f"#ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */\n") + generate_tier1_tracer_cases(analysis, out, TracerEmitter(out, analysis.labels)) + out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") + +def generate_tracer( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 2, lines) + out.emit("#define TIER_ONE 1\n") + out.emit("#define TRACING_JIT 1\n") + generate_tracer_cases(analysis, out) + out.emit("#undef TRACING_JIT\n") + out.emit("#undef TIER_ONE\n") + +# For use in unittest +def generate_tracer_from_files( + filenames: list[str], outfilename: str, lines: bool +) -> None: + data = analyze_files(filenames) + with open(outfilename, "w") as outfile: + generate_tracer(filenames, data, outfile, lines) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tracer(args.input, data, outfile, args.emit_line_directives) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 8f71010a1aff58..471e052c2d8d52 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -56,11 +56,11 @@ do { \ } while (0) #undef GOTO_TIER_ONE -#define GOTO_TIER_ONE(TARGET) \ +#define GOTO_TIER_ONE(TARGET, SHOULD_CONTINUE_TRACING) \ do { \ tstate->current_executor = NULL; \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET)) | SHOULD_CONTINUE_TRACING); \ } while (0) #undef LOAD_IP @@ -69,9 +69,11 @@ do { \ } while (0) #undef LLTRACE_RESUME_FRAME -#define LLTRACE_RESUME_FRAME() \ - do { \ - } while (0) +#ifdef Py_DEBUG +#define LLTRACE_RESUME_FRAME() frame->lltrace = 0; +#else +#define LLTRACE_RESUME_FRAME() +#endif #define PATCH_JUMP(ALIAS) \ do { \