diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 01c98c36f052a9..a9a438bccd5c8b 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -278,9 +278,13 @@ PyJitRef_IsBorrowed(JitOptRef ref) } struct _Py_UOpsAbstractFrame { + bool globals_watched; + // The version number of the globals dicts, once checked. 0 if unchecked. + uint32_t globals_checked_version; // Max stacklen int stack_len; int locals_len; + PyFunctionObject *func; JitOptRef *stack_pointer; JitOptRef *stack; @@ -299,6 +303,8 @@ typedef struct _JitOptContext { char done; char out_of_space; bool contradiction; + // Has the builtins dict been watched? + bool builtins_watched; // The current "executing" frame. _Py_UOpsAbstractFrame *frame; _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 749369a40aecdd..ff1d75c0cb1938 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -64,119 +64,118 @@ extern "C" { #define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 343 -#define _CHECK_FUNCTION_EXACT_ARGS 344 -#define _CHECK_FUNCTION_VERSION 345 -#define _CHECK_FUNCTION_VERSION_INLINE 346 -#define _CHECK_FUNCTION_VERSION_KW 347 -#define _CHECK_IS_NOT_PY_CALLABLE 348 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 -#define _CHECK_METHOD_VERSION 351 -#define _CHECK_METHOD_VERSION_KW 352 -#define _CHECK_PEP_523 353 -#define _CHECK_PERIODIC 354 -#define _CHECK_PERIODIC_AT_END 355 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 356 -#define _CHECK_RECURSION_REMAINING 357 -#define _CHECK_STACK_SPACE 358 -#define _CHECK_STACK_SPACE_OPERAND 359 -#define _CHECK_VALIDITY 360 -#define _COLD_EXIT 361 -#define _COMPARE_OP 362 -#define _COMPARE_OP_FLOAT 363 -#define _COMPARE_OP_INT 364 -#define _COMPARE_OP_STR 365 -#define _CONTAINS_OP 366 -#define _CONTAINS_OP_DICT 367 -#define _CONTAINS_OP_SET 368 +#define _CHECK_FUNCTION_EXACT_ARGS 343 +#define _CHECK_FUNCTION_VERSION 344 +#define _CHECK_FUNCTION_VERSION_INLINE 345 +#define _CHECK_FUNCTION_VERSION_KW 346 +#define _CHECK_IS_NOT_PY_CALLABLE 347 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 348 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 349 +#define _CHECK_METHOD_VERSION 350 +#define _CHECK_METHOD_VERSION_KW 351 +#define _CHECK_PEP_523 352 +#define _CHECK_PERIODIC 353 +#define _CHECK_PERIODIC_AT_END 354 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 +#define _CHECK_RECURSION_REMAINING 356 +#define _CHECK_STACK_SPACE 357 +#define _CHECK_STACK_SPACE_OPERAND 358 +#define _CHECK_VALIDITY 359 +#define _COLD_EXIT 360 +#define _COMPARE_OP 361 +#define _COMPARE_OP_FLOAT 362 +#define _COMPARE_OP_INT 363 +#define _COMPARE_OP_STR 364 +#define _CONTAINS_OP 365 +#define _CONTAINS_OP_DICT 366 +#define _CONTAINS_OP_SET 367 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 369 -#define _COPY_1 370 -#define _COPY_2 371 -#define _COPY_3 372 +#define _COPY 368 +#define _COPY_1 369 +#define _COPY_2 370 +#define _COPY_3 371 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 373 +#define _CREATE_INIT_FRAME 372 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 374 +#define _DEOPT 373 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 375 -#define _DO_CALL_FUNCTION_EX 376 -#define _DO_CALL_KW 377 +#define _DO_CALL 374 +#define _DO_CALL_FUNCTION_EX 375 +#define _DO_CALL_KW 376 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 378 +#define _ERROR_POP_N 377 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 379 -#define _EXPAND_METHOD_KW 380 -#define _FATAL_ERROR 381 +#define _EXPAND_METHOD 378 +#define _EXPAND_METHOD_KW 379 +#define _FATAL_ERROR 380 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 382 -#define _FOR_ITER_GEN_FRAME 383 -#define _FOR_ITER_TIER_TWO 384 +#define _FOR_ITER 381 +#define _FOR_ITER_GEN_FRAME 382 +#define _FOR_ITER_TIER_TWO 383 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 385 -#define _GUARD_CALLABLE_ISINSTANCE 386 -#define _GUARD_CALLABLE_LEN 387 -#define _GUARD_CALLABLE_LIST_APPEND 388 -#define _GUARD_CALLABLE_STR_1 389 -#define _GUARD_CALLABLE_TUPLE_1 390 -#define _GUARD_CALLABLE_TYPE_1 391 -#define _GUARD_DORV_NO_DICT 392 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 393 -#define _GUARD_GLOBALS_VERSION 394 -#define _GUARD_IS_FALSE_POP 395 -#define _GUARD_IS_NONE_POP 396 -#define _GUARD_IS_NOT_NONE_POP 397 -#define _GUARD_IS_TRUE_POP 398 -#define _GUARD_KEYS_VERSION 399 -#define _GUARD_NOS_DICT 400 -#define _GUARD_NOS_FLOAT 401 -#define _GUARD_NOS_INT 402 -#define _GUARD_NOS_LIST 403 -#define _GUARD_NOS_NOT_NULL 404 -#define _GUARD_NOS_NULL 405 -#define _GUARD_NOS_OVERFLOWED 406 -#define _GUARD_NOS_TUPLE 407 -#define _GUARD_NOS_UNICODE 408 -#define _GUARD_NOT_EXHAUSTED_LIST 409 -#define _GUARD_NOT_EXHAUSTED_RANGE 410 -#define _GUARD_NOT_EXHAUSTED_TUPLE 411 -#define _GUARD_THIRD_NULL 412 -#define _GUARD_TOS_ANY_SET 413 -#define _GUARD_TOS_DICT 414 -#define _GUARD_TOS_FLOAT 415 -#define _GUARD_TOS_INT 416 -#define _GUARD_TOS_LIST 417 -#define _GUARD_TOS_OVERFLOWED 418 -#define _GUARD_TOS_SLICE 419 -#define _GUARD_TOS_TUPLE 420 -#define _GUARD_TOS_UNICODE 421 -#define _GUARD_TYPE_VERSION 422 -#define _GUARD_TYPE_VERSION_AND_LOCK 423 -#define _HANDLE_PENDING_AND_DEOPT 424 +#define _GUARD_BINARY_OP_EXTEND 384 +#define _GUARD_CALLABLE_ISINSTANCE 385 +#define _GUARD_CALLABLE_LEN 386 +#define _GUARD_CALLABLE_LIST_APPEND 387 +#define _GUARD_CALLABLE_STR_1 388 +#define _GUARD_CALLABLE_TUPLE_1 389 +#define _GUARD_CALLABLE_TYPE_1 390 +#define _GUARD_DORV_NO_DICT 391 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 392 +#define _GUARD_GLOBALS_VERSION 393 +#define _GUARD_IS_FALSE_POP 394 +#define _GUARD_IS_NONE_POP 395 +#define _GUARD_IS_NOT_NONE_POP 396 +#define _GUARD_IS_TRUE_POP 397 +#define _GUARD_KEYS_VERSION 398 +#define _GUARD_NOS_DICT 399 +#define _GUARD_NOS_FLOAT 400 +#define _GUARD_NOS_INT 401 +#define _GUARD_NOS_LIST 402 +#define _GUARD_NOS_NOT_NULL 403 +#define _GUARD_NOS_NULL 404 +#define _GUARD_NOS_OVERFLOWED 405 +#define _GUARD_NOS_TUPLE 406 +#define _GUARD_NOS_UNICODE 407 +#define _GUARD_NOT_EXHAUSTED_LIST 408 +#define _GUARD_NOT_EXHAUSTED_RANGE 409 +#define _GUARD_NOT_EXHAUSTED_TUPLE 410 +#define _GUARD_THIRD_NULL 411 +#define _GUARD_TOS_ANY_SET 412 +#define _GUARD_TOS_DICT 413 +#define _GUARD_TOS_FLOAT 414 +#define _GUARD_TOS_INT 415 +#define _GUARD_TOS_LIST 416 +#define _GUARD_TOS_OVERFLOWED 417 +#define _GUARD_TOS_SLICE 418 +#define _GUARD_TOS_TUPLE 419 +#define _GUARD_TOS_UNICODE 420 +#define _GUARD_TYPE_VERSION 421 +#define _GUARD_TYPE_VERSION_AND_LOCK 422 +#define _HANDLE_PENDING_AND_DEOPT 423 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 425 -#define _INIT_CALL_PY_EXACT_ARGS 426 -#define _INIT_CALL_PY_EXACT_ARGS_0 427 -#define _INIT_CALL_PY_EXACT_ARGS_1 428 -#define _INIT_CALL_PY_EXACT_ARGS_2 429 -#define _INIT_CALL_PY_EXACT_ARGS_3 430 -#define _INIT_CALL_PY_EXACT_ARGS_4 431 -#define _INSERT_NULL 432 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 424 +#define _INIT_CALL_PY_EXACT_ARGS 425 +#define _INIT_CALL_PY_EXACT_ARGS_0 426 +#define _INIT_CALL_PY_EXACT_ARGS_1 427 +#define _INIT_CALL_PY_EXACT_ARGS_2 428 +#define _INIT_CALL_PY_EXACT_ARGS_3 429 +#define _INIT_CALL_PY_EXACT_ARGS_4 430 +#define _INSERT_NULL 431 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -186,177 +185,177 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 433 +#define _IS_NONE 432 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 434 -#define _ITER_CHECK_RANGE 435 -#define _ITER_CHECK_TUPLE 436 -#define _ITER_JUMP_LIST 437 -#define _ITER_JUMP_RANGE 438 -#define _ITER_JUMP_TUPLE 439 -#define _ITER_NEXT_LIST 440 -#define _ITER_NEXT_LIST_TIER_TWO 441 -#define _ITER_NEXT_RANGE 442 -#define _ITER_NEXT_TUPLE 443 -#define _JUMP_TO_TOP 444 +#define _ITER_CHECK_LIST 433 +#define _ITER_CHECK_RANGE 434 +#define _ITER_CHECK_TUPLE 435 +#define _ITER_JUMP_LIST 436 +#define _ITER_JUMP_RANGE 437 +#define _ITER_JUMP_TUPLE 438 +#define _ITER_NEXT_LIST 439 +#define _ITER_NEXT_LIST_TIER_TWO 440 +#define _ITER_NEXT_RANGE 441 +#define _ITER_NEXT_TUPLE 442 +#define _JUMP_TO_TOP 443 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 445 -#define _LOAD_ATTR_CLASS 446 +#define _LOAD_ATTR 444 +#define _LOAD_ATTR_CLASS 445 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 447 -#define _LOAD_ATTR_METHOD_LAZY_DICT 448 -#define _LOAD_ATTR_METHOD_NO_DICT 449 -#define _LOAD_ATTR_METHOD_WITH_VALUES 450 -#define _LOAD_ATTR_MODULE 451 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 452 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 453 -#define _LOAD_ATTR_PROPERTY_FRAME 454 -#define _LOAD_ATTR_SLOT 455 -#define _LOAD_ATTR_WITH_HINT 456 +#define _LOAD_ATTR_INSTANCE_VALUE 446 +#define _LOAD_ATTR_METHOD_LAZY_DICT 447 +#define _LOAD_ATTR_METHOD_NO_DICT 448 +#define _LOAD_ATTR_METHOD_WITH_VALUES 449 +#define _LOAD_ATTR_MODULE 450 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 451 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 452 +#define _LOAD_ATTR_PROPERTY_FRAME 453 +#define _LOAD_ATTR_SLOT 454 +#define _LOAD_ATTR_WITH_HINT 455 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 457 +#define _LOAD_BYTECODE 456 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 458 -#define _LOAD_CONST_INLINE_BORROW 459 -#define _LOAD_CONST_UNDER_INLINE 460 -#define _LOAD_CONST_UNDER_INLINE_BORROW 461 +#define _LOAD_CONST_INLINE 457 +#define _LOAD_CONST_INLINE_BORROW 458 +#define _LOAD_CONST_UNDER_INLINE 459 +#define _LOAD_CONST_UNDER_INLINE_BORROW 460 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 462 -#define _LOAD_FAST_0 463 -#define _LOAD_FAST_1 464 -#define _LOAD_FAST_2 465 -#define _LOAD_FAST_3 466 -#define _LOAD_FAST_4 467 -#define _LOAD_FAST_5 468 -#define _LOAD_FAST_6 469 -#define _LOAD_FAST_7 470 +#define _LOAD_FAST 461 +#define _LOAD_FAST_0 462 +#define _LOAD_FAST_1 463 +#define _LOAD_FAST_2 464 +#define _LOAD_FAST_3 465 +#define _LOAD_FAST_4 466 +#define _LOAD_FAST_5 467 +#define _LOAD_FAST_6 468 +#define _LOAD_FAST_7 469 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 471 -#define _LOAD_FAST_BORROW_0 472 -#define _LOAD_FAST_BORROW_1 473 -#define _LOAD_FAST_BORROW_2 474 -#define _LOAD_FAST_BORROW_3 475 -#define _LOAD_FAST_BORROW_4 476 -#define _LOAD_FAST_BORROW_5 477 -#define _LOAD_FAST_BORROW_6 478 -#define _LOAD_FAST_BORROW_7 479 +#define _LOAD_FAST_BORROW 470 +#define _LOAD_FAST_BORROW_0 471 +#define _LOAD_FAST_BORROW_1 472 +#define _LOAD_FAST_BORROW_2 473 +#define _LOAD_FAST_BORROW_3 474 +#define _LOAD_FAST_BORROW_4 475 +#define _LOAD_FAST_BORROW_5 476 +#define _LOAD_FAST_BORROW_6 477 +#define _LOAD_FAST_BORROW_7 478 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 480 -#define _LOAD_GLOBAL_BUILTINS 481 -#define _LOAD_GLOBAL_MODULE 482 +#define _LOAD_GLOBAL 479 +#define _LOAD_GLOBAL_BUILTINS 480 +#define _LOAD_GLOBAL_MODULE 481 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 483 -#define _LOAD_SMALL_INT_0 484 -#define _LOAD_SMALL_INT_1 485 -#define _LOAD_SMALL_INT_2 486 -#define _LOAD_SMALL_INT_3 487 -#define _LOAD_SPECIAL 488 +#define _LOAD_SMALL_INT 482 +#define _LOAD_SMALL_INT_0 483 +#define _LOAD_SMALL_INT_1 484 +#define _LOAD_SMALL_INT_2 485 +#define _LOAD_SMALL_INT_3 486 +#define _LOAD_SPECIAL 487 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 489 +#define _MAKE_CALLARGS_A_TUPLE 488 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 490 +#define _MAKE_WARM 489 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 491 -#define _MAYBE_EXPAND_METHOD_KW 492 -#define _MONITOR_CALL 493 -#define _MONITOR_CALL_KW 494 -#define _MONITOR_JUMP_BACKWARD 495 -#define _MONITOR_RESUME 496 +#define _MAYBE_EXPAND_METHOD 490 +#define _MAYBE_EXPAND_METHOD_KW 491 +#define _MONITOR_CALL 492 +#define _MONITOR_CALL_KW 493 +#define _MONITOR_JUMP_BACKWARD 494 +#define _MONITOR_RESUME 495 #define _NOP NOP -#define _POP_CALL 497 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 498 -#define _POP_CALL_ONE 499 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 500 -#define _POP_CALL_TWO 501 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 502 +#define _POP_CALL 496 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 497 +#define _POP_CALL_ONE 498 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 499 +#define _POP_CALL_TWO 500 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 501 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 503 -#define _POP_JUMP_IF_TRUE 504 +#define _POP_JUMP_IF_FALSE 502 +#define _POP_JUMP_IF_TRUE 503 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 505 -#define _POP_TOP_INT 506 -#define _POP_TOP_LOAD_CONST_INLINE 507 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 508 -#define _POP_TOP_NOP 509 -#define _POP_TOP_UNICODE 510 -#define _POP_TWO 511 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 512 +#define _POP_TOP_FLOAT 504 +#define _POP_TOP_INT 505 +#define _POP_TOP_LOAD_CONST_INLINE 506 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 507 +#define _POP_TOP_NOP 508 +#define _POP_TOP_UNICODE 509 +#define _POP_TWO 510 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 511 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 513 +#define _PUSH_FRAME 512 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 514 -#define _PY_FRAME_GENERAL 515 -#define _PY_FRAME_KW 516 -#define _QUICKEN_RESUME 517 -#define _REPLACE_WITH_TRUE 518 +#define _PUSH_NULL_CONDITIONAL 513 +#define _PY_FRAME_GENERAL 514 +#define _PY_FRAME_KW 515 +#define _QUICKEN_RESUME 516 +#define _REPLACE_WITH_TRUE 517 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 519 -#define _SEND 520 -#define _SEND_GEN_FRAME 521 +#define _SAVE_RETURN_OFFSET 518 +#define _SEND 519 +#define _SEND_GEN_FRAME 520 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 522 -#define _STORE_ATTR 523 -#define _STORE_ATTR_INSTANCE_VALUE 524 -#define _STORE_ATTR_SLOT 525 -#define _STORE_ATTR_WITH_HINT 526 +#define _START_EXECUTOR 521 +#define _STORE_ATTR 522 +#define _STORE_ATTR_INSTANCE_VALUE 523 +#define _STORE_ATTR_SLOT 524 +#define _STORE_ATTR_WITH_HINT 525 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 527 -#define _STORE_FAST_0 528 -#define _STORE_FAST_1 529 -#define _STORE_FAST_2 530 -#define _STORE_FAST_3 531 -#define _STORE_FAST_4 532 -#define _STORE_FAST_5 533 -#define _STORE_FAST_6 534 -#define _STORE_FAST_7 535 +#define _STORE_FAST 526 +#define _STORE_FAST_0 527 +#define _STORE_FAST_1 528 +#define _STORE_FAST_2 529 +#define _STORE_FAST_3 530 +#define _STORE_FAST_4 531 +#define _STORE_FAST_5 532 +#define _STORE_FAST_6 533 +#define _STORE_FAST_7 534 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 536 -#define _STORE_SUBSCR 537 -#define _STORE_SUBSCR_DICT 538 -#define _STORE_SUBSCR_LIST_INT 539 -#define _SWAP 540 -#define _SWAP_2 541 -#define _SWAP_3 542 -#define _TIER2_RESUME_CHECK 543 -#define _TO_BOOL 544 +#define _STORE_SLICE 535 +#define _STORE_SUBSCR 536 +#define _STORE_SUBSCR_DICT 537 +#define _STORE_SUBSCR_LIST_INT 538 +#define _SWAP 539 +#define _SWAP_2 540 +#define _SWAP_3 541 +#define _TIER2_RESUME_CHECK 542 +#define _TO_BOOL 543 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 545 +#define _TO_BOOL_LIST 544 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 546 +#define _TO_BOOL_STR 545 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 547 -#define _UNPACK_SEQUENCE_LIST 548 -#define _UNPACK_SEQUENCE_TUPLE 549 -#define _UNPACK_SEQUENCE_TWO_TUPLE 550 +#define _UNPACK_SEQUENCE 546 +#define _UNPACK_SEQUENCE_LIST 547 +#define _UNPACK_SEQUENCE_TUPLE 548 +#define _UNPACK_SEQUENCE_TWO_TUPLE 549 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 550 +#define MAX_UOP_ID 549 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index bf361233560c55..1248771996943b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -329,7 +329,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_LOAD_CONST_UNDER_INLINE] = 0, [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, - [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, @@ -404,7 +403,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", - [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", @@ -1291,8 +1289,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_CONST_UNDER_INLINE_BORROW: return 1; - case _CHECK_FUNCTION: - return 0; case _START_EXECUTOR: return 0; case _MAKE_WARM: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index ffd65dbb1464f8..fd30889655bf19 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -5,6 +5,7 @@ import unittest import gc import os +import types import _opcode @@ -16,6 +17,8 @@ from _testinternalcapi import TIER2_THRESHOLD +#For test of issue 136154 +GLOBAL_136154 = 42 @contextlib.contextmanager def clear_executors(func): @@ -2501,6 +2504,30 @@ def testfunc(n): # For now... until we constant propagate it away. self.assertIn("_BINARY_OP", uops) + def test_jitted_code_sees_changed_globals(self): + "Issue 136154: Check that jitted code spots the change in the globals" + + def make_f(): + def f(): + return GLOBAL_136154 + return f + + make_f_with_bad_globals = types.FunctionType(make_f.__code__, {}) + + def jitted(funcs): + for func in funcs: + func() + + # Make a "good" f: + f = make_f() + # Compile jitted for the "good" f: + jitted([f] * TIER2_THRESHOLD) + # This "bad" f has different globals, but the *same* code/function versions: + f_with_bad_globals = make_f_with_bad_globals() + # A "good" f to enter the JIT code, and a "bad" f to trigger the bug: + with self.assertRaises(NameError): + jitted([f, f_with_bad_globals]) + def test_reference_tracking_across_call_doesnt_crash(self): def f1(): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-03-10-16-09.gh-issue-138378.r6BQxV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-03-10-16-09.gh-issue-138378.r6BQxV.rst new file mode 100644 index 00000000000000..0584177b5e87aa --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-03-10-16-09.gh-issue-138378.r6BQxV.rst @@ -0,0 +1,2 @@ +Move the globals-to-const JIT optimizer pass into to the main JIT optimizer +pass diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6c3609d293890f..74025ffe9cedc9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5345,12 +5345,6 @@ dummy_func( value = PyStackRef_FromPyObjectBorrow(ptr); } - tier2 op(_CHECK_FUNCTION, (func_version/2 -- )) { - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - DEOPT_IF(func->func_version != func_version); - } - tier2 op(_START_EXECUTOR, (executor/4 --)) { #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1309c1317a6615..0e4d86463761a0 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7386,17 +7386,6 @@ break; } - case _CHECK_FUNCTION: { - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - if (func->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - break; - } - case _START_EXECUTOR: { PyObject *executor = (PyObject *)CURRENT_OPERAND0(); #ifndef _Py_JIT diff --git a/Python/optimizer.c b/Python/optimizer.c index bae5cfa50ead58..328857b6131303 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -887,7 +887,7 @@ translate_bytecode_to_trace( _Py_BloomFilter_Add(dependencies, new_code); /* Set the operand to the callee's function or code object, * to assist optimization passes. - * We prefer setting it to the function (for remove_globals()) + * We prefer setting it to the function * but if that's not available but the code is available, * use the code, setting the low bit so the optimizer knows. */ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index fd395d3c6c254f..a84d5806db4ac1 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -128,184 +128,16 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop) return res; } -static int -incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) +static bool +incorrect_keys(PyObject *obj, uint32_t version) { if (!PyDict_CheckExact(obj)) { - return 1; + return true; } PyDictObject *dict = (PyDictObject *)obj; - if (dict->ma_keys->dk_version != inst->operand0) { - return 1; - } - return 0; + return dict->ma_keys->dk_version != version; } -/* Returns 1 if successfully optimized - * 0 if the trace is not suitable for optimization (yet) - * -1 if there was an error. */ -static int -remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, - int buffer_size, _PyBloomFilter *dependencies) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *builtins = frame->f_builtins; - if (builtins != interp->builtins) { - OPT_STAT_INC(remove_globals_builtins_changed); - return 1; - } - PyObject *globals = frame->f_globals; - PyFunctionObject *function = _PyFrame_GetFunction(frame); - assert(PyFunction_Check(function)); - assert(function->func_builtins == builtins); - assert(function->func_globals == globals); - uint32_t function_version = _PyFunction_GetVersionForCurrentState(function); - /* In order to treat globals as constants, we need to - * know that the globals dict is the one we expected, and - * that it hasn't changed - * In order to treat builtins as constants, we need to - * know that the builtins dict is the one we expected, and - * that it hasn't changed and that the global dictionary's - * keys have not changed */ - - /* These values represent stacks of booleans (one bool per bit). - * Pushing a frame shifts left, popping a frame shifts right. */ - uint32_t function_checked = 0; - uint32_t builtins_watched = 0; - uint32_t globals_watched = 0; - uint32_t prechecked_function_version = 0; - if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { - interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; - } - if (interp->type_watchers[TYPE_WATCHER_ID] == NULL) { - interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; - } - for (int pc = 0; pc < buffer_size; pc++) { - _PyUOpInstruction *inst = &buffer[pc]; - int opcode = inst->opcode; - switch(opcode) { - case _GUARD_GLOBALS_VERSION: - if (incorrect_keys(inst, globals)) { - OPT_STAT_INC(remove_globals_incorrect_keys); - return 0; - } - if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { - continue; - } - if ((globals_watched & 1) == 0) { - PyDict_Watch(GLOBALS_WATCHER_ID, globals); - _Py_BloomFilter_Add(dependencies, globals); - globals_watched |= 1; - } - if (function_checked & 1) { - buffer[pc].opcode = NOP; - } - else { - buffer[pc].opcode = _CHECK_FUNCTION; - buffer[pc].operand0 = function_version; - function_checked |= 1; - } - break; - case _LOAD_GLOBAL_BUILTINS: - if (incorrect_keys(inst, builtins)) { - OPT_STAT_INC(remove_globals_incorrect_keys); - return 0; - } - if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { - continue; - } - if ((builtins_watched & 1) == 0) { - PyDict_Watch(BUILTINS_WATCHER_ID, builtins); - builtins_watched |= 1; - } - if (function_checked & globals_watched & 1) { - convert_global_to_const(inst, builtins, false); - } - break; - case _LOAD_GLOBAL_MODULE: - if (incorrect_keys(inst, globals)) { - OPT_STAT_INC(remove_globals_incorrect_keys); - return 0; - } - if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { - continue; - } - if ((globals_watched & 1) == 0) { - PyDict_Watch(GLOBALS_WATCHER_ID, globals); - _Py_BloomFilter_Add(dependencies, globals); - globals_watched |= 1; - } - if ((function_checked & 1) == 0 && buffer[pc-1].opcode == _NOP) { - buffer[pc-1].opcode = _CHECK_FUNCTION; - buffer[pc-1].operand0 = function_version; - function_checked |= 1; - } - if (function_checked & 1) { - convert_global_to_const(inst, globals, false); - } - break; - case _PUSH_FRAME: - { - builtins_watched <<= 1; - globals_watched <<= 1; - function_checked <<= 1; - uint64_t operand = buffer[pc].operand0; - if (operand == 0 || (operand & 1)) { - // It's either a code object or NULL, so bail - return 1; - } - PyFunctionObject *func = (PyFunctionObject *)operand; - if (func == NULL) { - return 1; - } - assert(PyFunction_Check(func)); - function_version = func->func_version; - if (prechecked_function_version == function_version) { - function_checked |= 1; - } - prechecked_function_version = 0; - globals = func->func_globals; - builtins = func->func_builtins; - if (builtins != interp->builtins) { - OPT_STAT_INC(remove_globals_builtins_changed); - return 1; - } - break; - } - case _RETURN_VALUE: - { - builtins_watched >>= 1; - globals_watched >>= 1; - function_checked >>= 1; - uint64_t operand = buffer[pc].operand0; - if (operand == 0 || (operand & 1)) { - // It's either a code object or NULL, so bail - return 1; - } - PyFunctionObject *func = (PyFunctionObject *)operand; - if (func == NULL) { - return 1; - } - assert(PyFunction_Check(func)); - function_version = func->func_version; - globals = func->func_globals; - builtins = func->func_builtins; - break; - } - case _CHECK_FUNCTION_EXACT_ARGS: - prechecked_function_version = (uint32_t)buffer[pc].operand0; - break; - default: - if (is_terminator(inst)) { - return 1; - } - break; - } - } - return 0; -} - - #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define STACK_SIZE() ((int)(ctx->frame->stack_len)) @@ -317,9 +149,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define GETLOCAL(idx) ((ctx->frame->locals[idx])) #define REPLACE_OP(INST, OP, ARG, OPERAND) \ - INST->opcode = OP; \ - INST->oparg = ARG; \ - INST->operand0 = OPERAND; + (INST)->opcode = OP; \ + (INST)->oparg = ARG; \ + (INST)->operand0 = OPERAND; /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null @@ -407,30 +239,6 @@ lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, return sym_new_not_null(ctx); } -/* _PUSH_FRAME/_RETURN_VALUE's operand can be 0, a PyFunctionObject *, or a - * PyCodeObject *. Retrieve the code object if possible. - */ -static PyCodeObject * -get_code(_PyUOpInstruction *op) -{ - assert(op->opcode == _PUSH_FRAME || op->opcode == _RETURN_VALUE || op->opcode == _RETURN_GENERATOR); - PyCodeObject *co = NULL; - uint64_t operand = op->operand0; - if (operand == 0) { - return NULL; - } - if (operand & 1) { - co = (PyCodeObject *)(operand & ~1); - } - else { - PyFunctionObject *func = (PyFunctionObject *)operand; - assert(PyFunction_Check(func)); - co = (PyCodeObject *)func->func_code; - } - assert(PyCode_Check(co)); - return co; -} - static PyCodeObject * get_code_with_logging(_PyUOpInstruction *op) { @@ -455,6 +263,19 @@ get_code_with_logging(_PyUOpInstruction *op) return co; } +static +PyCodeObject * +get_current_code_object(JitOptContext *ctx) +{ + return (PyCodeObject *)ctx->frame->func->func_code; +} + +static PyObject * +get_co_name(JitOptContext *ctx, int index) +{ + return PyTuple_GET_ITEM(get_current_code_object(ctx)->co_names, index); +} + // TODO (gh-134584) generate most of this table automatically const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, @@ -465,7 +286,7 @@ const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { /* 1 for success, 0 for not ready, cannot error at the moment. */ static int optimize_uops( - PyCodeObject *co, + PyFunctionObject *func, _PyUOpInstruction *trace, int trace_len, int curr_stacklen, @@ -481,11 +302,19 @@ optimize_uops( _PyUOpInstruction *first_valid_check_stack = NULL; _PyUOpInstruction *corresponding_check_stack = NULL; + // Make sure that watchers are set up + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { + interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + } + _Py_uop_abstractcontext_init(ctx); - _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, (PyCodeObject *)func->func_code, curr_stacklen, NULL, 0); if (frame == NULL) { return 0; } + frame->func = func; ctx->curr_frame_depth++; ctx->frame = frame; @@ -693,13 +522,8 @@ _Py_uop_analyze_and_optimize( { OPT_STAT_INC(optimizer_attempts); - int err = remove_globals(frame, buffer, length, dependencies); - if (err <= 0) { - return err; - } - length = optimize_uops( - _PyFrame_GetCode(frame), buffer, + _PyFrame_GetFunction(frame), buffer, length, curr_stacklen, dependencies); if (length <= 0) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index eccbddf0546ab3..49689a8fe11ce8 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -461,6 +461,7 @@ dummy_func(void) { } op(_LOAD_CONST, (-- value)) { + PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); @@ -597,7 +598,7 @@ dummy_func(void) { op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { (void)descr; PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); @@ -606,7 +607,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); @@ -615,7 +616,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); @@ -624,7 +625,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); @@ -634,7 +635,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); @@ -644,7 +645,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); @@ -702,15 +703,13 @@ dummy_func(void) { op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { int argcount = oparg; - PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; } - assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { @@ -733,9 +732,8 @@ dummy_func(void) { } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { - PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; @@ -766,6 +764,7 @@ dummy_func(void) { JitOptRef temp = PyJitRef_StripReferenceInfo(retval); DEAD(retval); SAVE_STACK(); + PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -778,17 +777,13 @@ dummy_func(void) { assert(framesize <= curr_space); curr_space -= framesize; - co = get_code(this_instr); - if (co == NULL) { - // might be impossible, but bailing is still safe - ctx->done = true; - } RELOAD_STACK(); res = temp; } op(_RETURN_GENERATOR, ( -- res)) { SYNC_SP(); + PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -801,12 +796,6 @@ dummy_func(void) { assert(framesize > 0); assert(framesize <= curr_space); curr_space -= framesize; - - co = get_code(this_instr); - if (co == NULL) { - // might be impossible, but bailing is still safe - ctx->done = true; - } } op(_YIELD_VALUE, (unused -- value)) { @@ -854,13 +843,16 @@ dummy_func(void) { ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; - co = get_code(this_instr); - if (co == NULL) { - // should be about to _EXIT_TRACE anyway + uint64_t operand = this_instr->operand0; + if (operand == 0 || (operand & 1)) { + // It's either a code object or NULL ctx->done = true; break; } - + PyFunctionObject *func = (PyFunctionObject *)operand; + PyCodeObject *co = (PyCodeObject *)func->func_code; + assert(PyFunction_Check(func)); + ctx->frame->func = func; /* Stack space handling */ int framesize = co->co_framesize; assert(framesize > 0); @@ -1241,6 +1233,96 @@ dummy_func(void) { } } + op(_GUARD_GLOBALS_VERSION, (version/1 --)) { + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + /* Do nothing */ + } + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; + } + if (ctx->frame->globals_checked_version == version) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + } + } + ctx->frame->globals_checked_version = version; + } + + op(_LOAD_GLOBAL_BUILTINS, (version/1, index/1 -- res)) { + (void)version; + (void)index; + PyObject *cnst = NULL; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *builtins = interp->builtins; + if (incorrect_keys(builtins, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + /* Do nothing */ + } + else { + if (!ctx->builtins_watched) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + ctx->builtins_watched = true; + } + if (ctx->frame->globals_checked_version != 0 && ctx->frame->globals_watched) { + cnst = convert_global_to_const(this_instr, builtins, false); + } + } + if (cnst == NULL) { + res = sym_new_not_null(ctx); + } + else { + res = sym_new_const(ctx, cnst); + } + } + + op(_LOAD_GLOBAL_MODULE, (version/1, unused/1, index/1 -- res)) { + (void)index; + PyObject *cnst = NULL; + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + /* Do nothing */ + } + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; + } + if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { + REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + ctx->frame->globals_checked_version = version; + } + if (ctx->frame->globals_checked_version == version) { + cnst = convert_global_to_const(this_instr, globals, false); + } + } + } + if (cnst == NULL) { + res = sym_new_not_null(ctx); + } + else { + res = sym_new_const(ctx, cnst); + } + } + + // END BYTECODES // } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 8617355e25f418..05fbbc31123254 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -70,6 +70,7 @@ case _LOAD_CONST: { JitOptRef value; + PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); @@ -1063,6 +1064,7 @@ JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -1072,10 +1074,6 @@ assert(framesize > 0); assert(framesize <= curr_space); curr_space -= framesize; - co = get_code(this_instr); - if (co == NULL) { - ctx->done = true; - } res = temp; stack_pointer[0] = res; stack_pointer += 1; @@ -1291,12 +1289,65 @@ } case _GUARD_GLOBALS_VERSION: { + uint16_t version = (uint16_t)this_instr->operand0; + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + } + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; + } + if (ctx->frame->globals_checked_version == version) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + } + } + ctx->frame->globals_checked_version = version; break; } case _LOAD_GLOBAL_MODULE: { JitOptRef res; - res = sym_new_not_null(ctx); + uint16_t version = (uint16_t)this_instr->operand0; + uint16_t index = (uint16_t)this_instr->operand0; + (void)index; + PyObject *cnst = NULL; + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + } + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; + } + if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { + REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + ctx->frame->globals_checked_version = version; + } + if (ctx->frame->globals_checked_version == version) { + cnst = convert_global_to_const(this_instr, globals, false); + } + } + } + if (cnst == NULL) { + res = sym_new_not_null(ctx); + } + else { + res = sym_new_const(ctx, cnst); + } stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1305,7 +1356,34 @@ case _LOAD_GLOBAL_BUILTINS: { JitOptRef res; - res = sym_new_not_null(ctx); + uint16_t version = (uint16_t)this_instr->operand0; + uint16_t index = (uint16_t)this_instr->operand0; + (void)version; + (void)index; + PyObject *cnst = NULL; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *builtins = interp->builtins; + if (incorrect_keys(builtins, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + } + else { + if (!ctx->builtins_watched) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + ctx->builtins_watched = true; + } + if (ctx->frame->globals_checked_version != 0 && ctx->frame->globals_watched) { + cnst = convert_global_to_const(this_instr, builtins, false); + } + } + if (cnst == NULL) { + res = sym_new_not_null(ctx); + } + else { + res = sym_new_const(ctx, cnst); + } stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1599,7 +1677,7 @@ PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); @@ -2133,7 +2211,7 @@ PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); @@ -2153,7 +2231,7 @@ PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); @@ -2172,7 +2250,7 @@ PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); @@ -2187,7 +2265,7 @@ PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); @@ -2207,7 +2285,7 @@ PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; PyTypeObject *type = sym_get_type(owner); - PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); + PyObject *name = get_co_name(ctx, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); @@ -2240,9 +2318,8 @@ case _PY_FRAME_GENERAL: { JitOptRef new_frame; - PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; @@ -2366,9 +2443,8 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; int argcount = oparg; - PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; @@ -2399,11 +2475,15 @@ ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; - co = get_code(this_instr); - if (co == NULL) { + uint64_t operand = this_instr->operand0; + if (operand == 0 || (operand & 1)) { ctx->done = true; break; } + PyFunctionObject *func = (PyFunctionObject *)operand; + PyCodeObject *co = (PyCodeObject *)func->func_code; + assert(PyFunction_Check(func)); + ctx->frame->func = func; int framesize = co->co_framesize; assert(framesize > 0); curr_space += framesize; @@ -2795,6 +2875,7 @@ case _RETURN_GENERATOR: { JitOptRef res; + PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -2808,10 +2889,6 @@ stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - co = get_code(this_instr); - if (co == NULL) { - ctx->done = true; - } break; } @@ -3180,10 +3257,6 @@ break; } - case _CHECK_FUNCTION: { - break; - } - case _START_EXECUTOR: { break; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index c0a876ddc2c716..0e6884b99232e9 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -827,6 +827,9 @@ _Py_uop_frame_new( frame->locals = ctx->n_consumed; frame->stack = frame->locals + co->co_nlocalsplus; frame->stack_pointer = frame->stack + curr_stackentries; + frame->globals_checked_version = 0; + frame->globals_watched = false; + frame->func = NULL; ctx->n_consumed = ctx->n_consumed + (co->co_nlocalsplus + co->co_stacksize); if (ctx->n_consumed >= ctx->limit) { ctx->done = true; @@ -895,6 +898,7 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) ctx->done = false; ctx->out_of_space = false; ctx->contradiction = false; + ctx->builtins_watched = false; } int