Allow pending call procesing to be deferred

Author: ncoghlan

Include/opcode.h

@@ -128,6 +128,7 @@ extern "C" {
 #define BUILD_TUPLE_UNPACK_WITH_CALL 158
 #define LOAD_METHOD             160
 #define CALL_METHOD             161
+#define DEFER_PENDING_UNTIL     162
 
 /* EXCEPT_HANDLER is a special, implicit block type which is created when
    entering an except handler. It is not an opcode but we define it here

Lib/importlib/_bootstrap_external.py

@@ -241,6 +241,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.6b2  3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257)
 #     Python 3.6rc1 3379 (more thorough __class__ validation #23722)
 #     Python 3.7a0  3390 (add LOAD_METHOD and CALL_METHOD opcodes)
+#                   3391 (add DEFER_PENDING_UNTIL opcode)
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
@@ -249,7 +250,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3390).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3391).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/opcode.py

@@ -215,4 +215,6 @@ def jabs_op(name, op):
 name_op('LOAD_METHOD', 160)
 def_op('CALL_METHOD', 161)
 
+jrel_op('DEFER_PENDING_UNTIL', 162) # When to resume pending call processing
+
 del def_op, name_op, jrel_op, jabs_op

Lib/test/test_with_signal_safety.py

@@ -23,7 +23,8 @@ def inject_exception(frame, event, arg):
             return
         frame.f_trace_opcodes = True
         if frame.f_lasti >= target_instruction:
-            raise InjectedException(f"Failing after {target_instruction}")
+            if frame.f_lasti > frame.f_pendingi:
+                raise InjectedException(f"Failing after {frame.f_lasti}")
         return inject_exception
     sys.settrace(inject_exception)
 
@@ -72,13 +73,9 @@ def traced_function():
             return
         target_instruction = -1
         num_instructions = len(traced_function.__code__.co_code) - 2
-        import dis
-        dis.dis(traced_function)
         while target_instruction < num_instructions:
             target_instruction += 1
             raise_after_instruction(traced_function, target_instruction)
-            if verbose:
-                print(f"Raising exception after {target_instruction}")
             try:
                 traced_function()
             except InjectedException:
@@ -107,8 +104,6 @@ async def traced_coroutine():
         while target_instruction < num_instructions:
             target_instruction += 1
             raise_after_instruction(traced_coroutine, target_instruction)
-            if verbose:
-                print(f"Raising exception after {target_instruction}")
             try:
                 loop.run_until_complete(traced_coroutine())
             except InjectedException:

Python/ceval.c

@@ -586,6 +586,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
     int instr_ub = -1, instr_lb = 0, instr_prev = -1;
 
     const _Py_CODEUNIT *first_instr;
+    const _Py_CODEUNIT *handle_pending_after;
     PyObject *names;
     PyObject *consts;
 
@@ -716,6 +717,8 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
     } while (0)
 #define JUMPTO(x)       (next_instr = first_instr + (x) / sizeof(_Py_CODEUNIT))
 #define JUMPBY(x)       (next_instr += (x) / sizeof(_Py_CODEUNIT))
+#define DEFERUNTIL(x)   (handle_pending_after = next_instr + (x) / sizeof(_Py_CODEUNIT))
+#define DEFER_OFFSET()  (sizeof(_Py_CODEUNIT) * (int)(handle_pending_after - first_instr))
 
 /* OpCode prediction macros
     Some opcodes tend to come in pairs thus making it possible to
@@ -894,6 +897,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
     assert(PyBytes_GET_SIZE(co->co_code) % sizeof(_Py_CODEUNIT) == 0);
     assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(co->co_code), sizeof(_Py_CODEUNIT)));
     first_instr = (_Py_CODEUNIT *) PyBytes_AS_STRING(co->co_code);
+    handle_pending_after = first_instr;
     /*
        f->f_lasti refers to the index of the last instruction,
        unless it's -1 in which case next_instr should be first_instr.
@@ -977,9 +981,12 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
                 */
                 goto fast_next_opcode;
             }
-            if (_Py_atomic_load_relaxed(&pendingcalls_to_do)) {
+            if ((next_instr > handle_pending_after) &&
+                    _Py_atomic_load_relaxed(&pendingcalls_to_do)) {
                 if (Py_MakePendingCalls() < 0)
                     goto error;
+                /* Allow for subsequent jumps backwards in the bytecode */
+                handle_pending_after = first_instr;
             }
             if (_Py_atomic_load_relaxed(&gil_drop_request)) {
                 /* Give another thread a chance */
@@ -3129,6 +3136,12 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
+        TARGET(DEFER_PENDING_UNTIL) {
+            DEFERUNTIL(oparg);
+            PREDICT(POP_BLOCK);
+            DISPATCH();
+        }
+
         TARGET(LOAD_METHOD) {
             /* Designed to work in tamdem with CALL_METHOD. */
             PyObject *name = GETITEM(names, oparg);

Python/compile.c

@@ -943,6 +943,8 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg)
             return 0;
         case YIELD_FROM:
             return -1;
+        case DEFER_PENDING_UNTIL:
+            return 0;
         case POP_BLOCK:
             return 0;
         case POP_EXCEPT:
@@ -4194,14 +4196,15 @@ expr_constant(struct compiler *c, expr_ty e)
 static int
 compiler_async_with(struct compiler *c, stmt_ty s, int pos)
 {
-    basicblock *block, *finally;
+    basicblock *block, *finally, *finally_await;
     withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
 
     assert(s->kind == AsyncWith_kind);
 
     block = compiler_new_block(c);
     finally = compiler_new_block(c);
-    if (!block || !finally)
+    finally_await = compiler_new_block(c);
+    if (!block || !finally || !finally_await)
         return 0;
 
     /* Evaluate EXPR */
@@ -4210,8 +4213,8 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
     ADDOP(c, BEFORE_ASYNC_WITH);
     ADDOP(c, GET_AWAITABLE);
     ADDOP_O(c, LOAD_CONST, Py_None, consts);
+    ADDOP_JREL(c, DEFER_PENDING_UNTIL, block);
     ADDOP(c, YIELD_FROM);
-
     ADDOP_JREL(c, SETUP_ASYNC_WITH, finally);
 
     /* SETUP_ASYNC_WITH pushes a finally block. */
@@ -4236,6 +4239,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
             return 0;
 
     /* End of try block; start the finally block */
+    ADDOP_JREL(c, DEFER_PENDING_UNTIL, finally_await);
     ADDOP(c, POP_BLOCK);
     compiler_pop_fblock(c, FINALLY_TRY, block);
 
@@ -4249,8 +4253,10 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
        opcode. */
     ADDOP(c, WITH_CLEANUP_START);
 
+    ADDOP_JREL(c, DEFER_PENDING_UNTIL, finally_await);
     ADDOP(c, GET_AWAITABLE);
     ADDOP_O(c, LOAD_CONST, Py_None, consts);
+    compiler_use_next_block(c, finally_await);
     ADDOP(c, YIELD_FROM);
 
     ADDOP(c, WITH_CLEANUP_FINISH);
@@ -4324,6 +4330,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos)
             return 0;
 
     /* End of try block; start the finally block */
+    ADDOP_JREL(c, DEFER_PENDING_UNTIL, finally);
     ADDOP(c, POP_BLOCK);
     compiler_pop_fblock(c, FINALLY_TRY, block);