gh-130704: Strength reduce LOAD_FAST{_LOAD_FAST}

Doc/library/dis.rst

@@ -76,7 +76,7 @@ the following command can be used to display the disassembly of
      2           RESUME                   0
    <BLANKLINE>
      3           LOAD_GLOBAL              1 (len + NULL)
-                 LOAD_FAST                0 (alist)
+                 LOAD_FAST_BORROW         0 (alist)
                  CALL                     1
                  RETURN_VALUE
 
@@ -215,7 +215,7 @@ Example:
     ...
     RESUME
     LOAD_GLOBAL
-    LOAD_FAST
+    LOAD_FAST_BORROW
     CALL
     RETURN_VALUE
 
@@ -1402,13 +1402,29 @@ iterations of the loop.
       This opcode is now only used in situations where the local variable is
       guaranteed to be initialized. It cannot raise :exc:`UnboundLocalError`.
 
+.. opcode:: LOAD_FAST_BORROW (var_num)
+
+   Pushes a borrowed reference to the local ``co_varnames[var_num]`` onto the stack
+   in free-threaded builds. In default builds this is identical to ``LOAD_FAST``.
+
+   .. versionadded:: 3.14
+
 .. opcode:: LOAD_FAST_LOAD_FAST (var_nums)
 
    Pushes references to ``co_varnames[var_nums >> 4]`` and
    ``co_varnames[var_nums & 15]`` onto the stack.
 
    .. versionadded:: 3.13
 
+
+.. opcode:: LOAD_FAST_BORROW_LOAD_FAST_BORROW (var_nums)
+
+   Pushes borrowed references to ``co_varnames[var_nums >> 4]`` and
+   ``co_varnames[var_nums & 15]`` onto the stack in free-threaded builds. This is
+   identical to ``LOAD_FAST_LOAD_FAST`` in default builds.
+
+   .. versionadded:: 3.14
+
 .. opcode:: LOAD_FAST_CHECK (var_num)
 
    Pushes a reference to the local ``co_varnames[var_num]`` onto the stack,
@@ -2023,4 +2039,3 @@ instructions:
 
    .. deprecated:: 3.13
       All jumps are now relative. This list is empty.
-

Include/internal/pycore_flowgraph.h

@@ -41,6 +41,9 @@ PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg(
         PyObject *consts,
         int nlocals);
 
+// Export for '_testinternalcapi' shared extension
+PyAPI_FUNC(PyObject*) _PyCompile_OptimizeLoadFast(PyObject *instructions);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_frame.h

@@ -31,6 +31,12 @@ struct _frame {
        PyEval_GetLocals requires a borrowed reference so the actual reference
        is stored here */
     PyObject *f_locals_cache;
+    /* A list containing strong references to fast locals that were overwritten
+     * via f_locals. Borrowed references to these locals may exist in frames
+     * closer to the top of the stack. The references in this list act as
+     * "support" for the borrowed references, ensuring that they remain valid.
+     */
+    PyObject *f_overwritten_fast_locals;
     /* The frame data, if this frame object owns the frame */
     PyObject *_f_frame_data[1];
 };
@@ -146,16 +152,29 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code)
     return code->co_framesize - FRAME_SPECIALS_SIZE;
 }
 
-static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest)
+static inline void
+_PyFrame_CopyToHeap(_PyInterpreterFrame *src, _PyInterpreterFrame *dest)
 {
     *dest = *src;
     assert(src->stackpointer != NULL);
     int stacktop = (int)(src->stackpointer - src->localsplus);
     assert(stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus);
     dest->stackpointer = dest->localsplus + stacktop;
-    for (int i = 1; i < stacktop; i++) {
-        dest->localsplus[i] = src->localsplus[i];
+    // The destination frame may outlive any references that were providing
+    // "support" for borrowed references in the source frame. Convert any
+    // borrowed references that were copied into dest into strong references.
+    for (int i = 0; i < stacktop; i++) {
+        dest->localsplus[i] =
+            _PyStackRef_NewIfBorrowedOrSteal(src->localsplus[i]);
     }
+    dest->f_executable = _PyStackRef_NewIfBorrowedOrSteal(dest->f_executable);
+    dest->f_funcobj = _PyStackRef_NewIfBorrowedOrSteal(dest->f_funcobj);
+}
+
+static inline void
+_PyFrame_CopyToNewGen(_PyInterpreterFrame *src, _PyInterpreterFrame *dest)
+{
+    _PyFrame_CopyToHeap(src, dest);
     // Don't leave a dangling pointer to the old frame when creating generators
     // and coroutines:
     dest->previous = NULL;

Include/internal/pycore_magic_number.h

@@ -282,7 +282,7 @@ PC/launcher.c must also be updated.
 
 */
 
-#define PYC_MAGIC_NUMBER 3617
+#define PYC_MAGIC_NUMBER 3618
 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
    (little-endian) and then appending b'\r\n'. */
 #define PYC_MAGIC_NUMBER_TOKEN \

Include/internal/pycore_stackref.h

@@ -101,6 +101,12 @@ PyStackRef_IsNone(_PyStackRef ref)
     return _Py_stackref_get_object(ref) == Py_None;
 }
 
+static inline bool
+PyStackRef_IsBorrowed(_PyStackRef ref)
+{
+    return false;
+}
+
 static inline PyObject *
 _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)
 {
@@ -146,6 +152,12 @@ PyStackRef_CLOSE(_PyStackRef ref)
     Py_DECREF(obj);
 }
 
+static inline _PyStackRef
+_PyStackRef_NewIfBorrowedOrSteal(_PyStackRef ref)
+{
+    return ref;
+}
+
 static inline _PyStackRef
 _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
 {
@@ -213,6 +225,27 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj)
 }
 #   define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj))
 
+static inline bool
+PyStackRef_IsBorrowed(_PyStackRef stackref)
+{
+    if (PyStackRef_IsNull(stackref) || !PyStackRef_IsDeferred(stackref)) {
+        return false;
+    }
+    PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);
+    return !(_Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj));
+}
+
+
+static inline _PyStackRef
+_PyStackRef_NewIfBorrowedOrSteal(_PyStackRef stackref)
+{
+    if (PyStackRef_IsBorrowed(stackref)) {
+        PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);
+        return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR };
+    }
+    return stackref;
+}
+
 static inline _PyStackRef
 PyStackRef_FromPyObjectNew(PyObject *obj)
 {
@@ -253,15 +286,18 @@ PyStackRef_DUP(_PyStackRef stackref)
 {
     assert(!PyStackRef_IsNull(stackref));
     if (PyStackRef_IsDeferred(stackref)) {
-        assert(_Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) ||
-               _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref))
-        );
         return stackref;
     }
     Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref));
     return stackref;
 }
 
+static inline _PyStackRef
+PyStackRef_AsDeferred(_PyStackRef stackref)
+{
+    return (_PyStackRef){ .bits = stackref.bits | Py_TAG_DEFERRED };
+}
+
 // Convert a possibly deferred reference to a strong reference.
 static inline _PyStackRef
 PyStackRef_AsStrongReference(_PyStackRef stackref)
@@ -280,6 +316,7 @@ static const _PyStackRef PyStackRef_NULL = { .bits = 0 };
 #define PyStackRef_True ((_PyStackRef){.bits = (uintptr_t)&_Py_TrueStruct })
 #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) })
 #define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) })
+#define PyStackRef_IsBorrowed(stackref) false
 
 #define PyStackRef_AsPyObjectBorrow(stackref) ((PyObject *)(stackref).bits)
 
@@ -295,6 +332,8 @@ static const _PyStackRef PyStackRef_NULL = { .bits = 0 };
 
 #define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)))
 
+#define _PyStackRef_NewIfBorrowedOrSteal(stackref) stackref
+
 #define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) _Py_DECREF_SPECIALIZED(PyStackRef_AsPyObjectBorrow(stackref), dealloc)
 
 #endif // Py_GIL_DISABLED