python · markshannon · Jun 11, 2025 · Jun 13, 2025 · Jun 13, 2025 · Jun 13, 2025
diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h
@@ -29,7 +29,7 @@
 #  error "this header file must not be included directly"
 #endif
 
-#define PYSTATS_MAX_UOP_ID 1024
+#define PYSTATS_MAX_UOP_ID 1500
 
 #define SPECIALIZATION_FAILURE_KINDS 60
 

@@ -15,7 +15,7 @@ extern "C" {
 
 #ifdef _Py_JIT
 
-typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate);
+typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2);
 
 int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length);
 void _PyJIT_Free(_PyExecutorObject *executor);

diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h
diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
@@ -120,7 +120,7 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp);
 #define JIT_CLEANUP_THRESHOLD 100000
 
 // This is the length of the trace we project initially.
-#define UOP_MAX_TRACE_LENGTH 800
+#define UOP_MAX_TRACE_LENGTH 1600
 
 #define TRACE_STACK_SIZE 5
 

@@ -70,6 +70,9 @@ static const _PyStackRef PyStackRef_ERROR = { .index = 2 };
 #define PyStackRef_False ((_PyStackRef){ .index = 6 })
 #define PyStackRef_True ((_PyStackRef){ .index = 8 })
 
+#define PyStackRef_ZERO_BITS PyStackRef_NULL
+
+
 #define INITIAL_STACKREF_INDEX 10
 
 static inline int
@@ -264,6 +267,10 @@ PyStackRef_IsNullOrInt(_PyStackRef ref);
 
 static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID };
 
+/* For use in the JIT to clear an unused value.
+ * PyStackRef_ZERO_BITS has no meaning and should not be used other than by the JIT. */
+static const _PyStackRef PyStackRef_ZERO_BITS = { .bits = 0 };
+
 /* Wrap a pointer in a stack ref.
  * The resulting stack reference is not safe and should only be used
  * in the interpreter to pass values from one uop to another.
@@ -303,6 +310,16 @@ PyStackRef_IsValid(_PyStackRef ref)
     return ref.bits >= Py_INT_TAG;
 }
 
+static inline bool
+PyStackRef_IsWrapped(_PyStackRef ref)
+{
+#ifdef Py_DEBUG
+    return (ref.bits & Py_TAG_INVALID) != 0 && !PyStackRef_IsError(ref);
+#else
+    Py_FatalError("Cannot determine if value is wrapped if Py_DEBUG is not set");
+#endif
+}
+
 static inline bool
 PyStackRef_IsTaggedInt(_PyStackRef i)
 {
@@ -810,6 +827,17 @@ _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref)
     PyStackRef_XCLOSE(ref->ref);
 }
 
+static inline _PyStackRef
+_PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref)
+{
+#ifdef Py_GIL_DISABLED
+    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
+    assert(tstate_impl->c_stack_refs == ref);
+    tstate_impl->c_stack_refs = ref->next;
+#endif
+    return ref->ref;
+}
+
 #ifdef Py_GIL_DISABLED
 
 static inline int

diff --git a/Include/internal/pycore_stats.h b/Include/internal/pycore_stats.h
@@ -28,7 +28,7 @@ extern "C" {
 #define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0)
 #define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0)
 #define OPT_STAT_ADD(name, n) do { if (_Py_stats) _Py_stats->optimization_stats.name += (n); } while (0)
-#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0)
+#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { _Py_stats->optimization_stats.opcode[opname].name++; } } while (0)
 #define UOP_PAIR_INC(uopcode, lastuop)                                              \
     do {                                                                            \
         if (lastuop && _Py_stats) {                                                 \