GH-116422: Tier2 hot/cold splitting (GH-116813)

Splits the "cold" path, deopts and exits, from the "hot" path, reducing the size of most jitted instructions, at the cost of slower exits.

Author: markshannon

Include/cpython/optimizer.h

@@ -30,16 +30,63 @@ typedef struct {
     PyCodeObject *code;  // Weak (NULL if no corresponding ENTER_EXECUTOR).
 } _PyVMData;
 
+#define UOP_FORMAT_TARGET 0
+#define UOP_FORMAT_EXIT 1
+#define UOP_FORMAT_JUMP 2
+#define UOP_FORMAT_UNUSED 3
+
+/* Depending on the format,
+ * the 32 bits between the oparg and operand are:
+ * UOP_FORMAT_TARGET:
+ *    uint32_t target;
+ * UOP_FORMAT_EXIT
+ *    uint16_t exit_index;
+ *    uint16_t error_target;
+ * UOP_FORMAT_JUMP
+ *    uint16_t jump_target;
+ *    uint16_t error_target;
+ */
 typedef struct {
-    uint16_t opcode;
+    uint16_t opcode:14;
+    uint16_t format:2;
     uint16_t oparg;
     union {
         uint32_t target;
-        uint32_t exit_index;
+        struct {
+            union {
+                uint16_t exit_index;
+                uint16_t jump_target;
+            };
+            uint16_t error_target;
+        };
     };
     uint64_t operand;  // A cache entry
 } _PyUOpInstruction;
 
+static inline uint32_t uop_get_target(const _PyUOpInstruction *inst)
+{
+    assert(inst->format == UOP_FORMAT_TARGET);
+    return inst->target;
+}
+
+static inline uint16_t uop_get_exit_index(const _PyUOpInstruction *inst)
+{
+    assert(inst->format == UOP_FORMAT_EXIT);
+    return inst->exit_index;
+}
+
+static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst)
+{
+    assert(inst->format == UOP_FORMAT_JUMP);
+    return inst->jump_target;
+}
+
+static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
+{
+    assert(inst->format != UOP_FORMAT_TARGET);
+    return inst->error_target;
+}
+
 typedef struct _exit_data {
     uint32_t target;
     int16_t temperature;

Include/internal/pycore_opcode_metadata.h

@@ -12,7 +12,7 @@ extern "C" {
 #include <stdbool.h>
 
 // This is the length of the trace we project initially.
-#define UOP_MAX_TRACE_LENGTH 512
+#define UOP_MAX_TRACE_LENGTH 800
 
 #define TRACE_STACK_SIZE 5