Store the fiber_serial in the EC to allow inlining

Mutexes spend a significant amount of time in `rb_fiber_serial`
because it can't be inlined (except with LTO).
The fiber struct is opaque the so function can't be defined as inlineable.

Ideally the while fiber struct would not be opaque to the rest of
Ruby core, but it's tricky to do.

Instead we can store the fiber serial in the execution context
itself, and make its access cheaper:

```
$ hyperfine './miniruby-baseline --yjit /tmp/mut.rb' './miniruby-inline-serial --yjit /tmp/mut.rb'
Benchmark 1: ./miniruby-baseline --yjit /tmp/mut.rb
  Time (mean ± σ):      4.011 s ±  0.084 s    [User: 3.977 s, System: 0.011 s]
  Range (min … max):    3.950 s …  4.245 s    10 runs

Benchmark 2: ./miniruby-inline-serial --yjit /tmp/mut.rb
  Time (mean ± σ):      3.495 s ±  0.150 s    [User: 3.448 s, System: 0.009 s]
  Range (min … max):    3.340 s …  3.869 s    10 runs

Summary
  ./miniruby-inline-serial --yjit /tmp/mut.rb ran
    1.15 ± 0.05 times faster than ./miniruby-baseline --yjit /tmp/mut.rb
```

```ruby
i = 10_000_000
mut = Mutex.new
while i > 0
  i -= 1
  mut.synchronize { }
  mut.synchronize { }
  mut.synchronize { }
  mut.synchronize { }
  mut.synchronize { }
  mut.synchronize { }
  mut.synchronize { }
  mut.synchronize { }
  mut.synchronize { }
  mut.synchronize { }
end
```

Author: byroot

cont.c

@@ -268,8 +268,6 @@ struct rb_fiber_struct {
 
     unsigned int killed : 1;
 
-    rb_serial_t serial;
-
     struct coroutine_context context;
     struct fiber_pool_stack stack;
 };
@@ -1012,13 +1010,6 @@ rb_fiber_threadptr(const rb_fiber_t *fiber)
     return fiber->cont.saved_ec.thread_ptr;
 }
 
-rb_serial_t
-rb_fiber_serial(const rb_fiber_t *fiber)
-{
-    VM_ASSERT(fiber->serial >= 1);
-    return fiber->serial;
-}
-
 static VALUE
 cont_thread_value(const rb_context_t *cont)
 {
@@ -2026,10 +2017,10 @@ fiber_t_alloc(VALUE fiber_value, unsigned int blocking)
     fiber->cont.type = FIBER_CONTEXT;
     fiber->blocking = blocking;
     fiber->killed = 0;
-    fiber->serial = next_fiber_serial(th->ractor);
     cont_init(&fiber->cont, th);
 
     fiber->cont.saved_ec.fiber_ptr = fiber;
+    fiber->cont.saved_ec.fiber_serial = next_fiber_serial(th->ractor);
     rb_ec_clear_vm_stack(&fiber->cont.saved_ec);
 
     fiber->prev = NULL;
@@ -2576,10 +2567,10 @@ rb_threadptr_root_fiber_setup(rb_thread_t *th)
     }
     fiber->cont.type = FIBER_CONTEXT;
     fiber->cont.saved_ec.fiber_ptr = fiber;
+    fiber->cont.saved_ec.fiber_serial = next_fiber_serial(th->ractor);
     fiber->cont.saved_ec.thread_ptr = th;
     fiber->blocking = 1;
     fiber->killed = 0;
-    fiber->serial = next_fiber_serial(th->ractor);
     fiber_status_set(fiber, FIBER_RESUMED); /* skip CREATED */
     th->ec = &fiber->cont.saved_ec;
     cont_init_jit_cont(&fiber->cont);

internal/cont.h

@@ -31,6 +31,11 @@ VALUE rb_fiber_inherit_storage(struct rb_execution_context_struct *ec, struct rb
 VALUE rb_fiberptr_self(struct rb_fiber_struct *fiber);
 unsigned int rb_fiberptr_blocking(struct rb_fiber_struct *fiber);
 struct rb_execution_context_struct * rb_fiberptr_get_ec(struct rb_fiber_struct *fiber);
-rb_serial_t rb_fiber_serial(const struct rb_fiber_struct *fiber);
 
+static inline rb_serial_t
+rb_ec_fiber_serial(struct rb_execution_context_struct *ec)
+{
+    VM_ASSERT(ec->fiber_serial >= 1);
+    return ec->fiber_serial;
+}
 #endif /* INTERNAL_CONT_H */

thread.c

@@ -454,7 +454,7 @@ rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th)
 
         // rb_warn("mutex #<%p> was not unlocked by thread #<%p>", (void *)mutex, (void*)th);
         VM_ASSERT(mutex->fiber_serial);
-        const char *error_message = rb_mutex_unlock_th(mutex, th, NULL);
+        const char *error_message = rb_mutex_unlock_th(mutex, th, 0);
         if (error_message) rb_bug("invalid keeping_mutexes: %s", error_message);
     }
 }
@@ -5283,7 +5283,7 @@ rb_thread_shield_owned(VALUE self)
 
     rb_mutex_t *m = mutex_ptr(mutex);
 
-    return m->fiber_serial == rb_fiber_serial(GET_EC()->fiber_ptr);
+    return m->fiber_serial == rb_ec_fiber_serial(GET_EC());
 }
 
 /*
@@ -5302,7 +5302,7 @@ rb_thread_shield_wait(VALUE self)
 
     if (!mutex) return Qfalse;
     m = mutex_ptr(mutex);
-    if (m->fiber_serial == rb_fiber_serial(GET_EC()->fiber_ptr)) return Qnil;
+    if (m->fiber_serial == rb_ec_fiber_serial(GET_EC())) return Qnil;
     rb_thread_shield_waiting_inc(self);
     rb_mutex_lock(mutex);
     rb_thread_shield_waiting_dec(self);
@@ -5860,7 +5860,7 @@ rb_check_deadlock(rb_ractor_t *r)
         }
         else if (th->locking_mutex) {
             rb_mutex_t *mutex = mutex_ptr(th->locking_mutex);
-            if (mutex->fiber_serial == rb_fiber_serial(th->ec->fiber_ptr) || (!mutex->fiber_serial && !ccan_list_empty(&mutex->waitq))) {
+            if (mutex->fiber_serial == rb_ec_fiber_serial(th->ec) || (!mutex->fiber_serial && !ccan_list_empty(&mutex->waitq))) {
                 found = 1;
             }
         }

thread_sync.c

@@ -81,7 +81,7 @@ static void rb_mutex_abandon_all(rb_mutex_t *mutexes);
 static void rb_mutex_abandon_keeping_mutexes(rb_thread_t *th);
 static void rb_mutex_abandon_locking_mutex(rb_thread_t *th);
 #endif
-static const char* rb_mutex_unlock_th(rb_mutex_t *mutex, rb_thread_t *th, rb_fiber_t *fiber);
+static const char* rb_mutex_unlock_th(rb_mutex_t *mutex, rb_thread_t *th, rb_serial_t fiber_serial);
 
 /*
  *  Document-class: Thread::Mutex
@@ -133,7 +133,7 @@ mutex_free(void *ptr)
 {
     rb_mutex_t *mutex = ptr;
     if (mutex_locked_p(mutex)) {
-        const char *err = rb_mutex_unlock_th(mutex, rb_thread_ptr(mutex->thread), NULL);
+        const char *err = rb_mutex_unlock_th(mutex, rb_thread_ptr(mutex->thread), 0);
         if (err) rb_bug("%s", err);
     }
     ruby_xfree(ptr);
@@ -221,26 +221,26 @@ thread_mutex_remove(rb_thread_t *thread, rb_mutex_t *mutex)
 }
 
 static void
-mutex_set_owner(rb_mutex_t *mutex, rb_thread_t *th, rb_fiber_t *fiber)
+mutex_set_owner(rb_mutex_t *mutex, rb_thread_t *th, rb_serial_t fiber_serial)
 {
     mutex->thread = th->self;
-    mutex->fiber_serial = rb_fiber_serial(fiber);
+    mutex->fiber_serial = fiber_serial;
 }
 
 static void
-mutex_locked(rb_mutex_t *mutex, rb_thread_t *th, rb_fiber_t *fiber)
+mutex_locked(rb_mutex_t *mutex, rb_thread_t *th, rb_serial_t fiber_serial)
 {
-    mutex_set_owner(mutex, th, fiber);
+    mutex_set_owner(mutex, th, fiber_serial);
     thread_mutex_insert(th, mutex);
 }
 
 static inline bool
-mutex_trylock(rb_mutex_t *mutex, rb_thread_t *th, rb_fiber_t *fiber)
+mutex_trylock(rb_mutex_t *mutex, rb_thread_t *th, rb_serial_t fiber_serial)
 {
     if (mutex->fiber_serial == 0) {
         RUBY_DEBUG_LOG("%p ok", mutex);
 
-        mutex_locked(mutex, th, fiber);
+        mutex_locked(mutex, th, fiber_serial);
         return true;
     }
     else {
@@ -252,7 +252,7 @@ mutex_trylock(rb_mutex_t *mutex, rb_thread_t *th, rb_fiber_t *fiber)
 static VALUE
 rb_mut_trylock(rb_execution_context_t *ec, VALUE self)
 {
-    return RBOOL(mutex_trylock(mutex_ptr(self), ec->thread_ptr, ec->fiber_ptr));
+    return RBOOL(mutex_trylock(mutex_ptr(self), ec->thread_ptr, rb_ec_fiber_serial(ec)));
 }
 
 VALUE
@@ -262,9 +262,9 @@ rb_mutex_trylock(VALUE self)
 }
 
 static VALUE
-mutex_owned_p(rb_fiber_t *fiber, rb_mutex_t *mutex)
+mutex_owned_p(rb_serial_t fiber_serial, rb_mutex_t *mutex)
 {
-    return RBOOL(mutex->fiber_serial == rb_fiber_serial(fiber));
+    return RBOOL(mutex->fiber_serial == fiber_serial);
 }
 
 static VALUE
@@ -305,6 +305,7 @@ do_mutex_lock(struct mutex_args *args, int interruptible_p)
     rb_execution_context_t *ec = args->ec;
     rb_thread_t *th = ec->thread_ptr;
     rb_fiber_t *fiber = ec->fiber_ptr;
+    rb_serial_t fiber_serial = rb_ec_fiber_serial(ec);
     rb_mutex_t *mutex = args->mutex;
     rb_atomic_t saved_ints = 0;
 
@@ -314,12 +315,12 @@ do_mutex_lock(struct mutex_args *args, int interruptible_p)
         rb_raise(rb_eThreadError, "can't be called from trap context");
     }
 
-    if (!mutex_trylock(mutex, th, fiber)) {
-        if (mutex->fiber_serial == rb_fiber_serial(fiber)) {
+    if (!mutex_trylock(mutex, th, fiber_serial)) {
+        if (mutex->fiber_serial == fiber_serial) {
             rb_raise(rb_eThreadError, "deadlock; recursive locking");
         }
 
-        while (mutex->fiber_serial != rb_fiber_serial(fiber)) {
+        while (mutex->fiber_serial != fiber_serial) {
             VM_ASSERT(mutex->fiber_serial != 0);
 
             VALUE scheduler = rb_fiber_scheduler_current();
@@ -335,7 +336,7 @@ do_mutex_lock(struct mutex_args *args, int interruptible_p)
                 rb_ensure(call_rb_fiber_scheduler_block, self, delete_from_waitq, (VALUE)&sync_waiter);
 
                 if (!mutex->fiber_serial) {
-                    mutex_set_owner(mutex, th, fiber);
+                    mutex_set_owner(mutex, th, fiber_serial);
                 }
             }
             else {
@@ -376,7 +377,7 @@ do_mutex_lock(struct mutex_args *args, int interruptible_p)
 
                 // unlocked by another thread while sleeping
                 if (!mutex->fiber_serial) {
-                    mutex_set_owner(mutex, th, fiber);
+                    mutex_set_owner(mutex, th, fiber_serial);
                 }
 
                 rb_ractor_sleeper_threads_dec(th->ractor);
@@ -389,13 +390,13 @@ do_mutex_lock(struct mutex_args *args, int interruptible_p)
             if (interruptible_p) {
                 /* release mutex before checking for interrupts...as interrupt checking
                  * code might call rb_raise() */
-                if (mutex->fiber_serial == rb_fiber_serial(fiber)) {
+                if (mutex->fiber_serial == fiber_serial) {
                     mutex->thread = Qfalse;
                     mutex->fiber_serial = 0;
                 }
                 RUBY_VM_CHECK_INTS_BLOCKING(th->ec); /* may release mutex */
                 if (!mutex->fiber_serial) {
-                    mutex_set_owner(mutex, th, fiber);
+                    mutex_set_owner(mutex, th, fiber_serial);
                 }
             }
             else {
@@ -414,13 +415,13 @@ do_mutex_lock(struct mutex_args *args, int interruptible_p)
         }
 
         if (saved_ints) th->ec->interrupt_flag = saved_ints;
-        if (mutex->fiber_serial == rb_fiber_serial(fiber)) mutex_locked(mutex, th, fiber);
+        if (mutex->fiber_serial == fiber_serial) mutex_locked(mutex, th, fiber_serial);
     }
 
     RUBY_DEBUG_LOG("%p locked", mutex);
 
     // assertion
-    if (mutex_owned_p(fiber, mutex) == Qfalse) rb_bug("do_mutex_lock: mutex is not owned.");
+    if (mutex_owned_p(fiber_serial, mutex) == Qfalse) rb_bug("do_mutex_lock: mutex is not owned.");
 
     return self;
 }
@@ -455,7 +456,7 @@ rb_mutex_lock(VALUE self)
 static VALUE
 rb_mut_owned_p(rb_execution_context_t *ec, VALUE self)
 {
-    return mutex_owned_p(ec->fiber_ptr, mutex_ptr(self));
+    return mutex_owned_p(rb_ec_fiber_serial(ec), mutex_ptr(self));
 }
 
 VALUE
@@ -465,14 +466,14 @@ rb_mutex_owned_p(VALUE self)
 }
 
 static const char *
-rb_mutex_unlock_th(rb_mutex_t *mutex, rb_thread_t *th, rb_fiber_t *fiber)
+rb_mutex_unlock_th(rb_mutex_t *mutex, rb_thread_t *th, rb_serial_t fiber_serial)
 {
     RUBY_DEBUG_LOG("%p", mutex);
 
     if (mutex->fiber_serial == 0) {
         return "Attempt to unlock a mutex which is not locked";
     }
-    else if (fiber && mutex->fiber_serial != rb_fiber_serial(fiber)) {
+    else if (fiber_serial && mutex->fiber_serial != fiber_serial) {
         return "Attempt to unlock a mutex which is locked by another thread/fiber";
     }
 
@@ -516,7 +517,7 @@ do_mutex_unlock(struct mutex_args *args)
     rb_mutex_t *mutex = args->mutex;
     rb_thread_t *th = rb_ec_thread_ptr(args->ec);
 
-    err = rb_mutex_unlock_th(mutex, th, args->ec->fiber_ptr);
+    err = rb_mutex_unlock_th(mutex, th, rb_ec_fiber_serial(args->ec));
     if (err) rb_raise(rb_eThreadError, "%s", err);
 }
 

vm_core.h

@@ -1041,6 +1041,7 @@ struct rb_execution_context_struct {
 
     rb_fiber_t *fiber_ptr;
     struct rb_thread_struct *thread_ptr;
+    rb_serial_t fiber_serial;
 
     /* storage (ec (fiber) local) */
     struct rb_id_table *local_storage;