Update stacklet

Author: saghul

setup.py

@@ -30,7 +30,7 @@
             os.system('ml64 /nologo /c /Fo src\switch_x64_msvc.obj src\switch_x64_msvc.asm')
 
     ext_modules  = [Extension('fibers._cfibers',
-                              sources = glob.glob('src/*.c'),
+                              sources=glob.glob('src/*.c'),
                               extra_objects=extra_objects,
                              )]
 

src/slp_platformselect.h

@@ -10,8 +10,12 @@
 #include "switch_x86_gcc.h" /* gcc on X86 */
 #elif defined(__GNUC__) && defined(__arm__)
 #include "switch_arm_gcc.h" /* gcc on arm */
+#elif defined(__GNUC__) && defined(__PPC64__)
+#include "switch_ppc64_gcc.h" /* gcc on ppc64 */
 #elif defined(__GNUC__) && defined(__mips__) && defined(_ABI64)
 #include "switch_mips64_gcc.h" /* gcc on mips64 */
+#elif defined(__GNUC__) && defined(__s390x__)
+#include "switch_s390x_gcc.h"
 #else
 #error "Unsupported platform!"
 #endif

src/stacklet.c

@@ -5,8 +5,8 @@
 #include "stacklet.h"
 
 #include <stddef.h>
-#include <assert.h>
 #include <string.h>
+#include <stdio.h>
 
 /************************************************************
  * platform specific code
@@ -16,6 +16,7 @@
  * can redefine it to upwards growing, 1.
  */
 #define STACK_DIRECTION 0   
+#define STATIC_NOINLINE   __attribute__((noinline)) static
 
 #include "slp_platformselect.h"
 
@@ -34,7 +35,7 @@
 /************************************************************/
 
 struct stacklet_s {
-    /* The portion of the real stack claimed by this paused tealet. */
+    /* The portion of the real stack claimed by this paused stacklet. */
     char *stack_start;                /* the "near" end of the stack */
     char *stack_stop;                 /* the "far" end of the stack */
 
@@ -56,11 +57,6 @@ struct stacklet_s {
     stacklet_thread_handle stack_thrd;  /* the thread where the stacklet is */
 };
 
-void *(*_stacklet_switchstack)(void*(*)(void*, void*),
-                               void*(*)(void*, void*), void*) = NULL;
-void (*_stacklet_initialstub)(struct stacklet_thread_s *,
-                              stacklet_run_fn, void *) = NULL;
-
 struct stacklet_thread_s {
     struct stacklet_s *g_stack_chain_head;  /* NULL <=> running main */
     char *g_current_stack_stop;
@@ -69,6 +65,19 @@ struct stacklet_thread_s {
     struct stacklet_s *g_target;
 };
 
+#define _check(x)  do { if (!(x)) _check_failed(#x); } while (0)
+
+static void _check_failed(const char *check)
+{
+    fprintf(stderr, "FATAL: stacklet: %s failed\n", check);
+    abort();
+}
+
+static void check_valid(struct stacklet_s *g)
+{
+    _check(g->stack_saved >= 0);
+}
+
 /***************************************************************/
 
 static void g_save(struct stacklet_s* g, char* stop
@@ -96,7 +105,8 @@ static void g_save(struct stacklet_s* g, char* stop
      */
     ptrdiff_t sz1 = g->stack_saved;
     ptrdiff_t sz2 = stop - g->stack_start;
-    assert(stop <= g->stack_stop);
+    check_valid(g);
+    _check(stop <= g->stack_stop);
 
     if (sz2 > sz1) {
         char *c = (char *)(g + 1);
@@ -146,11 +156,13 @@ static void g_clear_stack(struct stacklet_s *g_target,
 {
     struct stacklet_s *current = thrd->g_stack_chain_head;
     char *target_stop = g_target->stack_stop;
+    check_valid(g_target);
 
-    /* save and unlink tealets that are completely within
+    /* save and unlink stacklets that are completely within
        the area to clear. */
     while (current != NULL && current->stack_stop <= target_stop) {
         struct stacklet_s *prev = current->stack_prev;
+        check_valid(current);
         current->stack_prev = NULL;
         if (current != g_target) {
             /* don't bother saving away g_target, because
@@ -222,20 +234,31 @@ static void *g_restore_state(void *new_stack_pointer, void *rawthrd)
     struct stacklet_thread_s *thrd = (struct stacklet_thread_s *)rawthrd;
     struct stacklet_s *g = thrd->g_target;
     ptrdiff_t stack_saved = g->stack_saved;
+    check_valid(g);
 
-    assert(new_stack_pointer == g->stack_start);
+    _check(new_stack_pointer == g->stack_start);
 #if STACK_DIRECTION == 0
     memcpy(g->stack_start, g+1, stack_saved);
 #else
     memcpy(g->stack_start - stack_saved, g+1, stack_saved);
 #endif
     thrd->g_current_stack_stop = g->stack_stop;
+    g->stack_saved = -13;   /* debugging */
     free(g);
     return EMPTY_STACKLET_HANDLE;
 }
 
-static void g_initialstub(struct stacklet_thread_s *thrd,
-                          stacklet_run_fn run, void *run_arg)
+STATIC_NOINLINE
+void *_stacklet_switchstack(void *(*save_state)(void*, void*),
+                            void *(*restore_state)(void*, void*),
+                            void *extra)
+{
+    return slp_switch(save_state, restore_state, extra);
+}
+
+STATIC_NOINLINE
+void g_initialstub(struct stacklet_thread_s *thrd,
+                   stacklet_run_fn run, void *run_arg)
 {
     struct stacklet_s *result;
 
@@ -250,10 +273,11 @@ static void g_initialstub(struct stacklet_thread_s *thrd,
         result = run(thrd->g_source, run_arg);
 
         /* Then switch to 'result'. */
+        check_valid(result);
         thrd->g_target = result;
         _stacklet_switchstack(g_destroy_state, g_restore_state, thrd);
 
-        assert(!"stacklet: we should not return here");
+        _check_failed("we should not return here");
         abort();
     }
     /* The second time it returns. */
@@ -265,13 +289,6 @@ stacklet_thread_handle stacklet_newthread(void)
 {
     struct stacklet_thread_s *thrd;
 
-    if (_stacklet_switchstack == NULL) {
-        /* set up the following global with an indirection, which is needed
-           to prevent any inlining */
-        _stacklet_initialstub = g_initialstub;
-        _stacklet_switchstack = slp_switch;
-    }
-
     thrd = malloc(sizeof(struct stacklet_thread_s));
     if (thrd != NULL)
         memset(thrd, 0, sizeof(struct stacklet_thread_s));
@@ -287,19 +304,20 @@ stacklet_handle stacklet_new(stacklet_thread_handle thrd,
                              stacklet_run_fn run, void *run_arg)
 {
     long stackmarker;
-    assert((char *)NULL < (char *)&stackmarker);
+    _check((char *)NULL < (char *)&stackmarker);
     if (thrd->g_current_stack_stop <= (char *)&stackmarker)
         thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
 
     thrd->g_current_stack_marker = (char *)&stackmarker;
-    _stacklet_initialstub(thrd, run, run_arg);
+    g_initialstub(thrd, run, run_arg);
     return thrd->g_source;
 }
 
 stacklet_handle stacklet_switch(stacklet_handle target)
 {
     long stackmarker;
     stacklet_thread_handle thrd = target->stack_thrd;
+    check_valid(target);
     if (thrd->g_current_stack_stop <= (char *)&stackmarker)
         thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
 
@@ -310,6 +328,7 @@ stacklet_handle stacklet_switch(stacklet_handle target)
 
 void stacklet_destroy(stacklet_handle target)
 {
+    check_valid(target);
     if (target->stack_prev != NULL) {
         /* 'target' appears to be in the chained list 'unsaved_stack',
            so remove it from there.  Note that if 'thrd' was already
@@ -319,12 +338,15 @@ void stacklet_destroy(stacklet_handle target)
            we don't even read 'stack_thrd', already deallocated. */
         stacklet_thread_handle thrd = target->stack_thrd;
         struct stacklet_s **pp = &thrd->g_stack_chain_head;
-        for (; *pp != NULL; pp = &(*pp)->stack_prev)
+        for (; *pp != NULL; pp = &(*pp)->stack_prev) {
+            check_valid(*pp);
             if (*pp == target) {
                 *pp = target->stack_prev;
                 break;
             }
+        }
     }
+    target->stack_saved = -11;   /* debugging */
     free(target);
 }
 
@@ -334,6 +356,7 @@ char **_stacklet_translate_pointer(stacklet_handle context, char **ptr)
   long delta;
   if (context == NULL)
     return ptr;
+  check_valid(context);
   delta = p - context->stack_start;
   if (((unsigned long)delta) < ((unsigned long)context->stack_saved)) {
       /* a pointer to a saved away word */
@@ -345,8 +368,8 @@ char **_stacklet_translate_pointer(stacklet_handle context, char **ptr)
       /* out-of-stack pointer!  it's only ok if we are the main stacklet
          and we are reading past the end, because the main stacklet's
          stack stop is not exactly known. */
-      assert(delta >= 0);
-      assert(((long)context->stack_stop) & 1);
+      _check(delta >= 0);
+      _check(((long)context->stack_stop) & 1);
   }
   return ptr;
 }

src/switch_arm_gcc.h

@@ -5,16 +5,47 @@
 # define call_reg(x) "blx " #x "\n"
 #endif
 
-static void __attribute__((optimize("O3"))) *slp_switch(void *(*save_state)(void*, void*),
+static void *slp_switch(void *(*save_state)(void*, void*),
+                        void *(*restore_state)(void*, void*),
+                        void *extra) __attribute__((noinline));
+
+static void *slp_switch(void *(*save_state)(void*, void*),
                         void *(*restore_state)(void*, void*),
                         void *extra)
 {
   void *result;
+  /*
+      seven registers to preserve: r2, r3, r7, r8, r9, r10, r11
+      registers marked as clobbered: r0, r1, r4, r5, r6, r12, lr
+      others: r13 is sp; r14 is lr; r15 is pc
+  */
+
   __asm__ volatile (
-    "mov r3, %[save_state]\n"
-    /* save values in calee saved registers for later */
-    "mov r4, %[restore_state]\n"
-    "mov r5, %[extra]\n"
+
+    /* align the stack and save 7 more registers explicitly */
+    "mov r0, sp\n"
+    "and r1, r0, #-16\n"
+    "mov sp, r1\n"
+    "push {r0, r2, r3, r7, r8, r9, r10, r11}\n"   /* total 8, still aligned */
+#ifndef __SOFTFP__
+    /* We also push d8-d15 to preserve them explicitly.  This assumes
+     * that this code is in a function that doesn't use floating-point
+     * at all, and so don't touch the "d" registers (that's why we mark
+     * it as non-inlinable).  So here by pushing/poping d8-d15 we are
+     * saving precisely the callee-saved registers in all cases.  We
+     * could also try to list all "d" registers as clobbered, but it
+     * doesn't work: there is no way I could find to know if we have 16
+     * or 32 "d" registers (depends on the exact -mcpu=... and we don't
+     * know it from the C code).  If we have 32, then gcc would "save"
+     * d8-d15 by copying them into d16-d23 for example, and it doesn't
+     * work. */
+    "vpush {d8, d9, d10, d11, d12, d13, d14, d15}\n"  /* 16 words, still aligned */
+#endif
+
+    /* save values in callee saved registers for later */
+    "mov r4, %[restore_state]\n"  /* can't be r0 or r1: marked clobbered */
+    "mov r5, %[extra]\n"          /* can't be r0 or r1 or r4: marked clob. */
+    "mov r3, %[save_state]\n"     /* can't be r0, r1, r4, r5: marked clob. */
     "mov r0, sp\n"        	/* arg 1: current (old) stack pointer */
     "mov r1, r5\n"        	/* arg 2: extra                       */
     call_reg(r3)		/* call save_state()                  */
@@ -28,19 +59,26 @@ static void __attribute__((optimize("O3"))) *slp_switch(void *(*save_state)(void
 	/* From now on, the stack pointer is modified, but the content of the
 	stack is not restored yet.  It contains only garbage here. */
     "mov r1, r5\n"       	/* arg 2: extra                       */
-   	 						/* arg 1: current (new) stack pointer is already in r0*/
+                /* arg 1: current (new) stack pointer is already in r0*/
     call_reg(r4)		/* call restore_state()               */
 
     /* The stack's content is now restored. */
     "zero:\n"
+
+#ifndef __SOFTFP__
+    "vpop {d8, d9, d10, d11, d12, d13, d14, d15}\n"
+#endif
+    "pop {r1, r2, r3, r7, r8, r9, r10, r11}\n"
+    "mov sp, r1\n"
     "mov %[result], r0\n"
 
     : [result]"=r"(result)	/* output variables */
 	/* input variables  */
     : [restore_state]"r"(restore_state),
       [save_state]"r"(save_state),
       [extra]"r"(extra)
-    : "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r13"
+    : "r0", "r1", "r4", "r5", "r6", "r12", "lr",
+      "memory", "cc"
   );
   return result;
 }