ZJIT: Share more code with YJIT in jit.c (ruby#14520)

* ZJIT: Share more code with YJIT in jit.c

* Fix ZJIT references to JIT

Author: k0kubun

jit.c

@@ -533,10 +533,184 @@ for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data)
     return 0;
 }
 
+uint32_t
+rb_jit_get_page_size(void)
+{
+#if defined(_SC_PAGESIZE)
+    long page_size = sysconf(_SC_PAGESIZE);
+    if (page_size <= 0) rb_bug("jit: failed to get page size");
+
+    // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected.
+    // Though our design sort of assume we have fine grained control over memory protection
+    // which require small page sizes.
+    if (page_size > 0x40000000l) rb_bug("jit page size too large");
+
+    return (uint32_t)page_size;
+#else
+#error "JIT supports POSIX only for now"
+#endif
+}
+
+#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
+// Align the current write position to a multiple of bytes
+static uint8_t *
+align_ptr(uint8_t *ptr, uint32_t multiple)
+{
+    // Compute the pointer modulo the given alignment boundary
+    uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
+
+    // If the pointer is already aligned, stop
+    if (rem == 0)
+        return ptr;
+
+    // Pad the pointer by the necessary amount to align it
+    uint32_t pad = multiple - rem;
+
+    return ptr + pad;
+}
+#endif
+
+// Address space reservation. Memory pages are mapped on an as needed basis.
+// See the Rust mm module for details.
+uint8_t *
+rb_jit_reserve_addr_space(uint32_t mem_size)
+{
+#ifndef _WIN32
+    uint8_t *mem_block;
+
+    // On Linux
+    #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
+        uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE);
+        uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_jit_reserve_addr_space;
+        uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX;
+        // Align the requested address to page size
+        uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);
+
+        // Probe for addresses close to this function using MAP_FIXED_NOREPLACE
+        // to improve odds of being in range for 32-bit relative call instructions.
+        do {
+            mem_block = mmap(
+                req_addr,
+                mem_size,
+                PROT_NONE,
+                MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
+                -1,
+                0
+            );
+
+            // If we succeeded, stop
+            if (mem_block != MAP_FAILED) {
+                ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_jit_reserve_addr_space");
+                break;
+            }
+
+            // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux
+            // main_code_addr < heap_addr, and in case we are in a shared
+            // library mapped higher than the heap, downwards is still better
+            // since it's towards the end of the heap rather than the stack.)
+            req_addr -= 4 * 1024 * 1024;
+        } while (req_addr < probe_region_end);
+
+    // On MacOS and other platforms
+    #else
+        // Try to map a chunk of memory as executable
+        mem_block = mmap(
+            (void *)rb_jit_reserve_addr_space,
+            mem_size,
+            PROT_NONE,
+            MAP_PRIVATE | MAP_ANONYMOUS,
+            -1,
+            0
+        );
+    #endif
+
+    // Fallback
+    if (mem_block == MAP_FAILED) {
+        // Try again without the address hint (e.g., valgrind)
+        mem_block = mmap(
+            NULL,
+            mem_size,
+            PROT_NONE,
+            MAP_PRIVATE | MAP_ANONYMOUS,
+            -1,
+            0
+        );
+
+        if (mem_block != MAP_FAILED) {
+            ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_jit_reserve_addr_space:fallback");
+        }
+    }
+
+    // Check that the memory mapping was successful
+    if (mem_block == MAP_FAILED) {
+        perror("ruby: jit: mmap:");
+        if(errno == ENOMEM) {
+            // No crash report if it's only insufficient memory
+            exit(EXIT_FAILURE);
+        }
+        rb_bug("mmap failed");
+    }
+
+    return mem_block;
+#else
+    // Windows not supported for now
+    return NULL;
+#endif
+}
+
 // Walk all ISEQs in the heap and invoke the callback - shared between YJIT and ZJIT
 void
 rb_jit_for_each_iseq(rb_iseq_callback callback, void *data)
 {
     struct iseq_callback_data callback_data = { .callback = callback, .data = data };
     rb_objspace_each_objects(for_each_iseq_i, (void *)&callback_data);
 }
+
+bool
+rb_jit_mark_writable(void *mem_block, uint32_t mem_size)
+{
+    return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0;
+}
+
+void
+rb_jit_mark_executable(void *mem_block, uint32_t mem_size)
+{
+    // Do not call mprotect when mem_size is zero. Some platforms may return
+    // an error for it. https://github.com/Shopify/ruby/issues/450
+    if (mem_size == 0) {
+        return;
+    }
+    if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) {
+        rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s",
+            mem_block, (unsigned long)mem_size, strerror(errno));
+    }
+}
+
+// Free the specified memory block.
+bool
+rb_jit_mark_unused(void *mem_block, uint32_t mem_size)
+{
+    // On Linux, you need to use madvise MADV_DONTNEED to free memory.
+    // We might not need to call this on macOS, but it's not really documented.
+    // We generally prefer to do the same thing on both to ease testing too.
+    madvise(mem_block, mem_size, MADV_DONTNEED);
+
+    // On macOS, mprotect PROT_NONE seems to reduce RSS.
+    // We also call this on Linux to avoid executing unused pages.
+    return mprotect(mem_block, mem_size, PROT_NONE) == 0;
+}
+
+// Invalidate icache for arm64.
+// `start` is inclusive and `end` is exclusive.
+void
+rb_jit_icache_invalidate(void *start, void *end)
+{
+    // Clear/invalidate the instruction cache. Compiles to nothing on x86_64
+    // but required on ARM before running freshly written code.
+    // On Darwin it's the same as calling sys_icache_invalidate().
+#ifdef __GNUC__
+    __builtin___clear_cache(start, end);
+#elif defined(__aarch64__)
+#error No instruction cache clear available with this compiler on Aarch64!
+#endif
+}

yjit.c

@@ -69,60 +69,12 @@ STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM);
 // The "_yjit_" part is for trying to be informative. We might want different
 // suffixes for symbols meant for Rust and symbols meant for broader CRuby.
 
-bool
-rb_yjit_mark_writable(void *mem_block, uint32_t mem_size)
-{
-    return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0;
-}
-
-void
-rb_yjit_mark_executable(void *mem_block, uint32_t mem_size)
-{
-    // Do not call mprotect when mem_size is zero. Some platforms may return
-    // an error for it. https://github.com/Shopify/ruby/issues/450
-    if (mem_size == 0) {
-        return;
-    }
-    if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) {
-        rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s",
-            mem_block, (unsigned long)mem_size, strerror(errno));
-    }
-}
-
-// Free the specified memory block.
-bool
-rb_yjit_mark_unused(void *mem_block, uint32_t mem_size)
-{
-    // On Linux, you need to use madvise MADV_DONTNEED to free memory.
-    // We might not need to call this on macOS, but it's not really documented.
-    // We generally prefer to do the same thing on both to ease testing too.
-    madvise(mem_block, mem_size, MADV_DONTNEED);
-
-    // On macOS, mprotect PROT_NONE seems to reduce RSS.
-    // We also call this on Linux to avoid executing unused pages.
-    return mprotect(mem_block, mem_size, PROT_NONE) == 0;
-}
-
 long
 rb_yjit_array_len(VALUE a)
 {
     return rb_array_len(a);
 }
 
-// `start` is inclusive and `end` is exclusive.
-void
-rb_yjit_icache_invalidate(void *start, void *end)
-{
-    // Clear/invalidate the instruction cache. Compiles to nothing on x86_64
-    // but required on ARM before running freshly written code.
-    // On Darwin it's the same as calling sys_icache_invalidate().
-#ifdef __GNUC__
-    __builtin___clear_cache(start, end);
-#elif defined(__aarch64__)
-#error No instruction cache clear available with this compiler on Aarch64!
-#endif
-}
-
 # define PTR2NUM(x)   (rb_int2inum((intptr_t)(void *)(x)))
 
 // For a given raw_sample (frame), set the hash with the caller's
@@ -217,131 +169,6 @@ rb_yjit_exit_locations_dict(VALUE *yjit_raw_samples, int *yjit_line_samples, int
     return result;
 }
 
-uint32_t
-rb_yjit_get_page_size(void)
-{
-#if defined(_SC_PAGESIZE)
-    long page_size = sysconf(_SC_PAGESIZE);
-    if (page_size <= 0) rb_bug("yjit: failed to get page size");
-
-    // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected.
-    // Though our design sort of assume we have fine grained control over memory protection
-    // which require small page sizes.
-    if (page_size > 0x40000000l) rb_bug("yjit page size too large");
-
-    return (uint32_t)page_size;
-#else
-#error "YJIT supports POSIX only for now"
-#endif
-}
-
-#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
-// Align the current write position to a multiple of bytes
-static uint8_t *
-align_ptr(uint8_t *ptr, uint32_t multiple)
-{
-    // Compute the pointer modulo the given alignment boundary
-    uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
-
-    // If the pointer is already aligned, stop
-    if (rem == 0)
-        return ptr;
-
-    // Pad the pointer by the necessary amount to align it
-    uint32_t pad = multiple - rem;
-
-    return ptr + pad;
-}
-#endif
-
-// Address space reservation. Memory pages are mapped on an as needed basis.
-// See the Rust mm module for details.
-uint8_t *
-rb_yjit_reserve_addr_space(uint32_t mem_size)
-{
-#ifndef _WIN32
-    uint8_t *mem_block;
-
-    // On Linux
-    #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
-        uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE);
-        uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_yjit_reserve_addr_space;
-        uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX;
-        // Align the requested address to page size
-        uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);
-
-        // Probe for addresses close to this function using MAP_FIXED_NOREPLACE
-        // to improve odds of being in range for 32-bit relative call instructions.
-        do {
-            mem_block = mmap(
-                req_addr,
-                mem_size,
-                PROT_NONE,
-                MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
-                -1,
-                0
-            );
-
-            // If we succeeded, stop
-            if (mem_block != MAP_FAILED) {
-                ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_yjit_reserve_addr_space");
-                break;
-            }
-
-            // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux
-            // main_code_addr < heap_addr, and in case we are in a shared
-            // library mapped higher than the heap, downwards is still better
-            // since it's towards the end of the heap rather than the stack.)
-            req_addr -= 4 * 1024 * 1024;
-        } while (req_addr < probe_region_end);
-
-    // On MacOS and other platforms
-    #else
-        // Try to map a chunk of memory as executable
-        mem_block = mmap(
-            (void *)rb_yjit_reserve_addr_space,
-            mem_size,
-            PROT_NONE,
-            MAP_PRIVATE | MAP_ANONYMOUS,
-            -1,
-            0
-        );
-    #endif
-
-    // Fallback
-    if (mem_block == MAP_FAILED) {
-        // Try again without the address hint (e.g., valgrind)
-        mem_block = mmap(
-            NULL,
-            mem_size,
-            PROT_NONE,
-            MAP_PRIVATE | MAP_ANONYMOUS,
-            -1,
-            0
-        );
-
-        if (mem_block != MAP_FAILED) {
-            ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_yjit_reserve_addr_space:fallback");
-        }
-    }
-
-    // Check that the memory mapping was successful
-    if (mem_block == MAP_FAILED) {
-        perror("ruby: yjit: mmap:");
-        if(errno == ENOMEM) {
-            // No crash report if it's only insufficient memory
-            exit(EXIT_FAILURE);
-        }
-        rb_bug("mmap failed");
-    }
-
-    return mem_block;
-#else
-    // Windows not supported for now
-    return NULL;
-#endif
-}
-
 // Is anyone listening for :c_call and :c_return event currently?
 bool
 rb_c_method_tracing_currently_enabled(const rb_execution_context_t *ec)