[device-libs] Replace __memcpy_internal_aligned with __builtin_memcpy

Microbenchmarks on gfx1030 showed that __builtin_memcpy usually performs
similarly to or better than __memcpy_internal_aligned for
compile-time-constant copy sizes as they occur at the
__memcpy_internal_aligned call sites once the pipe read/write functions
are inlined.

SWDEV-150027

Author: ritter-x2a

amd/device-libs/opencl/src/pipes/memcpyia.cl

@@ -22,6 +22,11 @@ F(32,ulong4) \
 F(64,ulong8) \
 F(128,ulong16)
 
+// Assume that ptr is aligned by at least align bytes. In contrast to
+// __builtin_assume_aligned, this allows a non-constant alignment operand.
+#define ASSUME_ALIGNED(ptr, align) \
+    __builtin_assume(__builtin_is_aligned(ptr, align))
+
 struct pipeimp {
     atomic_size_t read_idx;
     atomic_size_t write_idx;
@@ -30,8 +35,6 @@ struct pipeimp {
     uchar packets[1];
 };
 
-extern void __memcpy_internal_aligned(void *, const void *, size_t, size_t);
-
 static __attribute__((always_inline)) size_t
 reserve(volatile __global atomic_size_t *pi, size_t lim, size_t n)
 {

amd/device-libs/opencl/src/pipes/pipes.h

@@ -40,7 +40,10 @@ __read_pipe_2(__global struct pipeimp* p, void* ptr, uint size, uint align)
         return -1;
 
     size_t pi = wrap(ri, p->end_idx);
-    __memcpy_internal_aligned(ptr, p->packets + pi*size, size, align);
+    void *pipe_ptr = p->packets + pi * size;
+    ASSUME_ALIGNED(ptr, align);
+    ASSUME_ALIGNED(pipe_ptr, align);
+    __builtin_memcpy(ptr, pipe_ptr, size);
 
     if (ri == wi-1) {
         __opencl_atomic_store(&p->write_idx, 0, memory_order_relaxed, memory_scope_device);
@@ -68,7 +71,10 @@ __read_pipe_4(__global struct pipeimp* p, reserve_id_t rid, uint i, void *ptr, u
 {
     size_t rin = __builtin_astype(rid, size_t) + i; \
     size_t pi = wrap(rin, p->end_idx);
-    __memcpy_internal_aligned(ptr, p->packets + pi*size, size, align);
+    void *pipe_ptr = p->packets + pi * size;
+    ASSUME_ALIGNED(ptr, align);
+    ASSUME_ALIGNED(pipe_ptr, align);
+    __builtin_memcpy(ptr, pipe_ptr, size);
 
     return 0;
 }

amd/device-libs/opencl/src/pipes/readp.cl

@@ -36,7 +36,10 @@ __write_pipe_2(__global struct pipeimp* p, const void* ptr, uint size, uint alig
         return -1;
 
     size_t pi = wrap(wi, ei);
-    __memcpy_internal_aligned(p->packets + pi*size, ptr, size, align);
+    void *pipe_ptr = p->packets + pi * size;
+    ASSUME_ALIGNED(pipe_ptr, align);
+    ASSUME_ALIGNED(ptr, align);
+    __builtin_memcpy(pipe_ptr, ptr, size);
 
     return 0;
 }
@@ -58,7 +61,10 @@ __write_pipe_4(__global struct pipeimp* p, reserve_id_t rid, uint i, const void
 {
     size_t rin = __builtin_astype(rid, size_t) + i; \
     size_t pi = wrap(rin, p->end_idx);
-    __memcpy_internal_aligned(p->packets + pi*size, ptr, size, align);
+    void *pipe_ptr = p->packets + pi * size;
+    ASSUME_ALIGNED(pipe_ptr, align);
+    ASSUME_ALIGNED(ptr, align);
+    __builtin_memcpy(pipe_ptr, ptr, size);
 
     return 0;
 }