[libclc] Move nextafter to the CLC library (#124097)

There were two implementations of this - one that implemented nextafter
in software, and another that called a clang builtin. No in-tree targets
called the builtin, so all targets build the software version. The
builtin version has been removed, and the software version has been
renamed to be the "default".

This commit also optimizes nextafter, to avoid scalarization as much as
possible. Note however that the (CLC) relational builtins still
scalarize; those will be optimized in a separate commit.

Since nextafter is used by some convert_type builtins, the diff to IR
codegen is not limited to the builtin itself.

Author: frasercrmck

libclc/amdgpu/lib/SOURCES

@@ -10,5 +10,4 @@ math/half_log2.cl
 math/half_recip.cl
 math/half_rsqrt.cl
 math/half_sqrt.cl
-math/nextafter.cl
 math/sqrt.cl

libclc/amdgpu/lib/math/nextafter.cl

@@ -159,6 +159,34 @@
   _CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, RET_TYPE, FUNCTION, ARG1_TYPE, \
                         ARG2_TYPE)
 
+// FIXME: Make _CLC_DEFINE_BINARY_BUILTIN avoid scalarization by default, and
+// introduce an explicit scalarizing version.
+#define _CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(RET_TYPE, FUNCTION, BUILTIN,   \
+                                                ARG1_TYPE, ARG2_TYPE)          \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {         \
+    return BUILTIN(x, y);                                                      \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x,                  \
+                                              ARG2_TYPE##2 y) {                \
+    return BUILTIN(x, y);                                                      \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x,                  \
+                                              ARG2_TYPE##3 y) {                \
+    return BUILTIN(x, y);                                                      \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x,                  \
+                                              ARG2_TYPE##4 y) {                \
+    return BUILTIN(x, y);                                                      \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x,                  \
+                                              ARG2_TYPE##8 y) {                \
+    return BUILTIN(x, y);                                                      \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE##16 FUNCTION(ARG1_TYPE##16 x,                \
+                                               ARG2_TYPE##16 y) {              \
+    return BUILTIN(x, y);                                                      \
+  }
+
 #define _CLC_DEFINE_BINARY_BUILTIN_WITH_SCALAR_SECOND_ARG(                     \
     RET_TYPE, FUNCTION, BUILTIN, ARG1_TYPE, ARG2_TYPE)                         \
   _CLC_DEFINE_BINARY_BUILTIN(RET_TYPE, FUNCTION, BUILTIN, ARG1_TYPE,           \

libclc/clc/include/clc/clcmacro.h

@@ -0,0 +1,4 @@
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
+                                                     __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
+                                                     __CLC_SCALAR_GENTYPE b);

libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc

@@ -0,0 +1,12 @@
+#ifndef __CLC_MATH_CLC_NEXTAFTER_H__
+#define __CLC_MATH_CLC_NEXTAFTER_H__
+
+#define __CLC_BODY <clc/shared/binary_decl.inc>
+#define __CLC_FUNCTION __clc_nextafter
+
+#include <clc/math/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_MATH_CLC_NEXTAFTER_H__

libclc/clc/include/clc/math/clc_nextafter.h

@@ -1,11 +1,6 @@
 #ifndef __CLC_RELATIONAL_CLC_ISNAN_H__
 #define __CLC_RELATIONAL_CLC_ISNAN_H__
 
-#if defined(CLC_CLSPV) || defined(CLC_SPIRV)
-// clspv and spir-v targets provide their own OpenCL-compatible isnan
-#define __clc_isnan isnan
-#else
-
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
@@ -37,6 +32,4 @@ _CLC_VECTOR_ISNAN_DECL(short, half)
 #undef _CLC_ISNAN_DECL
 #undef _CLC_VECTOR_ISNAN_DECL
 
-#endif
-
 #endif // __CLC_RELATIONAL_CLC_ISNAN_H__

libclc/clc/include/clc/relational/clc_isnan.h

@@ -0,0 +1,2 @@
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     __CLC_GENTYPE y);

libclc/clc/include/clc/shared/binary_decl.inc

@@ -2,6 +2,7 @@
 ../generic/math/clc_fabs.cl
 ../generic/math/clc_floor.cl
 ../generic/math/clc_mad.cl
+../generic/math/clc_nextafter.cl
 ../generic/math/clc_rint.cl
 ../generic/math/clc_trunc.cl
 ../generic/relational/clc_select.cl

libclc/clc/lib/clspv/SOURCES

@@ -8,6 +8,7 @@ math/clc_ceil.cl
 math/clc_fabs.cl
 math/clc_floor.cl
 math/clc_mad.cl
+math/clc_nextafter.cl
 math/clc_rint.cl
 math/clc_trunc.cl
 relational/clc_all.cl

libclc/clc/lib/generic/SOURCES

@@ -0,0 +1,62 @@
+#include <clc/clcmacro.h>
+#include <clc/internal/clc.h>
+#include <clc/relational/clc_isnan.h>
+
+// This file provides OpenCL C implementations of __clc_nextafter for
+// targets that don't support the clang builtin.
+
+#define CLC_AS_TYPE(x) __clc_as_##x
+
+#define NEXTAFTER(FLOAT_TYPE, UINT_TYPE, INT_TYPE, INT_TYPE_SCALAR)            \
+  _CLC_OVERLOAD _CLC_DEF FLOAT_TYPE __clc_nextafter(FLOAT_TYPE x,              \
+                                                    FLOAT_TYPE y) {            \
+    const UINT_TYPE sign_bit = (UINT_TYPE)1                                    \
+                               << (sizeof(INT_TYPE_SCALAR) * 8 - 1);           \
+    const UINT_TYPE sign_bit_mask = sign_bit - (UINT_TYPE)1;                   \
+    INT_TYPE ix = CLC_AS_TYPE(INT_TYPE)(x);                                    \
+    UINT_TYPE ax = CLC_AS_TYPE(UINT_TYPE)(ix) & sign_bit_mask;                 \
+    INT_TYPE mx = CLC_AS_TYPE(INT_TYPE)(sign_bit) - ix;                        \
+    mx = CLC_AS_TYPE(INT_TYPE)(ix) < (INT_TYPE)0 ? mx : ix;                    \
+    INT_TYPE iy = CLC_AS_TYPE(INT_TYPE)(y);                                    \
+    UINT_TYPE ay = CLC_AS_TYPE(UINT_TYPE)(iy) & sign_bit_mask;                 \
+    INT_TYPE my = CLC_AS_TYPE(INT_TYPE)(sign_bit) - iy;                        \
+    my = iy < (INT_TYPE)0 ? my : iy;                                           \
+    INT_TYPE t = mx + (mx < my ? (INT_TYPE)1 : (INT_TYPE)-1);                  \
+    INT_TYPE r = CLC_AS_TYPE(INT_TYPE)(sign_bit) - t;                          \
+    r = t < (INT_TYPE)0 ? r : t;                                               \
+    r = __clc_isnan(x) ? ix : r;                                               \
+    r = __clc_isnan(y) ? CLC_AS_TYPE(INT_TYPE)(iy) : r;                        \
+    r = ((ax | ay) == (UINT_TYPE)0 || ix == iy) ? iy : r;                      \
+    return CLC_AS_TYPE(FLOAT_TYPE)(r);                                         \
+  }
+
+NEXTAFTER(float, uint, int, int)
+NEXTAFTER(float2, uint2, int2, int)
+NEXTAFTER(float3, uint3, int3, int)
+NEXTAFTER(float4, uint4, int4, int)
+NEXTAFTER(float8, uint8, int8, int)
+NEXTAFTER(float16, uint16, int16, int)
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+
+NEXTAFTER(double, ulong, long, long)
+NEXTAFTER(double2, ulong2, long2, long)
+NEXTAFTER(double3, ulong3, long3, long)
+NEXTAFTER(double4, ulong4, long4, long)
+NEXTAFTER(double8, ulong8, long8, long)
+NEXTAFTER(double16, ulong16, long16, long)
+
+#endif
+
+#ifdef cl_khr_fp16
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+NEXTAFTER(half, ushort, short, short)
+NEXTAFTER(half2, ushort2, short2, short)
+NEXTAFTER(half3, ushort3, short3, short)
+NEXTAFTER(half4, ushort4, short4, short)
+NEXTAFTER(half8, ushort8, short8, short)
+NEXTAFTER(half16, ushort16, short16, short)
+
+#endif