[libclc] Move log1p/asinh/acosh/atanh to the CLC library (#132956)

These four functions all related in that they share tables and helper
functions. Furthermore, the acosh and atanh builtins call log1p.

As with other work in this area, these builtins are now vectorized. To
enable this, there are new table accessor functions which return a
vector of table values using a vector of indices. These are internally
scalarized, in the absence of gather operations. Some tables which were
tables of multiple entries (e.g., double2) are split into two separate
"low" and "high" tables. This might affect the performance of memory
operations but are hopefully mitigated by better codegen overall.

Author: frasercrmck

libclc/clc/include/clc/math/clc_acosh.h

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_MATH_CLC_ACOSH_H__
+#define __CLC_MATH_CLC_ACOSH_H__
+
+#define __CLC_BODY <clc/math/unary_decl.inc>
+#define __CLC_FUNCTION __clc_acosh
+
+#include <clc/math/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_MATH_CLC_ACOSH_H__

libclc/clc/include/clc/math/clc_asinh.h

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_MATH_CLC_ASINH_H__
+#define __CLC_MATH_CLC_ASINH_H__
+
+#define __CLC_BODY <clc/math/unary_decl.inc>
+#define __CLC_FUNCTION __clc_asinh
+
+#include <clc/math/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_MATH_CLC_ASINH_H__

libclc/clc/include/clc/math/clc_atanh.h

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_MATH_CLC_ATANH_H__
+#define __CLC_MATH_CLC_ATANH_H__
+
+#define __CLC_BODY <clc/math/unary_decl.inc>
+#define __CLC_FUNCTION __clc_atanh
+
+#include <clc/math/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_MATH_CLC_ATANH_H__

libclc/clc/include/clc/math/clc_ep_log.h

@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_MATH_CLC_EP_LOG_H__
+#define __CLC_MATH_CLC_EP_LOG_H__
+
+#define __CLC_BODY <clc/math/clc_ep_log.inc>
+#include <clc/math/gentype.inc>
+
+#endif // __CLC_MATH_CLC_EP_LOG_H__

libclc/clc/include/clc/math/clc_ep_log.inc

@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if __CLC_FPSIZE == 64
+
+_CLC_DECL _CLC_OVERLOAD void __clc_ep_log(__CLC_GENTYPE x,
+                                          private __CLC_INTN *xexp,
+                                          private __CLC_GENTYPE *r1,
+                                          private __CLC_GENTYPE *r2);
+
+#endif

libclc/clc/include/clc/math/clc_log1p.h

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_MATH_CLC_LOG1P_H__
+#define __CLC_MATH_CLC_LOG1P_H__
+
+#define __CLC_BODY <clc/math/unary_decl.inc>
+#define __CLC_FUNCTION __clc_log1p
+
+#include <clc/math/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_MATH_CLC_LOG1P_H__

libclc/clc/include/clc/math/tables.h

@@ -20,12 +20,45 @@
 #define TABLE_FUNCTION(TYPE, TABLE, NAME)                                      \
   TYPE TABLE_MANGLE(NAME)(size_t idx) { return TABLE[idx]; }
 
+#define CLC_TABLE_FUNCTION(TYPE, TABLE, NAME)                                  \
+  _CLC_DEF _CLC_OVERLOAD TYPE TABLE_MANGLE(NAME)(int idx) {                    \
+    return TABLE[idx];                                                         \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD TYPE##2 TABLE_MANGLE(NAME)(int##2 idx) {              \
+    return (TYPE##2){TABLE[idx.s0], TABLE[idx.s1]};                            \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD TYPE##3 TABLE_MANGLE(NAME)(int##3 idx) {              \
+    return (TYPE##3){TABLE[idx.s0], TABLE[idx.s1], TABLE[idx.s2]};             \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD TYPE##4 TABLE_MANGLE(NAME)(int##4 idx) {              \
+    return (TYPE##4){TABLE[idx.s0], TABLE[idx.s1], TABLE[idx.s2],              \
+                     TABLE[idx.s3]};                                           \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD TYPE##8 TABLE_MANGLE(NAME)(int##8 idx) {              \
+    return (TYPE##8){TABLE[idx.s0], TABLE[idx.s1], TABLE[idx.s2],              \
+                     TABLE[idx.s3], TABLE[idx.s4], TABLE[idx.s5],              \
+                     TABLE[idx.s6], TABLE[idx.s7]};                            \
+  }                                                                            \
+  _CLC_DEF _CLC_OVERLOAD TYPE##16 TABLE_MANGLE(NAME)(int##16 idx) {            \
+    return (TYPE##16){                                                         \
+        TABLE[idx.s0], TABLE[idx.s1], TABLE[idx.s2], TABLE[idx.s3],            \
+        TABLE[idx.s4], TABLE[idx.s5], TABLE[idx.s6], TABLE[idx.s7],            \
+        TABLE[idx.s8], TABLE[idx.s9], TABLE[idx.sA], TABLE[idx.sB],            \
+        TABLE[idx.sC], TABLE[idx.sD], TABLE[idx.sE], TABLE[idx.sF]};           \
+  }
+
 #define TABLE_FUNCTION_DECL(TYPE, NAME) TYPE TABLE_MANGLE(NAME)(size_t idx);
 
+#define CLC_TABLE_FUNCTION_DECL(TYPE, NAME)                                    \
+  _CLC_DECL _CLC_OVERLOAD TYPE TABLE_MANGLE(NAME)(int idx);                    \
+  _CLC_DECL _CLC_OVERLOAD TYPE##2 TABLE_MANGLE(NAME)(int##2 idx);              \
+  _CLC_DECL _CLC_OVERLOAD TYPE##3 TABLE_MANGLE(NAME)(int##3 idx);              \
+  _CLC_DECL _CLC_OVERLOAD TYPE##4 TABLE_MANGLE(NAME)(int##4 idx);              \
+  _CLC_DECL _CLC_OVERLOAD TYPE##8 TABLE_MANGLE(NAME)(int##8 idx);              \
+  _CLC_DECL _CLC_OVERLOAD TYPE##16 TABLE_MANGLE(NAME)(int##16 idx);
+
 #define USE_TABLE(NAME, IDX) TABLE_MANGLE(NAME)(IDX)
 
-TABLE_FUNCTION_DECL(float2, loge_tbl);
-TABLE_FUNCTION_DECL(float, log_inv_tbl);
 TABLE_FUNCTION_DECL(float2, log_inv_tbl_ep);
 TABLE_FUNCTION_DECL(float2, log2_tbl);
 TABLE_FUNCTION_DECL(float2, log10_tbl);
@@ -35,11 +68,17 @@ TABLE_FUNCTION_DECL(float2, cbrt_tbl);
 TABLE_FUNCTION_DECL(float, exp_tbl);
 TABLE_FUNCTION_DECL(float2, exp_tbl_ep);
 
+CLC_TABLE_FUNCTION_DECL(float, loge_tbl_lo);
+CLC_TABLE_FUNCTION_DECL(float, loge_tbl_hi);
+CLC_TABLE_FUNCTION_DECL(float, log_inv_tbl);
+
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
 TABLE_FUNCTION_DECL(double2, ln_tbl);
+CLC_TABLE_FUNCTION_DECL(double, ln_tbl_lo);
+CLC_TABLE_FUNCTION_DECL(double, ln_tbl_hi);
 TABLE_FUNCTION_DECL(double2, atan_jby256_tbl);
 TABLE_FUNCTION_DECL(double2, two_to_jby64_ep_tbl);
 TABLE_FUNCTION_DECL(double2, sinh_tbl);

libclc/clc/lib/generic/SOURCES

@@ -18,13 +18,17 @@ integer/clc_rotate.cl
 integer/clc_sub_sat.cl
 integer/clc_upsample.cl
 math/clc_acos.cl
+math/clc_acosh.cl
 math/clc_acospi.cl
 math/clc_asin.cl
+math/clc_asinh.cl
 math/clc_asinpi.cl
 math/clc_atan.cl
+math/clc_atanh.cl
 math/clc_atanpi.cl
 math/clc_ceil.cl
 math/clc_copysign.cl
+math/clc_ep_log.cl
 math/clc_fabs.cl
 math/clc_fma.cl
 math/clc_floor.cl
@@ -33,6 +37,7 @@ math/clc_hypot.cl
 math/clc_ldexp.cl
 math/clc_log.cl
 math/clc_log10.cl
+math/clc_log1p.cl
 math/clc_log2.cl
 math/clc_mad.cl
 math/clc_modf.cl
@@ -44,6 +49,7 @@ math/clc_rsqrt.cl
 math/clc_sincos_helpers.cl
 math/clc_sqrt.cl
 math/clc_sw_fma.cl
+math/clc_tables.cl
 math/clc_trunc.cl
 relational/clc_all.cl
 relational/clc_any.cl

libclc/clc/lib/generic/math/clc_acosh.cl

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/clc_convert.h>
+#include <clc/float/definitions.h>
+#include <clc/internal/clc.h>
+#include <clc/math/clc_ep_log.h>
+#include <clc/math/clc_fabs.h>
+#include <clc/math/clc_fma.h>
+#include <clc/math/clc_log1p.h>
+#include <clc/math/clc_mad.h>
+#include <clc/math/clc_sqrt.h>
+#include <clc/math/math.h>
+#include <clc/relational/clc_isinf.h>
+#include <clc/relational/clc_isnan.h>
+#include <clc/relational/clc_select.h>
+
+#define __CLC_BODY <clc_acosh.inc>
+#include <clc/math/gentype.inc>

libclc/clc/lib/generic/math/clc_acosh.inc

@@ -0,0 +1,111 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if __CLC_FPSIZE == 32
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_acosh(__CLC_GENTYPE x) {
+  __CLC_UINTN ux = __CLC_AS_UINTN(x);
+
+  // Arguments greater than 1/sqrt(epsilon) in magnitude are approximated by
+  // acosh(x) = ln(2) + ln(x)
+  // For 2.0 <= x <= 1/sqrt(epsilon) the approximation is:
+  //   acosh(x) = ln(x + sqrt(x * x - 1))
+  __CLC_INTN high = ux > 0x46000000U;
+  __CLC_INTN med = ux > 0x40000000U;
+
+  __CLC_GENTYPE w = x - 1.0f;
+  __CLC_GENTYPE s = w * w + 2.0f * w;
+  __CLC_GENTYPE t = x * x - 1.0f;
+  __CLC_GENTYPE r = __clc_sqrt(med ? t : s) + (med ? x : w);
+  __CLC_GENTYPE v = (high ? x : r) - (med ? 1.0f : 0.0f);
+  __CLC_GENTYPE z = __clc_log1p(v) + (high ? 0x1.62e430p-1f : 0.0f);
+
+  z = __clc_select(z, x, __clc_isnan(x) || __clc_isinf(x));
+  z = x < 1.0f ? __CLC_GENTYPE_NAN : z;
+
+  return z;
+}
+
+#elif __CLC_FPSIZE == 64
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_acosh(__CLC_GENTYPE x) {
+  // 1/sqrt(eps) = 9.49062656242515593767e+07
+  const __CLC_GENTYPE recrteps = 0x1.6a09e667f3bcdp+26;
+  // log2_lead and log2_tail sum to an extra-precise version of log(2)
+  const __CLC_GENTYPE log2_lead = 0x1.62e42ep-1;
+  const __CLC_GENTYPE log2_tail = 0x1.efa39ef35793cp-25;
+
+  // Handle x >= 128 here
+  __CLC_LONGN xlarge = x > recrteps;
+  __CLC_GENTYPE r = x + __clc_sqrt(__clc_fma(x, x, -1.0));
+  r = xlarge ? x : r;
+
+  __CLC_INTN xexp;
+  __CLC_GENTYPE r1, r2;
+  __clc_ep_log(r, &xexp, &r1, &r2);
+
+  __CLC_GENTYPE dxexp = __CLC_CONVERT_GENTYPE(
+      __CLC_CONVERT_LONGN(xexp) + (xlarge ? (__CLC_LONGN)1 : (__CLC_LONGN)0));
+  r1 = __clc_fma(dxexp, log2_lead, r1);
+  r2 = __clc_fma(dxexp, log2_tail, r2);
+
+  __CLC_GENTYPE ret1 = r1 + r2;
+
+  // Handle 1 < x < 128 here
+  // We compute the value
+  // t = x - 1.0 + sqrt(2.0*(x - 1.0) + (x - 1.0)*(x - 1.0))
+  // using simulated quad precision.
+  __CLC_GENTYPE t = x - 1.0;
+  __CLC_GENTYPE u1 = t * 2.0;
+
+  // (t,0) * (t,0) -> (v1, v2)
+  __CLC_GENTYPE v1 = t * t;
+  __CLC_GENTYPE v2 = __clc_fma(t, t, -v1);
+
+  // (u1,0) + (v1,v2) -> (w1,w2)
+  r = u1 + v1;
+  __CLC_GENTYPE s = (((u1 - r) + v1) + v2);
+  __CLC_GENTYPE w1 = r + s;
+  __CLC_GENTYPE w2 = (r - w1) + s;
+
+  // sqrt(w1,w2) -> (u1,u2)
+  __CLC_GENTYPE p1 = __clc_sqrt(w1);
+  __CLC_GENTYPE a1 = p1 * p1;
+  __CLC_GENTYPE a2 = __clc_fma(p1, p1, -a1);
+  __CLC_GENTYPE temp = (((w1 - a1) - a2) + w2);
+  __CLC_GENTYPE p2 = MATH_DIVIDE(temp * 0.5, p1);
+  u1 = p1 + p2;
+  __CLC_GENTYPE u2 = (p1 - u1) + p2;
+
+  // (u1,u2) + (t,0) -> (r1,r2)
+  r = u1 + t;
+  s = ((u1 - r) + t) + u2;
+  // r1 = r + s;
+  // r2 = (r - r1) + s;
+  // t = r1 + r2;
+  t = r + s;
+
+  // For arguments 1.13 <= x <= 1.5 the log1p function is good enough
+  __CLC_GENTYPE ret2 = __clc_log1p(t);
+
+  __CLC_GENTYPE ret = x >= 128.0 ? ret1 : ret2;
+
+  ret = (__clc_isinf(x) || __clc_isnan(x)) ? x : ret;
+  ret = x == 1.0 ? 0.0 : ret;
+  ret = x < 1.0 ? __CLC_GENTYPE_NAN : ret;
+
+  return ret;
+}
+
+#elif __CLC_FPSIZE == 16
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_acosh(__CLC_GENTYPE x) {
+  return __CLC_CONVERT_GENTYPE(__clc_acosh(__CLC_CONVERT_FLOATN(x)));
+}
+
+#endif