Skip to content

Commit cf7d34a

Browse files
lntuelntue
authored
[libc] Extend fputil::sqrt to use floating point instructions for arm32. (#134499)
1 parent deef649 commit cf7d34a

File tree

6 files changed

+97
-33
lines changed

6 files changed

+97
-33
lines changed

‎libc/src/__support/FPUtil/aarch64/sqrt.h‎ renamed to ‎libc/src/__support/FPUtil/arm/sqrt.h‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,35 +6,38 @@
66
//
77
//===----------------------------------------------------------------------===//
88

9-
#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_AARCH64_SQRT_H
10-
#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_AARCH64_SQRT_H
9+
#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_ARM_SQRT_H
10+
#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_ARM_SQRT_H
1111

1212
#include "src/__support/common.h"
1313
#include "src/__support/macros/config.h"
1414
#include "src/__support/macros/properties/architectures.h"
15+
#include "src/__support/macros/properties/cpu_features.h"
1516

16-
#if !defined(LIBC_TARGET_ARCH_IS_AARCH64)
17+
#if !defined(LIBC_TARGET_ARCH_IS_ANY_ARM)
1718
#error "Invalid include"
1819
#endif
1920

20-
#include "src/__support/FPUtil/generic/sqrt.h"
21-
2221
namespace LIBC_NAMESPACE_DECL {
2322
namespace fputil {
2423

24+
#ifdef LIBC_TARGET_CPU_HAS_FPU_FLOAT
2525
template <> LIBC_INLINE float sqrt<float>(float x) {
2626
float y;
27-
__asm__ __volatile__("fsqrt %s0, %s1\n\t" : "=w"(y) : "w"(x));
27+
asm("fsqrt %s0, %s1\n\t" : "=w"(y) : "w"(x));
2828
return y;
2929
}
30+
#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT
3031

32+
#ifdef LIBC_TARGET_CPU_HAS_FPU_DOUBLE
3133
template <> LIBC_INLINE double sqrt<double>(double x) {
3234
double y;
33-
__asm__ __volatile__("fsqrt %d0, %d1\n\t" : "=w"(y) : "w"(x));
35+
asm("fsqrt %d0, %d1\n\t" : "=w"(y) : "w"(x));
3436
return y;
3537
}
38+
#endif // LIBC_TARGET_CPU_HAS_FPU_DOUBLE
3639

3740
} // namespace fputil
3841
} // namespace LIBC_NAMESPACE_DECL
3942

40-
#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_AARCH64_SQRT_H
43+
#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_ARM_SQRT_H

‎libc/src/__support/FPUtil/riscv/sqrt.h‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,31 +12,30 @@
1212
#include "src/__support/common.h"
1313
#include "src/__support/macros/config.h"
1414
#include "src/__support/macros/properties/architectures.h"
15+
#include "src/__support/macros/properties/cpu_features.h"
1516

1617
#if !defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
1718
#error "Invalid include"
1819
#endif
1920

20-
#include "src/__support/FPUtil/generic/sqrt.h"
21-
2221
namespace LIBC_NAMESPACE_DECL {
2322
namespace fputil {
2423

25-
#ifdef __riscv_flen
24+
#ifdef LIBC_TARGET_CPU_HAS_FPU_FLOAT
2625
template <> LIBC_INLINE float sqrt<float>(float x) {
2726
float result;
28-
__asm__ __volatile__("fsqrt.s %0, %1\n\t" : "=f"(result) : "f"(x));
27+
asm("fsqrt.s %0, %1\n\t" : "=f"(result) : "f"(x));
2928
return result;
3029
}
30+
#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT
3131

32-
#if __riscv_flen >= 64
32+
#if LIBC_TARGET_CPU_HAS_FPU_DOUBLE
3333
template <> LIBC_INLINE double sqrt<double>(double x) {
3434
double result;
35-
__asm__ __volatile__("fsqrt.d %0, %1\n\t" : "=f"(result) : "f"(x));
35+
asm("fsqrt.d %0, %1\n\t" : "=f"(result) : "f"(x));
3636
return result;
3737
}
38-
#endif // __riscv_flen >= 64
39-
#endif // __riscv_flen
38+
#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT
4039

4140
} // namespace fputil
4241
} // namespace LIBC_NAMESPACE_DECL

‎libc/src/__support/FPUtil/sqrt.h‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,44 @@
1212
#include "src/__support/macros/properties/architectures.h"
1313
#include "src/__support/macros/properties/cpu_features.h"
1414

15-
#if defined(LIBC_TARGET_ARCH_IS_X86_64) && defined(LIBC_TARGET_CPU_HAS_SSE2)
15+
#include "src/__support/FPUtil/generic/sqrt.h"
16+
17+
// Generic instruction specializations with __builtin_elementwise_sqrt.
18+
#if defined(LIBC_TARGET_CPU_HAS_FPU_FLOAT) || \
19+
defined(LIBC_TARGET_CPU_HAS_FPU_DOUBLE)
20+
21+
#if __has_builtin(__builtin_elementwise_sqrt)
22+
23+
namespace LIBC_NAMESPACE_DECL {
24+
namespace fputil {
25+
26+
#ifdef LIBC_TARGET_CPU_HAS_FPU_FLOAT
27+
template <> LIBC_INLINE float sqrt<float>(float x) {
28+
return __builtin_elementwise_sqrt(x);
29+
}
30+
#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT
31+
32+
#ifdef LIBC_TARGET_CPU_HAS_FPU_DOUBLE
33+
template <> LIBC_INLINE double sqrt<double>(double x) {
34+
return __builtin_elementwise_sqrt(x);
35+
}
36+
#endif // LIBC_TARGET_CPU_HAS_FPU_DOUBLE
37+
38+
} // namespace fputil
39+
} // namespace LIBC_NAMESPACE_DECL
40+
41+
#else // __builtin_elementwise_sqrt
42+
// Use inline assembly when __builtin_elementwise_sqrt is not available.
43+
#if defined(LIBC_TARGET_CPU_HAS_SSE2)
1644
#include "x86_64/sqrt.h"
17-
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
18-
#include "aarch64/sqrt.h"
45+
#elif defined(LIBC_TARGET_ARCH_IS_ANY_ARM)
46+
#include "arm/sqrt.h"
1947
#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
2048
#include "riscv/sqrt.h"
21-
#else
22-
#include "generic/sqrt.h"
49+
#endif // Target specific header of inline asm.
50+
51+
#endif // __builtin_elementwise_sqrt
52+
53+
#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT or DOUBLE
2354

24-
#endif
2555
#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_SQRT_H

‎libc/src/__support/FPUtil/x86_64/sqrt.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818
#error "sqrtss / sqrtsd need SSE2"
1919
#endif
2020

21-
#include "src/__support/FPUtil/generic/sqrt.h"
22-
2321
namespace LIBC_NAMESPACE_DECL {
2422
namespace fputil {
2523

‎libc/src/__support/macros/properties/cpu_features.h‎

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020

2121
#if defined(__SSE2__)
2222
#define LIBC_TARGET_CPU_HAS_SSE2
23+
#define LIBC_TARGET_CPU_HAS_FPU_FLOAT
24+
#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE
2325
#endif
2426

2527
#if defined(__SSE4_2__)
@@ -42,24 +44,55 @@
4244
#define LIBC_TARGET_CPU_HAS_AVX512BW
4345
#endif
4446

47+
#if defined(__ARM_FP)
48+
#if (__ARM_FP & 0x2)
49+
#define LIBC_TARGET_CPU_HAS_ARM_FPU_HALF
50+
#define LIBC_TARGET_CPU_HAS_FPU_HALF
51+
#endif // LIBC_TARGET_CPU_HAS_ARM_FPU_HALF
52+
#if (__ARM_FP & 0x4)
53+
#define LIBC_TARGET_CPU_HAS_ARM_FPU_FLOAT
54+
#define LIBC_TARGET_CPU_HAS_FPU_FLOAT
55+
#endif // LIBC_TARGET_CPU_HAS_ARM_FPU_FLOAT
56+
#if (__ARM_FP & 0x8)
57+
#define LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE
58+
#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE
59+
#endif // LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE
60+
#endif // __ARM_FP
61+
62+
#if defined(__riscv_flen)
63+
// https://github.com/riscv-non-isa/riscv-c-api-doc/blob/main/src/c-api.adoc
64+
#if (__riscv_flen & 0x10)
65+
#define LIBC_TARGET_CPU_HAS_RISCV_FPU_HALF
66+
#define LIBC_TARGET_CPU_HAS_FPU_HALF
67+
#endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_HALF
68+
#if (__riscv_flen & 0x20)
69+
#define LIBC_TARGET_CPU_HAS_RISCV_FPU_FLOAT
70+
#define LIBC_TARGET_CPU_HAS_FPU_FLOAT
71+
#endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_FLOAT
72+
#if (__riscv_flen & 0x40)
73+
#define LIBC_TARGET_CPU_HAS_RISCV_FPU_DOUBLE
74+
#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE
75+
#endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_DOUBLE
76+
#endif // __riscv_flen
77+
78+
#if defined(__NVPTX__) || defined(__AMDGPU__)
79+
#define LIBC_TARGET_CPU_HAS_FPU_FLOAT
80+
#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE
81+
#endif
82+
4583
#if defined(__ARM_FEATURE_FMA) || (defined(__AVX2__) && defined(__FMA__)) || \
4684
defined(__NVPTX__) || defined(__AMDGPU__) || defined(__LIBC_RISCV_USE_FMA)
4785
#define LIBC_TARGET_CPU_HAS_FMA
4886
// Provide a more fine-grained control of FMA instruction for ARM targets.
49-
#if defined(__ARM_FP)
50-
#if (__ARM_FP & 0x2)
87+
#if defined(LIBC_TARGET_CPU_HAS_FPU_HALF)
5188
#define LIBC_TARGET_CPU_HAS_FMA_HALF
5289
#endif // LIBC_TARGET_CPU_HAS_FMA_HALF
53-
#if (__ARM_FP & 0x4)
90+
#if defined(LIBC_TARGET_CPU_HAS_FPU_FLOAT)
5491
#define LIBC_TARGET_CPU_HAS_FMA_FLOAT
5592
#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
56-
#if (__ARM_FP & 0x8)
93+
#if defined(LIBC_TARGET_CPU_HAS_FPU_DOUBLE)
5794
#define LIBC_TARGET_CPU_HAS_FMA_DOUBLE
5895
#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
59-
#else
60-
#define LIBC_TARGET_CPU_HAS_FMA_FLOAT
61-
#define LIBC_TARGET_CPU_HAS_FMA_DOUBLE
62-
#endif
6396
#endif
6497

6598
#if defined(LIBC_TARGET_ARCH_IS_AARCH64) || \

‎utils/bazel/llvm-project-overlay/libc/BUILD.bazel‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1179,7 +1179,7 @@ sqrt_hdrs = selects.with_or({
11791179
"src/__support/FPUtil/x86_64/sqrt.h",
11801180
],
11811181
PLATFORM_CPU_ARM64: sqrt_common_hdrs + [
1182-
"src/__support/FPUtil/aarch64/sqrt.h",
1182+
"src/__support/FPUtil/arm/sqrt.h",
11831183
],
11841184
})
11851185

@@ -1195,6 +1195,7 @@ libc_support_library(
11951195
":__support_fputil_fenv_impl",
11961196
":__support_fputil_fp_bits",
11971197
":__support_fputil_rounding_mode",
1198+
":__support_macros_properties_cpu_features",
11981199
":__support_uint128",
11991200
],
12001201
)

0 commit comments

Comments
 (0)