Skip to content

Commit 2e5f7d2

Browse files
saturn691saturn691
committed
[libc] Add support for string/memory_utils functions for AArch64 without HW FP/SIMD
Add conditional compilation to add support for AArch64 without vector registers and/or hardware FPUs by using the generic implementation
1 parent bc2a5b5 commit 2e5f7d2

File tree

11 files changed

+93
-53
lines changed

11 files changed

+93
-53
lines changed

‎libc/src/__support/FPUtil/FEnvImpl.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
#include "src/__support/macros/properties/architectures.h"
1818
#include "src/errno/libc_errno.h"
1919

20-
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
20+
#if defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_FP)
2121
#if defined(__APPLE__)
2222
#include "aarch64/fenv_darwin_impl.h"
2323
#else

‎libc/src/__support/FPUtil/nearest_integer.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
#if (defined(LIBC_TARGET_ARCH_IS_X86_64) && defined(LIBC_TARGET_CPU_HAS_SSE4_2))
1818
#include "x86_64/nearest_integer.h"
19-
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
19+
#elif (defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_FP))
2020
#include "aarch64/nearest_integer.h"
2121
#elif defined(LIBC_TARGET_ARCH_IS_GPU)
2222

‎libc/src/__support/FPUtil/sqrt.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ template <> LIBC_INLINE double sqrt<double>(double x) {
4242
// Use inline assembly when __builtin_elementwise_sqrt is not available.
4343
#if defined(LIBC_TARGET_CPU_HAS_SSE2)
4444
#include "x86_64/sqrt.h"
45-
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
45+
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_FP)
4646
#include "aarch64/sqrt.h"
4747
#elif defined(LIBC_TARGET_ARCH_IS_ARM)
4848
#include "arm/sqrt.h"

‎libc/src/string/memory_utils/aarch64/inline_bcmp.h‎

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,35 @@
1919

2020
namespace LIBC_NAMESPACE_DECL {
2121

22-
[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_aarch64(CPtr p1,
23-
CPtr p2,
24-
size_t count) {
25-
if (LIBC_LIKELY(count <= 32)) {
26-
if (LIBC_UNLIKELY(count >= 16)) {
27-
return aarch64::Bcmp<16>::head_tail(p1, p2, count);
28-
}
22+
[[maybe_unused]] LIBC_INLINE BcmpReturnType
23+
inline_bcmp_aarch64_no_fp(CPtr p1, CPtr p2, size_t count) {
24+
return generic::Bcmp<uint64_t>::loop_and_tail_align_above(256, p1, p2, count);
25+
}
26+
27+
#ifdef __ARM_NEON
28+
[[maybe_unused]] LIBC_INLINE BcmpReturnType
29+
inline_bcmp_aarch64_with_fp(CPtr p1, CPtr p2, size_t count) {
30+
if (count <= 32) {
31+
return aarch64::Bcmp<16>::head_tail(p1, p2, count);
32+
}
33+
34+
if (count <= 64) {
35+
return aarch64::Bcmp<32>::head_tail(p1, p2, count);
36+
}
37+
38+
if (LIBC_UNLIKELY(count > 256)) {
39+
if (auto value = aarch64::Bcmp<32>::block(p1, p2))
40+
return value;
41+
align_to_next_boundary<16, Arg::P1>(p1, p2, count);
42+
}
43+
44+
return aarch64::Bcmp<32>::loop_and_tail(p1, p2, count);
45+
}
46+
#endif
47+
48+
[[gnu::flatten]] LIBC_INLINE BcmpReturnType
49+
inline_bcmp_aarch64_dispatch(CPtr p1, CPtr p2, size_t count) {
50+
if (LIBC_LIKELY(count <= 16)) {
2951
switch (count) {
3052
case 0:
3153
return BcmpReturnType::zero();
@@ -54,16 +76,11 @@ namespace LIBC_NAMESPACE_DECL {
5476
}
5577
}
5678

57-
if (count <= 64)
58-
return aarch64::Bcmp<32>::head_tail(p1, p2, count);
59-
60-
// Aligned loop if > 256, otherwise normal loop
61-
if (LIBC_UNLIKELY(count > 256)) {
62-
if (auto value = aarch64::Bcmp<32>::block(p1, p2))
63-
return value;
64-
align_to_next_boundary<16, Arg::P1>(p1, p2, count);
65-
}
66-
return aarch64::Bcmp<32>::loop_and_tail(p1, p2, count);
79+
#if defined(__ARM_NEON)
80+
return inline_bcmp_aarch64_with_fp(p1, p2, count);
81+
#else
82+
return inline_bcmp_aarch64_no_fp(p1, p2, count);
83+
#endif
6784
}
6885

6986
} // namespace LIBC_NAMESPACE_DECL

‎libc/src/string/memory_utils/aarch64/inline_memcmp.h‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,11 @@ namespace LIBC_NAMESPACE_DECL {
1818

1919
[[maybe_unused]] LIBC_INLINE MemcmpReturnType
2020
inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
21-
if (LIBC_UNLIKELY(count >= 384)) {
22-
if (auto value = generic::Memcmp<uint8x16_t>::block(p1, p2))
23-
return value;
24-
align_to_next_boundary<16, Arg::P1>(p1, p2, count);
25-
}
26-
return generic::Memcmp<uint8x16_t>::loop_and_tail(p1, p2, count);
21+
return generic::Memcmp<uint64_t>::loop_and_tail_align_above(384, p1, p2,
22+
count);
2723
}
2824

25+
#if defined(__ARM_NEON)
2926
[[maybe_unused]] LIBC_INLINE MemcmpReturnType
3027
inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) {
3128
if (LIBC_UNLIKELY(count >= 128)) { // [128, ∞]
@@ -46,9 +43,10 @@ inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) {
4643
return generic::Memcmp<uint8x16_t>::loop_and_tail(p1 + 32, p2 + 32,
4744
count - 32);
4845
}
46+
#endif
4947

50-
LIBC_INLINE MemcmpReturnType inline_memcmp_aarch64(CPtr p1, CPtr p2,
51-
size_t count) {
48+
[[gnu::flatten]] LIBC_INLINE MemcmpReturnType
49+
inline_memcmp_aarch64_dispatch(CPtr p1, CPtr p2, size_t count) {
5250
if (count == 0)
5351
return MemcmpReturnType::zero();
5452
if (count == 1)
@@ -61,10 +59,12 @@ LIBC_INLINE MemcmpReturnType inline_memcmp_aarch64(CPtr p1, CPtr p2,
6159
return generic::Memcmp<uint32_t>::head_tail(p1, p2, count);
6260
if (count <= 16)
6361
return generic::Memcmp<uint64_t>::head_tail(p1, p2, count);
64-
if constexpr (aarch64::kNeon)
65-
return inline_memcmp_aarch64_neon_gt16(p1, p2, count);
66-
else
67-
return inline_memcmp_generic_gt16(p1, p2, count);
62+
63+
#if defined(__ARM_NEON)
64+
return inline_memcmp_aarch64_neon_gt16(p1, p2, count);
65+
#else
66+
return inline_memcmp_generic_gt16(p1, p2, count);
67+
#endif
6868
}
6969
} // namespace LIBC_NAMESPACE_DECL
7070

‎libc/src/string/memory_utils/aarch64/inline_memmove.h‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@
88
#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMMOVE_H
99
#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMMOVE_H
1010

11-
#include "src/__support/macros/attributes.h" // LIBC_INLINE
12-
#include "src/string/memory_utils/op_aarch64.h" // aarch64::kNeon
11+
#include "src/__support/macros/attributes.h" // LIBC_INLINE
1312
#include "src/string/memory_utils/op_builtin.h"
1413
#include "src/string/memory_utils/op_generic.h"
1514
#include "src/string/memory_utils/utils.h"
@@ -19,7 +18,6 @@
1918
namespace LIBC_NAMESPACE_DECL {
2019

2120
LIBC_INLINE void inline_memmove_aarch64(Ptr dst, CPtr src, size_t count) {
22-
static_assert(aarch64::kNeon, "aarch64 supports vector types");
2321
using uint128_t = generic_v128;
2422
using uint256_t = generic_v256;
2523
using uint512_t = generic_v512;

‎libc/src/string/memory_utils/aarch64/inline_memset.h‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,34 @@
1818

1919
namespace LIBC_NAMESPACE_DECL {
2020

21+
using uint128_t = generic_v128;
22+
using uint256_t = generic_v256;
23+
using uint512_t = generic_v512;
24+
2125
[[maybe_unused]] LIBC_INLINE static void
22-
inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
23-
static_assert(aarch64::kNeon, "aarch64 supports vector types");
24-
using uint128_t = generic_v128;
25-
using uint256_t = generic_v256;
26-
using uint512_t = generic_v512;
26+
inline_memset_aarch64_no_fp(Ptr dst, uint8_t value, size_t count) {
27+
generic::Memset<uint128_t>::block(dst, value);
28+
align_to_next_boundary<16>(dst, count);
29+
return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);
30+
}
31+
32+
#if defined(__ARM_NEON)
33+
[[maybe_unused]] LIBC_INLINE static void
34+
inline_memset_aarch64_with_fp(Ptr dst, uint8_t value, size_t count) {
35+
if (count >= 448 && value == 0 && aarch64::neon::hasZva()) {
36+
generic::Memset<uint512_t>::block(dst, 0);
37+
align_to_next_boundary<64>(dst, count);
38+
return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count);
39+
}
40+
41+
generic::Memset<uint128_t>::block(dst, value);
42+
align_to_next_boundary<16>(dst, count);
43+
return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);
44+
}
45+
#endif
46+
47+
[[gnu::flatten]] [[maybe_unused]] LIBC_INLINE static void
48+
inline_memset_aarch64_dispatch(Ptr dst, uint8_t value, size_t count) {
2749
if (count == 0)
2850
return;
2951
if (count <= 3) {
@@ -46,15 +68,12 @@ inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
4668
generic::Memset<uint256_t>::tail(dst, value, count);
4769
return;
4870
}
49-
if (count >= 448 && value == 0 && aarch64::neon::hasZva()) {
50-
generic::Memset<uint512_t>::block(dst, 0);
51-
align_to_next_boundary<64>(dst, count);
52-
return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count);
53-
} else {
54-
generic::Memset<uint128_t>::block(dst, value);
55-
align_to_next_boundary<16>(dst, count);
56-
return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);
57-
}
71+
72+
#if defined(__ARM_NEON)
73+
return inline_memset_aarch64_with_fp(dst, value, count);
74+
#else
75+
return inline_memset_aarch64_no_fp(dst, value, count);
76+
#endif
5877
}
5978

6079
} // namespace LIBC_NAMESPACE_DECL

‎libc/src/string/memory_utils/inline_bcmp.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
#define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_x86
2222
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
2323
#include "src/string/memory_utils/aarch64/inline_bcmp.h"
24-
#define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_aarch64
24+
#define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_aarch64_dispatch
2525
#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
2626
#include "src/string/memory_utils/riscv/inline_bcmp.h"
2727
#define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_riscv

‎libc/src/string/memory_utils/inline_memcmp.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_x86
2121
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
2222
#include "src/string/memory_utils/aarch64/inline_memcmp.h"
23-
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_aarch64
23+
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_aarch64_dispatch
2424
#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
2525
#include "src/string/memory_utils/riscv/inline_memcmp.h"
2626
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_riscv

‎libc/src/string/memory_utils/inline_memset.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_x86
2121
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
2222
#include "src/string/memory_utils/aarch64/inline_memset.h"
23-
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_aarch64
23+
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_aarch64_dispatch
2424
#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
2525
#include "src/string/memory_utils/riscv/inline_memset.h"
2626
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_riscv

0 commit comments

Comments
 (0)