Reapply "[libc] Enable wide-read memory operations by default on Linux (#154602)" #154640
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-libc Author: Joseph Huber (jhuber6) ChangesThis reverts commit 27fc967. Full diff: https://github.com/llvm/llvm-project/pull/154640.diff 5 Files Affected:
diff --git a/libc/config/linux/arm/config.json b/libc/config/linux/arm/config.json
new file mode 100644
index 0000000000000..e7ad4544b104d
--- /dev/null
+++ b/libc/config/linux/arm/config.json
@@ -0,0 +1,7 @@
+{
+ "string": {
+ "LIBC_CONF_STRING_UNSAFE_WIDE_READ": {
+ "value": false
+ }
+ }
+}
diff --git a/libc/config/linux/config.json b/libc/config/linux/config.json
new file mode 100644
index 0000000000000..30e8b2cdadabe
--- /dev/null
+++ b/libc/config/linux/config.json
@@ -0,0 +1,7 @@
+{
+ "string": {
+ "LIBC_CONF_STRING_UNSAFE_WIDE_READ": {
+ "value": true
+ }
+ }
+}
diff --git a/libc/src/string/memory_utils/aarch64/inline_strlen.h b/libc/src/string/memory_utils/aarch64/inline_strlen.h
index 79487f4752b83..fe2bc8dd6da95 100644
--- a/libc/src/string/memory_utils/aarch64/inline_strlen.h
+++ b/libc/src/string/memory_utils/aarch64/inline_strlen.h
@@ -17,14 +17,16 @@
namespace LIBC_NAMESPACE_DECL {
namespace neon {
-[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {
+[[maybe_unused]] [[clang::no_sanitize("address")]] LIBC_INLINE static size_t
+string_length(const char *src) {
using Vector __attribute__((may_alias)) = uint8x8_t;
uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
- Vector *block_ptr = reinterpret_cast<Vector *>(src - misalign_bytes);
+ const Vector *block_ptr =
+ reinterpret_cast<const Vector *>(src - misalign_bytes);
Vector v = *block_ptr;
Vector vcmp = vceqz_u8(v);
- uint64x1_t cmp_mask = vreinterpret_u64_s8(vcmp);
+ uint64x1_t cmp_mask = vreinterpret_u64_u8(vcmp);
uint64_t cmp = vget_lane_u64(cmp_mask, 0);
cmp = cmp >> (misalign_bytes << 3);
if (cmp)
@@ -34,7 +36,7 @@ namespace neon {
++block_ptr;
v = *block_ptr;
vcmp = vceqz_u8(v);
- cmp_mask = vreinterpret_u64_s8(vcmp);
+ cmp_mask = vreinterpret_u64_u8(vcmp);
cmp = vget_lane_u64(cmp_mask, 0);
if (cmp)
return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
diff --git a/libc/src/string/memory_utils/x86_64/inline_strlen.h b/libc/src/string/memory_utils/x86_64/inline_strlen.h
index 5eb184cbf8107..7f74db319cc42 100644
--- a/libc/src/string/memory_utils/x86_64/inline_strlen.h
+++ b/libc/src/string/memory_utils/x86_64/inline_strlen.h
@@ -18,22 +18,23 @@ namespace LIBC_NAMESPACE_DECL {
namespace string_length_internal {
// Return a bit-mask with the nth bit set if the nth-byte in block_ptr is zero.
template <typename Vector, typename Mask>
-Mask CompareAndMask(const Vector *block_ptr);
+LIBC_INLINE static Mask compare_and_mask(const Vector *block_ptr);
template <typename Vector, typename Mask,
- decltype(CompareAndMask<Vector, Mask>)>
-size_t string_length_vector(const char *src) {
+ decltype(compare_and_mask<Vector, Mask>)>
+LIBC_INLINE static [[clang::no_sanitize("address")]] size_t
+string_length_vector(const char *src) {
uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
const Vector *block_ptr =
reinterpret_cast<const Vector *>(src - misalign_bytes);
- auto cmp = CompareAndMask<Vector, Mask>(block_ptr) >> misalign_bytes;
+ auto cmp = compare_and_mask<Vector, Mask>(block_ptr) >> misalign_bytes;
if (cmp)
return cpp::countr_zero(cmp);
while (true) {
block_ptr++;
- cmp = CompareAndMask<Vector, Mask>(block_ptr);
+ cmp = compare_and_mask<Vector, Mask>(block_ptr);
if (cmp)
return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
reinterpret_cast<uintptr_t>(src) +
@@ -42,7 +43,8 @@ size_t string_length_vector(const char *src) {
}
template <>
-uint32_t CompareAndMask<__m128i, uint32_t>(const __m128i *block_ptr) {
+LIBC_INLINE uint32_t
+compare_and_mask<__m128i, uint32_t>(const __m128i *block_ptr) {
__m128i v = _mm_load_si128(block_ptr);
__m128i z = _mm_setzero_si128();
__m128i c = _mm_cmpeq_epi8(z, v);
@@ -52,13 +54,14 @@ uint32_t CompareAndMask<__m128i, uint32_t>(const __m128i *block_ptr) {
namespace sse2 {
[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {
return string_length_vector<__m128i, uint32_t,
- CompareAndMask<__m128i, uint32_t>>(src);
+ compare_and_mask<__m128i, uint32_t>>(src);
}
} // namespace sse2
#if defined(__AVX2__)
template <>
-uint32_t CompareAndMask<__m256i, uint32_t>(const __m256i *block_ptr) {
+LIBC_INLINE
+ uint32_t compare_and_mask<__m256i, uint32_t>(const __m256i *block_ptr) {
__m256i v = _mm256_load_si256(block_ptr);
__m256i z = _mm256_setzero_si256();
__m256i c = _mm256_cmpeq_epi8(z, v);
@@ -68,14 +71,15 @@ uint32_t CompareAndMask<__m256i, uint32_t>(const __m256i *block_ptr) {
namespace avx2 {
[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {
return string_length_vector<__m256i, uint32_t,
- CompareAndMask<__m256i, uint32_t>>(src);
+ compare_and_mask<__m256i, uint32_t>>(src);
}
} // namespace avx2
#endif
#if defined(__AVX512F__)
template <>
-__mmask64 CompareAndMask<__m512i, __mmask64>(const __m512i *block_ptr) {
+LIBC_INLINE
+ __mmask64 compare_and_mask<__m512i, __mmask64>(const __m512i *block_ptr) {
__m512i v = _mm512_load_si512(block_ptr);
__m512i z = _mm512_setzero_si512();
return _mm512_cmp_epu8_mask(z, v, _MM_CMPINT_EQ);
@@ -83,7 +87,7 @@ __mmask64 CompareAndMask<__m512i, __mmask64>(const __m512i *block_ptr) {
namespace avx512 {
[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {
return string_length_vector<__m512i, __mmask64,
- CompareAndMask<__m512i, __mmask64>>(src);
+ compare_and_mask<__m512i, __mmask64>>(src);
}
} // namespace avx512
#endif
diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h
index cc99633aa49d8..00bd13a590aea 100644
--- a/libc/src/string/string_utils.h
+++ b/libc/src/string/string_utils.h
@@ -22,15 +22,14 @@
#include "src/__support/macros/config.h"
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
-#if defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ)
-#if defined(LIBC_TARGET_ARCH_IS_X86)
+#if defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ) && \
+ defined(LIBC_TARGET_ARCH_IS_X86)
#include "src/string/memory_utils/x86_64/inline_strlen.h"
+namespace string_length_impl = LIBC_NAMESPACE::wide_read;
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_NEON)
#include "src/string/memory_utils/aarch64/inline_strlen.h"
-#else
namespace string_length_impl = LIBC_NAMESPACE::wide_read;
#endif
-#endif
namespace LIBC_NAMESPACE_DECL {
namespace internal {
|
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
jhuber6
force-pushed
the
wide2
branch
5 times, most recently
from
13b47b6 to
9e5ebea
Compare
llvm#154602)" This reverts commit 27fc967.
lntue
approved these changes
Aug 21, 2025
|
I think this PR broke the rv32 buildbot somehow: I guess that you need to forward declare |
|
Should be fixed hopefully |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Reland afterr the sanitizer and arm32 builds complained.