|
| 1 | +//===-- Strlen implementation for x86_64 ----------------------------------===// |
| 2 | +// |
| 3 | +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | +// See https://llvm.org/LICENSE.txt for license information. |
| 5 | +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | +// |
| 7 | +//===----------------------------------------------------------------------===// |
| 8 | +#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H |
| 9 | +#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H |
| 10 | + |
| 11 | +#include "src/string/memory_utils/op_x86.h" // K_AVX |
| 12 | + |
| 13 | +#include <stddef.h> // size_t |
| 14 | +#include <x86intrin.h> |
| 15 | +namespace LIBC_NAMESPACE_DECL { |
| 16 | + |
| 17 | +#if defined(__SSE2__) |
| 18 | +[[maybe_unused]] LIBC_INLINE size_t string_length_sse2(const char *src) { |
| 19 | + using Vector __attribute__((may_alias)) = __m128i; |
| 20 | + Vector z = _mm_setzero_si128(); |
| 21 | + uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector); |
| 22 | + const Vector *block_ptr = reinterpret_cast<const Vector *>(src - misalign_bytes); |
| 23 | + if (misalign_bytes) |
| 24 | + { |
| 25 | + Vector v = _mm_load_si128 (block_ptr); |
| 26 | + Vector vcmp = _mm_cmpeq_epi8 (z, v); |
| 27 | + // shift away results in irrelevant bytes. |
| 28 | + int cmp = _mm_movemask_epi8 (vcmp) >> misalign_bytes; |
| 29 | + if (cmp) |
| 30 | + return __builtin_ctz (cmp); |
| 31 | + block_ptr++; |
| 32 | + } |
| 33 | + while (true) |
| 34 | + { |
| 35 | + Vector v = _mm_load_si128 (block_ptr); |
| 36 | + Vector vcmp = _mm_cmpeq_epi8 (z, v); |
| 37 | + int cmp = _mm_movemask_epi8 (vcmp); |
| 38 | + if (cmp) |
| 39 | + return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) - |
| 40 | + reinterpret_cast<uintptr_t>(src) + |
| 41 | + __builtin_ctz(cmp)); |
| 42 | + block_ptr++; |
| 43 | + } |
| 44 | +} |
| 45 | +#endif |
| 46 | + |
| 47 | +#if defined(__AVX2__) |
| 48 | +[[maybe_unused]] LIBC_INLINE size_t string_length_avx2(const char *src) { |
| 49 | + using Vector __attribute__((may_alias)) = __mm256i; |
| 50 | + Vector z = _mm256_setzero_si256(); |
| 51 | + uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector); |
| 52 | + const Vector *block_ptr = reinterpret_cast<const Vector *>(src - misalign_bytes); |
| 53 | + if (misalign_bytes) |
| 54 | + { |
| 55 | + Vector v = _mm256_load_si256 (block_ptr); |
| 56 | + Vector vcmp = _mm256_cmpeq_epi8 (z, v); |
| 57 | + // shift away results in irrelevant bytes. |
| 58 | + int cmp = _mm256_movemask_epi8 (vcmp) >> misalign_bytes; |
| 59 | + if (cmp) |
| 60 | + return __builtin_ctz(cmp); |
| 61 | + block_ptr++; |
| 62 | + } |
| 63 | + while (true) |
| 64 | + { |
| 65 | + Vector v = _mm256_load_si256 (block_ptr); |
| 66 | + Vector vcmp = _mm256_cmpeq_epi8 (z, v); |
| 67 | + int cmp = _mm256_movemask_epi8 (vcmp); |
| 68 | + if (cmp) |
| 69 | + return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) - |
| 70 | + reinterpret_cast<uintptr_t>(src) + |
| 71 | + __builtin_ctz(cmp)); |
| 72 | + block_ptr++; |
| 73 | + } |
| 74 | +} |
| 75 | +#endif // __AVX__ |
| 76 | + |
| 77 | +#if defined(__AVX512F__) |
| 78 | +[[maybe_unused]] LIBC_INLINE size_t string_length_avx512(const char *src) { |
| 79 | + using Vector __attribute__((may_alias)) = __mm512i; |
| 80 | + Vector z = _mm512_setzero_si512(); |
| 81 | + uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector); |
| 82 | + const Vector *block_ptr = reinterpret_cast<const Vector *>(src - misalign_bytes); |
| 83 | + if (misalign_bytes) { |
| 84 | + Vector v = _mm512_load_si512(block_ptr); |
| 85 | + __mmask64 cmp = _mm512_cmp_epu8_mask(z, v, _MM_CMPINT_EQ) >> misalign_bytes; |
| 86 | + if (cmp) |
| 87 | + return __builtin_ctzl(cmp); |
| 88 | + block_ptr++; |
| 89 | + } |
| 90 | + while (true) |
| 91 | + { |
| 92 | + Vector v = _mm512_load_si512(block_ptr); |
| 93 | + __mmask64 cmp = _mm512_cmp_epu8_mask(z, v, _MM_CMPINT_EQ); |
| 94 | + if (cmp) |
| 95 | + return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) - |
| 96 | + reinterpret_cast<uintptr_t>(src) + |
| 97 | + __builtin_ctz(cmp)); |
| 98 | + block_ptr++; |
| 99 | + } |
| 100 | +} |
| 101 | +#endif // __AVX512F__ |
| 102 | + |
| 103 | +template<typename T> LIBC_INLINE |
| 104 | +size_t string_length_x86_64(const char *src) { |
| 105 | +#if defined(__AVX512F__) |
| 106 | + return string_length_avx512(src); |
| 107 | +#endif |
| 108 | +#if defined(__AVX__) |
| 109 | + return string_length_avx2(src); |
| 110 | +#endif |
| 111 | + return string_length_sse2(src); |
| 112 | +} |
| 113 | + |
| 114 | +} // namespace LIBC_NAMESPACE_DECL |
| 115 | + |
| 116 | +#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H |
0 commit comments