Add vector-based strlen implementation for x86_64 and aarch64

libc/src/string/inline_strlen.h

@@ -0,0 +1,38 @@
+//===-- Strlen implementation -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRLEN_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRLEN_H
+
+#include "src/__support/macros/attributes.h"               // LIBC_INLINE
+#include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_
+
+#include <stddef.h> // size_t
+
+#if defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ)
+#if defined(LIBC_TARGET_ARCH_IS_X86)
+#include "src/string/memory_utils/x86_64/inline_strlen.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_STRLEN_WIDE_READ string_length_x86
+#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
+#include "src/string/memory_utils/aarch64/inline_memcpy.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_STRLEN_WIDE_READ string_length_aarch64
+#else
+#define LIBC_SRC_STRING_MEMORY_UTILS_STRLEN_WIDE_READ string_length_wide_read
+#endif
+
+namespace LIBC_NAMESPACE_DECL {
+
+[[gnu::flatten]] LIBC_INLINE void
+inline_memcpy(void *__restrict dst, const void *__restrict src, size_t count) {
+  LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY(reinterpret_cast<Ptr>(dst),
+                                      reinterpret_cast<CPtr>(src), count);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRLEN_H

libc/src/string/memory_utils/aarch64/inline_strlen.h

@@ -0,0 +1,50 @@
+//===-- Strlen implementation for aarch64 ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H
+
+#include <arm_neon.h>
+#include <stddef.h> // size_t
+
+namespace LIBC_NAMESPACE_DECL {
+
+size_t string_length_neon(const char *src) {
+  using Vector __attribute__((may_alias)) = uint8x8_t;
+  uintptr_t misalign_bytes = reinterpret_case<uintptr_t>(src) % sizeof(Vector);
+  Vector *block_ptr = reinterpret_cast<Vector *>(src - misalign_bytes);
+  if (misalign_bytes) {
+    Vector v = *block_ptr;
+    Vector vcmp = vceqz_u8(v);
+    uint64x1_t cmp_mask = vreinterpret_u64_s8(vcmp);
+    uint64_t cmp = vget_lane_u64(cmp_mask, 0);
+    cmp = cmp >> (misalign_bytes << 3);
+    if (cmp)
+      return __builtin_ctzl(cmp) >> 3;
+    ++block_ptr;
+  }
+  while (true) {
+    Vector v = *block_ptr;
+    Vector vcmp = vceqz_u8(v);
+    uint64x1_t cmp_mask = vreinterpret_u64_s8(vcmp);
+    uint64_t cmp = vget_lane_u64(cmp_mask, 0);
+    if (cmp)
+      return static_cast<size_t>(reinterpret_case<uintptr_t>(block_ptr) -
+                                 reinterpret_case<uintptr_t>(src) +
+                                 (__builtin_ctzl(cmp) >> 3));
+    block_ptr++;
+  }
+}
+
+template <typename T>
+[[maybe_unused]] LIBC_INLINE void string_length_aarch64(const char *src) {
+  return inline_string_length_neon(src);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H

libc/src/string/memory_utils/x86_64/inline_strlen.h

@@ -0,0 +1,113 @@
+//===-- Strlen implementation for x86_64 ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H
+
+#include "src/string/memory_utils/op_x86.h" // K_AVX
+
+#include <stddef.h> // size_t
+#include <x86intrin.h>
+namespace LIBC_NAMESPACE_DECL {
+
+#if defined(__SSE2__)
+[[maybe_unused]] LIBC_INLINE size_t string_length_sse2(const char *src) {
+  using Vector __attribute__((may_alias)) = __m128i;
+  Vector z = _mm_setzero_si128();
+  uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
+  const Vector *block_ptr =
+      reinterpret_cast<const Vector *>(src - misalign_bytes);
+  if (misalign_bytes) {
+    Vector v = _mm_load_si128(block_ptr);
+    Vector vcmp = _mm_cmpeq_epi8(z, v);
+    // shift away results in irrelevant bytes.
+    int cmp = _mm_movemask_epi8(vcmp) >> misalign_bytes;
+    if (cmp)
+      return __builtin_ctz(cmp);
+    block_ptr++;
+  }
+  while (true) {
+    Vector v = _mm_load_si128(block_ptr);
+    Vector vcmp = _mm_cmpeq_epi8(z, v);
+    int cmp = _mm_movemask_epi8(vcmp);
+    if (cmp)
+      return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
+                                 reinterpret_cast<uintptr_t>(src) +
+                                 __builtin_ctz(cmp));
+    block_ptr++;
+  }
+}
+#endif
+
+#if defined(__AVX2__)
+[[maybe_unused]] LIBC_INLINE size_t string_length_avx2(const char *src) {
+  using Vector __attribute__((may_alias)) = __mm256i;
+  Vector z = _mm256_setzero_si256();
+  uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
+  const Vector *block_ptr =
+      reinterpret_cast<const Vector *>(src - misalign_bytes);
+  if (misalign_bytes) {
+    Vector v = _mm256_load_si256(block_ptr);
+    Vector vcmp = _mm256_cmpeq_epi8(z, v);
+    // shift away results in irrelevant bytes.
+    int cmp = _mm256_movemask_epi8(vcmp) >> misalign_bytes;
+    if (cmp)
+      return __builtin_ctz(cmp);
+    block_ptr++;
+  }
+  while (true) {
+    Vector v = _mm256_load_si256(block_ptr);
+    Vector vcmp = _mm256_cmpeq_epi8(z, v);
+    int cmp = _mm256_movemask_epi8(vcmp);
+    if (cmp)
+      return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
+                                 reinterpret_cast<uintptr_t>(src) +
+                                 __builtin_ctz(cmp));
+    block_ptr++;
+  }
+}
+#endif // __AVX__
+
+#if defined(__AVX512F__)
+[[maybe_unused]] LIBC_INLINE size_t string_length_avx512(const char *src) {
+  using Vector __attribute__((may_alias)) = __mm512i;
+  Vector z = _mm512_setzero_si512();
+  uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
+  const Vector *block_ptr =
+      reinterpret_cast<const Vector *>(src - misalign_bytes);
+  if (misalign_bytes) {
+    Vector v = _mm512_load_si512(block_ptr);
+    __mmask64 cmp = _mm512_cmp_epu8_mask(z, v, _MM_CMPINT_EQ) >> misalign_bytes;
+    if (cmp)
+      return __builtin_ctzl(cmp);
+    block_ptr++;
+  }
+  while (true) {
+    Vector v = _mm512_load_si512(block_ptr);
+    __mmask64 cmp = _mm512_cmp_epu8_mask(z, v, _MM_CMPINT_EQ);
+    if (cmp)
+      return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
+                                 reinterpret_cast<uintptr_t>(src) +
+                                 __builtin_ctz(cmp));
+    block_ptr++;
+  }
+}
+#endif // __AVX512F__
+
+template <typename T> LIBC_INLINE size_t string_length_x86_64(const char *src) {
+#if defined(__AVX512F__)
+  return string_length_avx512(src);
+#endif
+#if defined(__AVX__)
+  return string_length_avx2(src);
+#endif
+  return string_length_sse2(src);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H

libc/src/string/string_utils.h

@@ -22,6 +22,18 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
 
+#if defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ)
+#if defined(LIBC_TARGET_ARCH_IS_X86)
+#include "src/string/memory_utils/x86_64/inline_strlen.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_STRLEN_WIDE_READ string_length_x86_64
+#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
+#include "src/string/memory_utils/aarch64/inline_strlen.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_STRLEN_WIDE_READ string_length_aarch64
+#else
+#define LIBC_SRC_STRING_MEMORY_UTILS_STRLEN_WIDE_READ string_length_wide_read
+#endif
+#endif
+
 namespace LIBC_NAMESPACE_DECL {
 namespace internal {
 
@@ -90,7 +102,7 @@ template <typename T> LIBC_INLINE size_t string_length(const T *src) {
   // be aligned to a word boundary, so it's the size we use for reading the
   // string a block at a time.
   if constexpr (cpp::is_same_v<T, char>)
-    return string_length_wide_read<unsigned int>(src);
+    return LIBC_SRC_STRING_MEMORY_UTILS_STRLEN_WIDE_READ<unsigned int>(src);
 #endif
   size_t length;
   for (length = 0; *src; ++src, ++length)

utils/bazel/llvm-project-overlay/libc/BUILD.bazel

@@ -4865,6 +4865,7 @@ libc_support_library(
         "src/string/memory_utils/aarch64/inline_memcpy.h",
         "src/string/memory_utils/aarch64/inline_memmove.h",
         "src/string/memory_utils/aarch64/inline_memset.h",
+        "src/string/memory_utils/aarch64/inline_strlen.h",
         "src/string/memory_utils/arm/common.h",
         "src/string/memory_utils/arm/inline_memcpy.h",
         "src/string/memory_utils/arm/inline_memset.h",
@@ -4889,6 +4890,7 @@ libc_support_library(
         "src/string/memory_utils/x86_64/inline_memcpy.h",
         "src/string/memory_utils/x86_64/inline_memmove.h",
         "src/string/memory_utils/x86_64/inline_memset.h",
+        "src/string/memory_utils/x86_64/inline_strlen.h",
     ],
     deps = [
         ":__support_common",
@@ -4913,6 +4915,7 @@ libc_support_library(
         ":__support_macros_optimization",
         ":hdr_limits_macros",
         ":llvm_libc_types_size_t",
+        ":string_memory_utils",
         ":types_size_t",
     ],
 )