[libc] wcstok implementation

[uzairnawaz]

Implemented wcstok and added tests

[llvmbot]

@llvm/pr-subscribers-libc

Author: Uzair Nawaz (uzairnawaz)

Changes

Implemented wcstok and added tests

---

Full diff: https://github.com/llvm/llvm-project/pull/145989.diff

7 Files Affected:

• (modified) libc/config/linux/x86_64/entrypoints.txt (+1)
• (modified) libc/include/wchar.yaml (+8)
• (modified) libc/src/wchar/CMakeLists.txt (+10)
• (added) libc/src/wchar/wcstok.cpp (+46)
• (added) libc/src/wchar/wcstok.h (+22)
• (modified) libc/test/src/wchar/CMakeLists.txt (+10)
• (added) libc/test/src/wchar/wcstok_test.cpp (+145)

diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 6b3fc9485ec1a..bf04ae2e83fb3 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -386,6 +386,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.wchar.wmemchr
     libc.src.wchar.wcpcpy
     libc.src.wchar.wcpncpy
+    libc.src.wchar.wcstok
 
     # sys/uio.h entrypoints
     libc.src.sys.uio.writev
diff --git a/libc/include/wchar.yaml b/libc/include/wchar.yaml
index 397296894829d..15025f42c0723 100644
--- a/libc/include/wchar.yaml
+++ b/libc/include/wchar.yaml
@@ -189,6 +189,14 @@ functions:
     arguments:
       - type: wchar_t *__restrict
       - type: const wchar_t *__restrict
+  - name: wcstok
+    standards:
+      - stdc
+    return_type: wchar_t *
+    arguments:
+      - type: wchar_t *__restrict
+      - type: const wchar_t *__restrict
+      - type: wchar_t** __restrict
   - name: wcpcpy
     standards:
       - stdc
diff --git a/libc/src/wchar/CMakeLists.txt b/libc/src/wchar/CMakeLists.txt
index 16664100d42c7..6d93b82b2d2bf 100644
--- a/libc/src/wchar/CMakeLists.txt
+++ b/libc/src/wchar/CMakeLists.txt
@@ -34,6 +34,16 @@ add_entrypoint_object(
     libc.src.__support.wctype_utils
 )
 
+add_entrypoint_object(
+  wcstok
+  SRCS
+    wcstok.cpp
+  HDRS
+    wcstok.h
+  DEPENDS
+    libc.hdr.types.wchar_t
+)
+
 add_entrypoint_object(
   wcrtomb
   SRCS
diff --git a/libc/src/wchar/wcstok.cpp b/libc/src/wchar/wcstok.cpp
new file mode 100644
index 0000000000000..a02332ac1fbbd
--- /dev/null
+++ b/libc/src/wchar/wcstok.cpp
@@ -0,0 +1,46 @@
+//===-- Implementation of wcstok ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcstok.h"
+
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(wchar_t *, wcstok,
+                   (wchar_t *__restrict str, const wchar_t *__restrict delim,
+                    wchar_t **__restrict ptr)) {
+  if (str == nullptr)
+    str = *ptr;
+
+  bool foundTokenStart = false;
+  wchar_t *out = nullptr;
+  wchar_t *str_ptr;
+  for (str_ptr = str; *str_ptr != L'\0'; str_ptr++) {
+    bool inDelim = false;
+    for (const wchar_t *delim_ptr = delim; *delim_ptr != L'\0' && !inDelim;
+         delim_ptr++)
+      if (*str_ptr == *delim_ptr)
+        inDelim = true;
+
+    if (!inDelim && !foundTokenStart) {
+      foundTokenStart = true;
+      out = str_ptr;
+    } else if (inDelim && foundTokenStart) {
+      *str_ptr = L'\0';
+      *ptr = str_ptr + 1;
+      return out;
+    }
+  }
+
+  *ptr = str_ptr;
+  return out;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/wcstok.h b/libc/src/wchar/wcstok.h
new file mode 100644
index 0000000000000..9f41ea37a947a
--- /dev/null
+++ b/libc/src/wchar/wcstok.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wcstok ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSTOK_H
+#define LLVM_LIBC_SRC_WCHAR_WCSTOK_H
+
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+wchar_t *wcstok(wchar_t *__restrict str, const wchar_t *__restrict delim,
+                wchar_t **__restrict ptr);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSTOK_H
diff --git a/libc/test/src/wchar/CMakeLists.txt b/libc/test/src/wchar/CMakeLists.txt
index bf16fdd7f8c4d..8967cc1e8d353 100644
--- a/libc/test/src/wchar/CMakeLists.txt
+++ b/libc/test/src/wchar/CMakeLists.txt
@@ -111,6 +111,16 @@ add_libc_test(
     libc.src.wchar.wcschr
 )
 
+add_libc_test(
+  wcstok_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcstok_test.cpp
+  DEPENDS
+    libc.src.wchar.wcstok
+)
+
 add_libc_test(
   wcsncmp_test
   SUITE
diff --git a/libc/test/src/wchar/wcstok_test.cpp b/libc/test/src/wchar/wcstok_test.cpp
new file mode 100644
index 0000000000000..79153ccc0adad
--- /dev/null
+++ b/libc/test/src/wchar/wcstok_test.cpp
@@ -0,0 +1,145 @@
+//===-- Unittests for wcstok ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcstok.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcStrTokTest, NoTokenFound) {
+  wchar_t empty[] = L"";
+  wchar_t *buf;
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"", &buf), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"_", &buf), nullptr);
+
+  wchar_t single[] = L"_";
+  wchar_t *token = LIBC_NAMESPACE::wcstok(single, L"", &buf);
+  ASSERT_TRUE(token[0] == L'_');
+  ASSERT_TRUE(token[1] == L'\0');
+
+  wchar_t multiple[] = L"1,2";
+  token = LIBC_NAMESPACE::wcstok(multiple, L":", &buf);
+  ASSERT_TRUE(multiple[0] == L'1');
+  ASSERT_TRUE(multiple[1] == L',');
+  ASSERT_TRUE(multiple[2] == L'2');
+  ASSERT_TRUE(multiple[3] == L'\0');
+}
+
+TEST(LlvmLibcStrTokTest, DelimiterAsFirstCharacterShouldBeIgnored) {
+  wchar_t *buf;
+  wchar_t src[] = L".123";
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L".", &buf);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'3');
+  ASSERT_TRUE(token[3] == L'\0');
+}
+
+TEST(LlvmLibcStrTokTest, DelimiterIsMiddleCharacter) {
+  wchar_t src[] = L"12,34";
+  wchar_t *buf;
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L",", &buf);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'\0');
+}
+
+TEST(LlvmLibcStrTokTest, DelimiterAsLastCharacterShouldBeIgnored) {
+  wchar_t src[] = L"1234:";
+  wchar_t *buf;
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L":", &buf);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'3');
+  ASSERT_TRUE(token[3] == L'4');
+  ASSERT_TRUE(token[4] == L'\0');
+}
+
+TEST(LlvmLibcStrTokTest, MultipleDelimiters) {
+  wchar_t src[] = L"12,.34";
+  wchar_t *buf;
+  wchar_t *token;
+
+  token = LIBC_NAMESPACE::wcstok(src, L".", &buf);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L',');
+  ASSERT_TRUE(token[3] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(src, L".,", &buf);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(src, L",.", &buf);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(src, L":,.", &buf);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'\0');
+}
+
+TEST(LlvmLibcStrTokTest, ShouldNotGoPastNullTerminator) {
+  wchar_t src[] = {L'1', L'2', L'\0', L',', L'3'};
+  wchar_t *buf;
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L",", &buf);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'\0');
+}
+
+TEST(LlvmLibcStrTokTest, SubsequentCallsShouldFindFollowingDelimiters) {
+  wchar_t src[] = L"12,34.56";
+  wchar_t *buf;
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L",.", &buf);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L",.", &buf);
+  ASSERT_TRUE(token[0] == L'3');
+  ASSERT_TRUE(token[1] == L'4');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L",.", &buf);
+  ASSERT_TRUE(token[0] == L'5');
+  ASSERT_TRUE(token[1] == L'6');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,_", &buf);
+  ASSERT_EQ(token, nullptr);
+  // Subsequent calls after hitting the end of the string should also return
+  // nullptr.
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,_", &buf);
+  ASSERT_EQ(token, nullptr);
+}
+
+TEST(LlvmLibcStrTokTest, DelimitersShouldNotBeIncludedInToken) {
+  wchar_t *buf;
+  wchar_t src[] = L"__ab__:_cd__:__ef__:__";
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L"_:", &buf);
+  ASSERT_TRUE(token[0] == L'a');
+  ASSERT_TRUE(token[1] == L'b');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L":_", &buf);
+  ASSERT_TRUE(token[0] == L'c');
+  ASSERT_TRUE(token[1] == L'd');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,", &buf);
+  ASSERT_TRUE(token[0] == L'e');
+  ASSERT_TRUE(token[1] == L'f');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,_", &buf);
+  ASSERT_EQ(token, nullptr);
+}

[michaelrj-google]

Two nits, but other than that LGTM