Merge: Case-Folding UTF-8 (ẞ → ss)

Author: ashvardanian

CMakeLists.txt

@@ -435,6 +435,7 @@ if (STRINGZILLA_BUILD_BENCHMARK)
     define_launcher(stringzilla_bench_find_cpp20 scripts/bench_find.cpp 20 "${STRINGZILLA_TARGET_ARCH}")
     define_launcher(stringzilla_bench_sequence_cpp20 scripts/bench_sequence.cpp 20 "${STRINGZILLA_TARGET_ARCH}")
     define_launcher(stringzilla_bench_token_cpp20 scripts/bench_token.cpp 20 "${STRINGZILLA_TARGET_ARCH}")
+    define_launcher(stringzilla_bench_unicode_cpp20 scripts/bench_unicode.cpp 20 "${STRINGZILLA_TARGET_ARCH}")
     define_launcher(stringzilla_bench_container_cpp20 scripts/bench_container.cpp 20 "${STRINGZILLA_TARGET_ARCH}")
     define_launcher(stringzilla_bench_memory_cpp20 scripts/bench_memory.cpp 20 "${STRINGZILLA_TARGET_ARCH}")
 

README.md

@@ -27,13 +27,13 @@ It __accelerates exact and fuzzy string matching, hashing, edit distance computa
 - 🐂 __[C](#basic-usage-with-c-99-and-newer):__ Upgrade LibC's `<string.h>` to `<stringzilla/stringzilla.h>`  in C 99
 - 🐉 __[C++](#basic-usage-with-c-11-and-newer):__ Upgrade STL's `<string>` to `<stringzilla/stringzilla.hpp>` in C++ 11
 - 🧮 __[CUDA](#cuda):__ Process in-bulk with `<stringzillas/stringzillas.cuh>` in CUDA C++ 17
-- 🐍 __[Python](#quick-start-python-🐍):__ Upgrade your `str` to faster `Str`
-- 🦀 __[Rust](#quick-start-rust-🦀):__ Use the `StringZilla` traits crate
-- 🦫 __[Go](#quick-start-golang-🦫):__ Use the `StringZilla` cGo module
-- 🍎 __[Swift](#quick-start-swift-🍏):__ Use the `String+StringZilla` extension
-- 🟨 __[JavaScript](#quick-start-javascript-🟨):__ Use the `StringZilla` library
+- 🐍 __[Python](#quick-start-python):__ Upgrade your `str` to faster `Str`
+- 🦀 __[Rust](#quick-start-rust):__ Use the `StringZilla` traits crate
+- 🦫 __[Go](#quick-start-golang):__ Use the `StringZilla` cGo module
+- 🍎 __[Swift](#quick-start-swift):__ Use the `String+StringZilla` extension
+- 🟨 __[JavaScript](#quick-start-javascript):__ Use the `StringZilla` library
 - 🐚 __[Shell][faq-shell]__: Accelerate common CLI tools with `sz_` prefix
-- 📚 Researcher? Jump to [Algorithms & Design Decisions](#algorithms--design-decisions-📚)
+- 📚 Researcher? Jump to [Algorithms & Design Decisions](#algorithms--design-decisions)
 - 💡 Thinking to contribute? Look for ["good first issues"][first-issues]
 - 🤝 And check the [guide](https://github.com/ashvardanian/StringZilla/blob/main/CONTRIBUTING.md) to set up the environment
 - Want more bindings or features? Let [me](https://github.com/ashvardanian) know!
@@ -343,7 +343,7 @@ Consider contributing if you need a feature that's not yet implemented.
 > ⚪ are considered.
 > ❌ are not intended.
 
-## Quick Start: Python 🐍
+## Quick Start: Python
 
 Python bindings are available on PyPI for Python 3.8+, and can be installed with `pip`.
 
@@ -751,7 +751,7 @@ arr = pa.Array.from_buffers(
 That means you can convert `Str` to `pyarrow.Buffer` and `Strs` to `pyarrow.Array` without extra copies.
 For more details on the tape-like layouts, refer to the [StringTape](https://github.com/ashvardanian/StringTape) repository.
 
-## Quick Start: C/C++ 🛠️
+## Quick Start: C/C++
 
 The C library is header-only, so you can just copy the `stringzilla.h` header into your project.
 Same applies to C++, where you would copy the `stringzilla.hpp` header.
@@ -1527,7 +1527,7 @@ __`STRINGZILLA_BUILD_SHARED`, `STRINGZILLA_BUILD_TEST`, `STRINGZILLA_BUILD_BENCH
 > It's synonymous to GCC's `-march` flag and is used to enable/disable the appropriate instruction sets.
 > You can also disable the shared library build, if you don't need it.
 
-## Quick Start: Rust 🦀
+## Quick Start: Rust
 
 StringZilla is available as a Rust crate, with documentation available on [docs.rs/stringzilla](https://docs.rs/stringzilla).
 You can immediately check the installed version and the used hardware capabilities with following commands:
@@ -1760,7 +1760,7 @@ assert!(hashes.iter().any(|&h| h != u32::MAX));  // Verify computation occurred
 assert!(counts.iter().any(|&c| c != u32::MAX));
 ```
 
-## Quick Start: JavaScript 🟨
+## Quick Start: JavaScript
 
 Install the Node.js package and use zero-copy `Buffer` APIs.
 
@@ -1833,7 +1833,7 @@ const digestBuffer = hasher.digest();     // returns Buffer (32 bytes)
 const digestHex = hasher.hexdigest();     // returns string (64 hex chars)
 ```
 
-## Quick Start: Swift 🍏
+## Quick Start: Swift
 
 StringZilla can be added as a dependency in the Swift Package Manager.
 In your `Package.swift` file, add the following:
@@ -1891,7 +1891,7 @@ let digestBytes = hasher.digest()     // [UInt8] (32 bytes)
 let digestHex = hasher.hexdigest()    // String (64 hex chars)
 ```
 
-## Quick Start: GoLang 🦫
+## Quick Start: GoLang
 
 Add the Go binding as a module dependency:
 
@@ -2003,7 +2003,7 @@ size := hasher.Size()               // 32
 blockSize := hasher.BlockSize()     // 64
 ```
 
-## Algorithms & Design Decisions 📚
+## Algorithms & Design Decisions
 
 StringZilla aims to optimize some of the slowest string operations.
 Some popular operations, however, like equality comparisons and relative order checking, almost always complete on some of the very first bytes in either string.

c/stringzilla.c

@@ -60,10 +60,13 @@ typedef struct sz_implementations_t {
 
     sz_utf8_count_t utf8_count;
     sz_utf8_find_nth_t utf8_find_nth;
-    sz_utf8_unpack_chunk_t utf8_unpack_chunk;
     sz_utf8_find_boundary_t utf8_find_newline;
     sz_utf8_find_boundary_t utf8_find_whitespace;
+    sz_utf8_unpack_chunk_t utf8_unpack_chunk;
+
     sz_utf8_case_fold_t utf8_case_fold;
+    sz_utf8_case_insensitive_find_t utf8_case_insensitive_find;
+    sz_utf8_case_insensitive_order_t utf8_case_insensitive_order;
 
     sz_sequence_argsort_t sequence_argsort;
     sz_sequence_intersect_t sequence_intersect;
@@ -108,10 +111,13 @@ static void sz_dispatch_table_update_implementation_(sz_capability_t caps) {
 
     impl->utf8_count = sz_utf8_count_serial;
     impl->utf8_find_nth = sz_utf8_find_nth_serial;
-    impl->utf8_unpack_chunk = sz_utf8_unpack_chunk_serial;
     impl->utf8_find_newline = sz_utf8_find_newline_serial;
     impl->utf8_find_whitespace = sz_utf8_find_whitespace_serial;
+    impl->utf8_unpack_chunk = sz_utf8_unpack_chunk_serial;
+
     impl->utf8_case_fold = sz_utf8_case_fold_serial;
+    impl->utf8_case_insensitive_find = sz_utf8_case_insensitive_find_serial;
+    impl->utf8_case_insensitive_order = sz_utf8_case_insensitive_order_serial;
 
     impl->sequence_argsort = sz_sequence_argsort_serial;
     impl->sequence_intersect = sz_sequence_intersect_serial;
@@ -164,10 +170,8 @@ static void sz_dispatch_table_update_implementation_(sz_capability_t caps) {
 
         impl->utf8_count = sz_utf8_count_haswell;
         impl->utf8_find_nth = sz_utf8_find_nth_haswell;
-        impl->utf8_unpack_chunk = sz_utf8_unpack_chunk_haswell;
         impl->utf8_find_newline = sz_utf8_find_newline_haswell;
         impl->utf8_find_whitespace = sz_utf8_find_whitespace_haswell;
-        impl->utf8_case_fold = sz_utf8_case_fold_haswell;
     }
 #endif
 
@@ -204,10 +208,12 @@ static void sz_dispatch_table_update_implementation_(sz_capability_t caps) {
 
         impl->utf8_count = sz_utf8_count_ice;
         impl->utf8_find_nth = sz_utf8_find_nth_ice;
-        impl->utf8_unpack_chunk = sz_utf8_unpack_chunk_ice;
         impl->utf8_find_newline = sz_utf8_find_newline_ice;
         impl->utf8_find_whitespace = sz_utf8_find_whitespace_ice;
+        impl->utf8_unpack_chunk = sz_utf8_unpack_chunk_ice;
+
         impl->utf8_case_fold = sz_utf8_case_fold_ice;
+        impl->utf8_case_insensitive_find = sz_utf8_case_insensitive_find_ice;
 
         impl->lookup = sz_lookup_ice;
 
@@ -246,10 +252,12 @@ static void sz_dispatch_table_update_implementation_(sz_capability_t caps) {
 
         impl->utf8_count = sz_utf8_count_neon;
         impl->utf8_find_nth = sz_utf8_find_nth_neon;
-        impl->utf8_unpack_chunk = sz_utf8_unpack_chunk_neon;
         impl->utf8_find_newline = sz_utf8_find_newline_neon;
         impl->utf8_find_whitespace = sz_utf8_find_whitespace_neon;
+        impl->utf8_unpack_chunk = sz_utf8_unpack_chunk_neon;
+
         impl->utf8_case_fold = sz_utf8_case_fold_neon;
+        impl->utf8_case_insensitive_find = sz_utf8_case_insensitive_find_neon;
     }
 #endif
 
@@ -507,6 +515,18 @@ SZ_DYNAMIC sz_size_t sz_utf8_case_fold(sz_cptr_t source, sz_size_t source_length
     return sz_dispatch_table.utf8_case_fold(source, source_length, destination);
 }
 
+SZ_DYNAMIC sz_cptr_t sz_utf8_case_insensitive_find( //
+    sz_cptr_t haystack, sz_size_t haystack_length,  //
+    sz_cptr_t needle, sz_size_t needle_length, sz_size_t *matched_length) {
+    return sz_dispatch_table.utf8_case_insensitive_find(haystack, haystack_length, needle, needle_length,
+                                                        matched_length);
+}
+
+SZ_DYNAMIC sz_ordering_t sz_utf8_case_insensitive_order( //
+    sz_cptr_t a, sz_size_t a_length, sz_cptr_t b, sz_size_t b_length) {
+    return sz_dispatch_table.utf8_case_insensitive_order(a, a_length, b, b_length);
+}
+
 // Provide overrides for the libc mem* functions
 #if SZ_OVERRIDE_LIBC && !defined(__CYGWIN__)
 

include/stringzilla/stringzilla.h

@@ -75,7 +75,7 @@
 #include "hash.h"         // `sz_bytesum`, `sz_hash`, `sz_state_init`, `sz_state_stream`, `sz_state_fold`
 #include "find.h"         // `sz_find`, `sz_find_byteset`, `sz_rfind`
 #include "utf8.h"         // `sz_utf8_find_newline`, `sz_utf8_find_whitespace`, `sz_utf8_find_nth`, `sz_utf8_valid`
-#include "utf8_unpack.h"  // `sz_utf8_case_insensitive_find`, `sz_utf8_unpack_chunk`
+#include "utf8_case.h"    // `sz_utf8_case_insensitive_find`, `sz_utf8_unpack_chunk`
 #include "small_string.h" // `sz_string_t`, `sz_string_init`, `sz_string_free`
 #include "sort.h"         // `sz_sequence_argsort`, `sz_pgrams_sort`
 #include "intersect.h"    // `sz_sequence_intersect`

include/stringzilla/types.h

@@ -788,6 +788,12 @@ typedef sz_cptr_t (*sz_utf8_unpack_chunk_t)(sz_cptr_t, sz_size_t, sz_rune_t *, s
 /** @brief Signature of `sz_utf8_case_fold`. */
 typedef sz_size_t (*sz_utf8_case_fold_t)(sz_cptr_t, sz_size_t, sz_ptr_t);
 
+/** @brief Signature of `sz_utf8_case_insensitive_find`. */
+typedef sz_cptr_t (*sz_utf8_case_insensitive_find_t)(sz_cptr_t, sz_size_t, sz_cptr_t, sz_size_t, sz_size_t *);
+
+/** @brief Signature of `sz_utf8_case_insensitive_order`. */
+typedef sz_ordering_t (*sz_utf8_case_insensitive_order_t)(sz_cptr_t, sz_size_t, sz_cptr_t, sz_size_t);
+
 /** @brief Signature of `sz_fill_random`. */
 typedef void (*sz_fill_random_t)(sz_ptr_t, sz_size_t, sz_u64_t);