Tokenizing UTF-8 with SIMD ㊗️

.github/workflows/prerelease.yml

@@ -802,7 +802,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-13, windows-2022]
+        os: [ubuntu-24.04, macos-15, windows-2022]
         python-version: ["38", "39", "310", "311", "312", "313"]
     steps:
       - uses: actions/checkout@v5

.github/workflows/release.yml

@@ -110,7 +110,7 @@ jobs:
 
   build_wheels_stringzilla_macos:
     name: Build StringZilla ${{ matrix.python-version }} for macOS
-    runs-on: macos-13
+    runs-on: macos-15
     needs: versioning
     strategy:
       fail-fast: false
@@ -137,7 +137,7 @@ jobs:
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
-          name: cibw-wheels-stringzilla-macos-13-${{ strategy.job-index }}
+          name: cibw-wheels-stringzilla-macos-15-${{ strategy.job-index }}
           path: ./wheelhouse/*.whl
           overwrite: true
 
@@ -211,7 +211,7 @@ jobs:
 
   build_wheels_stringzillas_cpus_macos:
     name: Build StringZillas-CPUs ${{ matrix.python-version }} for macOS
-    runs-on: macos-13
+    runs-on: macos-15
     needs: versioning
     strategy:
       fail-fast: false
@@ -239,7 +239,7 @@ jobs:
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
-          name: cibw-wheels-stringzillas-cpus-macos-13-${{ strategy.job-index }}
+          name: cibw-wheels-stringzillas-cpus-macos-15-${{ strategy.job-index }}
           path: ./wheelhouse/*.whl
           overwrite: true
 

.vscode/build_current.cmake

@@ -0,0 +1,52 @@
+# build_current.cmake
+# Cross-platform script to build the current file's target
+#
+# Usage: cmake -DFILE=<file> -DBUILD_TYPE=<Debug|Release> -P build_current.cmake
+
+cmake_minimum_required(VERSION 3.14)
+
+if (NOT DEFINED FILE)
+    message(FATAL_ERROR "FILE not specified. Usage: cmake -DFILE=path/to/file.cpp -P build_current.cmake")
+endif ()
+
+if (NOT DEFINED BUILD_TYPE)
+    set(BUILD_TYPE "Debug")
+    message(STATUS "BUILD_TYPE not specified, defaulting to Debug")
+endif ()
+
+# Extract basename without extension
+get_filename_component(BASENAME "${FILE}" NAME_WE)
+
+# Map filename patterns to CMake targets
+if (BASENAME MATCHES "^bench_(.+)$")
+    # Benchmark files: bench_find.cpp -> stringzilla_bench_find_cpp20
+    set(TARGET "stringzilla_${BASENAME}_cpp20")
+else ()
+    message(FATAL_ERROR "Unknown file pattern: ${BASENAME}\nSupported patterns:\n  - bench_*.cpp\n  - test_stringzilla.cpp\n  - test_stringzillas.cpp")
+endif ()
+
+# Determine build directory
+string(TOLOWER "${BUILD_TYPE}" build_type_lower)
+set(BUILD_DIR "${CMAKE_CURRENT_LIST_DIR}/../build_${build_type_lower}")
+
+# Verify build directory exists
+if (NOT EXISTS "${BUILD_DIR}")
+    message(FATAL_ERROR "Build directory not found: ${BUILD_DIR}\nRun: cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -B ${BUILD_DIR}")
+endif ()
+
+message(STATUS "Building target: ${TARGET}")
+message(STATUS "Build directory: ${BUILD_DIR}")
+message(STATUS "Build type: ${BUILD_TYPE}")
+
+# Execute the build
+execute_process(
+    COMMAND cmake --build ${BUILD_DIR} --config ${BUILD_TYPE} --target ${TARGET}
+    RESULT_VARIABLE result
+    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/..
+)
+
+if (result EQUAL 0)
+    message(STATUS "Build succeeded: ${TARGET}")
+else ()
+    message(FATAL_ERROR "Build failed with exit code: ${result}")
+endif ()

.vscode/launch.json

@@ -106,13 +106,17 @@
           "name": "ASAN_OPTIONS",
           "value": "detect_leaks=0:atexit=1:strict_init_order=1:strict_string_checks=1"
         },
+        {
+          "name": "STRINGWARS_TOKENS",
+          "value": "file"
+        },
         {
           "name": "STRINGWARS_DATASET",
           "value": "utf8.txt"
         }
       ],
       "stopAtEntry": false,
-      "preLaunchTask": "Build Benchmarks: Debug",
+      "preLaunchTask": "Build Current File: Debug",
       "linux": {
         "MIMode": "gdb",
         "setupCommands": [
@@ -149,7 +153,7 @@
         }
       ],
       "stopAtEntry": false,
-      "preLaunchTask": "Build Benchmarks: Debug",
+      "preLaunchTask": "Build Current File: Debug",
       "linux": {
         "MIMode": "gdb",
         "setupCommands": [
@@ -182,7 +186,7 @@
         }
       ],
       "stopAtEntry": false,
-      "preLaunchTask": "Build Benchmarks: Debug"
+      "preLaunchTask": "Build Current File: Debug"
     },
     {
       "name": "Current Python File",

.vscode/settings.json

@@ -263,6 +263,22 @@
   "editor.insertSpaces": true,
   "editor.rulers": [120],
   "editor.tabSize": 4,
+  // Rust analyzer memory optimizations
+  "rust-analyzer.checkOnSave.enable": true,
+  "rust-analyzer.checkOnSave.allTargets": false,
+  "rust-analyzer.cargo.allFeatures": false,
+  "rust-analyzer.cargo.buildScripts.enable": false,
+  "rust-analyzer.procMacro.enable": false,
+  "rust-analyzer.diagnostics.enable": true,
+  "rust-analyzer.diagnostics.experimental.enable": false,
+  "rust-analyzer.completion.autoimport.enable": false,
+  "rust-analyzer.lens.enable": false,
+  "rust-analyzer.inlayHints.enable": false,
+  "rust-analyzer.hover.actions.enable": false,
+  "rust-analyzer.assist.emitMustUse": false,
+  "rust-analyzer.cachePriming.enable": false,
+  "rust-analyzer.files.watcher": "client",
+  "rust-analyzer.updates.channel": "stable",
   "files.associations": {
     "__availability": "cpp",
     "__bit_reference": "cpp",

.vscode/tasks.json

@@ -7,16 +7,12 @@
       "args": [],
       "type": "shell",
       "osx": {
-        "environment": [
-          {
-            "name": "CMAKE_CXX_COMPILER",
-            "value": "$(brew --prefix llvm)/bin/clang++"
-          },
-          {
-            "name": "CMAKE_C_COMPILER",
-            "value": "$(brew --prefix llvm)/bin/clang"
+        "options": {
+          "env": {
+            "CMAKE_CXX_COMPILER": "$(brew --prefix llvm)/bin/clang++",
+            "CMAKE_C_COMPILER": "$(brew --prefix llvm)/bin/clang"
           }
-        ]
+        }
       }
     },
     {
@@ -25,16 +21,12 @@
       "args": [],
       "type": "shell",
       "osx": {
-        "environment": [
-          {
-            "name": "CMAKE_CXX_COMPILER",
-            "value": "$(brew --prefix llvm)/bin/clang++"
-          },
-          {
-            "name": "CMAKE_C_COMPILER",
-            "value": "$(brew --prefix llvm)/bin/clang"
+        "options": {
+          "env": {
+            "CMAKE_CXX_COMPILER": "$(brew --prefix llvm)/bin/clang++",
+            "CMAKE_C_COMPILER": "$(brew --prefix llvm)/bin/clang"
           }
-        ]
+        }
       }
     },
     {
@@ -49,16 +41,12 @@
       "args": [],
       "type": "shell",
       "osx": {
-        "environment": [
-          {
-            "name": "CMAKE_CXX_COMPILER",
-            "value": "$(brew --prefix llvm)/bin/clang++"
-          },
-          {
-            "name": "CMAKE_C_COMPILER",
-            "value": "$(brew --prefix llvm)/bin/clang"
+        "options": {
+          "env": {
+            "CMAKE_CXX_COMPILER": "$(brew --prefix llvm)/bin/clang++",
+            "CMAKE_C_COMPILER": "$(brew --prefix llvm)/bin/clang"
           }
-        ]
+        }
       }
     },
     {
@@ -67,16 +55,47 @@
       "args": [],
       "type": "shell",
       "osx": {
-        "environment": [
-          {
-            "name": "CMAKE_CXX_COMPILER",
-            "value": "$(brew --prefix llvm)/bin/clang++"
-          },
-          {
-            "name": "CMAKE_C_COMPILER",
-            "value": "$(brew --prefix llvm)/bin/clang"
+        "options": {
+          "env": {
+            "CMAKE_CXX_COMPILER": "$(brew --prefix llvm)/bin/clang++",
+            "CMAKE_C_COMPILER": "$(brew --prefix llvm)/bin/clang"
           }
-        ]
+        }
+      }
+    },
+    {
+      "label": "Build Current File: Debug",
+      "command": "cmake",
+      "args": [
+        "-DFILE=${file}",
+        "-DBUILD_TYPE=Debug",
+        "-P",
+        "${workspaceFolder}/.vscode/build_current.cmake"
+      ],
+      "type": "shell",
+      "group": {
+        "kind": "build",
+        "isDefault": true
+      },
+      "presentation": {
+        "reveal": "always",
+        "panel": "shared"
+      }
+    },
+    {
+      "label": "Build Current File: Release",
+      "command": "cmake",
+      "args": [
+        "-DFILE=${file}",
+        "-DBUILD_TYPE=Release",
+        "-P",
+        "${workspaceFolder}/.vscode/build_current.cmake"
+      ],
+      "type": "shell",
+      "group": "build",
+      "presentation": {
+        "reveal": "always",
+        "panel": "shared"
       }
     }
   ]

CMakeLists.txt

@@ -702,7 +702,10 @@ function (define_stringzillas_shared target source_file backend_flags)
     target_include_directories(${target} PRIVATE fork_union/include)
     target_compile_definitions(${target} PRIVATE "SZ_DYNAMIC_DISPATCH=1")
     target_compile_definitions(${target} PRIVATE "SZ_AVOID_LIBC=0")
-    target_compile_definitions(${target} PRIVATE "SZ_DEBUG=0")
+    # Only define SZ_DEBUG=0 in Release builds; Debug builds inherit from types.h
+    if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+        target_compile_definitions(${target} PRIVATE "SZ_DEBUG=0")
+    endif ()
 
     # Set backend-specific compilation flags
     foreach (flag ${backend_flags})

Cargo.toml

@@ -36,10 +36,10 @@ rocm = ["std", "cpus"] # ROCm GPU backend (includes multi-threaded CPU backend)
 
 [dependencies]
 allocator-api2 = { version = "0.3.0", optional = true }
-stringtape = { version = "2.0.3", optional = true }
+stringtape = { version = "2.4.1", optional = true }
 
 [build-dependencies]
-cc = "1.2.40"
+cc = "1.2.47"
 
 [lints.clippy]
 # Catch platform-specific type issues like `c_char` differences

README.md

@@ -1485,6 +1485,13 @@ __`SZ_USE_CUDA`, `SZ_USE_KEPLER`, `SZ_USE_HOPPER`__:
 > One can explicitly disable certain families of PTX instructions for compatibility purposes.
 > Default values are inferred at compile time depending on compiler support (for dynamic dispatch) and the target architecture (for static dispatch).
 
+__`SZ_ENFORCE_SVE_OVER_NEON`__:
+
+> SVE and SVE2 are expected to supersede NEON on ARM architectures.
+> Still, oftentimes the equivalent SVE kernels are slower due to equally small register files and higher complexity of the instructions.
+> By default, when both SVE and NEON are available, SVE is used selectively only for the algorithms that benefit from it.
+> If you want to enforce SVE usage everywhere, define this flag.
+
 __`SZ_DYNAMIC_DISPATCH`__:
 
 > By default, StringZilla is a header-only library.

build.rs

@@ -130,14 +130,16 @@ fn build_stringzilla() -> HashMap<String, bool> {
     println!("cargo:rerun-if-changed=include/stringzilla/stringzilla.h");
 
     // Constituent parts:
-    println!("cargo:rerun-if-changed=include/stringzilla/compare.h");
     println!("cargo:rerun-if-changed=include/stringzilla/find.h");
     println!("cargo:rerun-if-changed=include/stringzilla/hash.h");
+    println!("cargo:rerun-if-changed=include/stringzilla/sort.h");
+    println!("cargo:rerun-if-changed=include/stringzilla/utf8.h");
+    println!("cargo:rerun-if-changed=include/stringzilla/utf8_unpack.h");
+    println!("cargo:rerun-if-changed=include/stringzilla/types.h");
     println!("cargo:rerun-if-changed=include/stringzilla/memory.h");
+    println!("cargo:rerun-if-changed=include/stringzilla/compare.h");
     println!("cargo:rerun-if-changed=include/stringzilla/similarities.h");
     println!("cargo:rerun-if-changed=include/stringzilla/small_string.h");
-    println!("cargo:rerun-if-changed=include/stringzilla/sort.h");
-    println!("cargo:rerun-if-changed=include/stringzilla/types.h");
 
     // Rerun if SIMD backend environment variables change
     for flag in flags_to_try.iter() {