perf: Optimize string handling with C++17 std::string_view

Introduce std::string_view across core hot-path APIs to eliminate
unnecessary string copies and improve performance through zero-copy
string references.

Modified APIs:
- string_utils: NormalizeText, GenerateHybridNgrams, Utf8ToCodepoints
- Index: AddDocument, UpdateDocument, RemoveDocument, Count, GetPostingList
- DocumentStore: AddDocument, GetDocId, GetFilterValue, FilterByValue
- QueryParser: Parse, Tokenize, ParseFilterOp (most frequent hot path)
- StructuredLog: Added string_view overload for Field()

Technical implementations:
- Handle C++17 unordered_map limitations (no heterogeneous lookup)
- Integrate with ICU StringPiece using explicit int32_t casts
- Use move semantics to minimize copies when conversion is required

Testing:
- All 1182 tests pass
- No breaking changes
- Zero compiler warnings

Author: libraz

src/index/index.cpp

@@ -22,7 +22,7 @@ Index::Index(int ngram_size, int kanji_ngram_size, double roaring_threshold)
       kanji_ngram_size_(kanji_ngram_size > 0 ? kanji_ngram_size : ngram_size),
       roaring_threshold_(roaring_threshold) {}
 
-void Index::AddDocument(DocId doc_id, const std::string& text) {
+void Index::AddDocument(DocId doc_id, std::string_view text) {
   // Generate n-grams using hybrid mode (no lock needed for this CPU-intensive operation)
   std::vector<std::string> ngrams = utils::GenerateHybridNgrams(text, ngram_size_, kanji_ngram_size_);
 
@@ -82,7 +82,7 @@ void Index::AddDocumentBatch(const std::vector<DocumentItem>& documents) {
   }
 }
 
-void Index::UpdateDocument(DocId doc_id, const std::string& old_text, const std::string& new_text) {
+void Index::UpdateDocument(DocId doc_id, std::string_view old_text, std::string_view new_text) {
   // Generate n-grams for both texts (no lock needed for CPU-intensive operation)
   std::vector<std::string> old_ngrams = utils::GenerateHybridNgrams(old_text, ngram_size_, kanji_ngram_size_);
   std::vector<std::string> new_ngrams = utils::GenerateHybridNgrams(new_text, ngram_size_, kanji_ngram_size_);
@@ -120,7 +120,7 @@ void Index::UpdateDocument(DocId doc_id, const std::string& old_text, const std:
   spdlog::debug("Updated document {}", doc_id);
 }
 
-void Index::RemoveDocument(DocId doc_id, const std::string& text) {
+void Index::RemoveDocument(DocId doc_id, std::string_view text) {
   // Generate n-grams (no lock needed for CPU-intensive operation)
   std::vector<std::string> ngrams = utils::GenerateHybridNgrams(text, ngram_size_, kanji_ngram_size_);
 
@@ -397,7 +397,7 @@ std::vector<DocId> Index::SearchNot(const std::vector<DocId>& all_docs, const st
   return result;
 }
 
-uint64_t Index::Count(const std::string& term) const {
+uint64_t Index::Count(std::string_view term) const {
   // Acquire shared lock for read-only access (allows concurrent readers)
   std::shared_lock<std::shared_mutex> lock(postings_mutex_);
   const auto* posting = GetPostingList(term);
@@ -705,22 +705,25 @@ void Index::Clear() {
   spdlog::info("Cleared index");
 }
 
-PostingList* Index::GetOrCreatePostingList(const std::string& term) {
-  auto iterator = term_postings_.find(term);
+PostingList* Index::GetOrCreatePostingList(std::string_view term) {
+  // C++17: unordered_map doesn't support heterogeneous lookup, convert to std::string
+  std::string term_str(term);
+  auto iterator = term_postings_.find(term_str);
   if (iterator != term_postings_.end()) {
     return iterator->second.get();
   }
 
   // Create new posting list
   auto posting = std::make_shared<PostingList>(roaring_threshold_);
   auto* ptr = posting.get();
-  term_postings_[term] = std::move(posting);
+  term_postings_[std::move(term_str)] = std::move(posting);
   return ptr;
 }
 
-const PostingList* Index::GetPostingList(const std::string& term) const {
+const PostingList* Index::GetPostingList(std::string_view term) const {
   // NOTE: This method assumes postings_mutex_ is already locked by the caller
-  auto iterator = term_postings_.find(term);
+  // C++17: unordered_map doesn't support heterogeneous lookup, so we convert to std::string
+  auto iterator = term_postings_.find(std::string(term));
   return iterator != term_postings_.end() ? iterator->second.get() : nullptr;
 }
 

src/index/index.h

@@ -12,6 +12,7 @@
 #include <mutex>
 #include <shared_mutex>
 #include <string>
+#include <string_view>
 #include <unordered_map>
 #include <vector>
 
@@ -66,7 +67,7 @@ class Index {
    * @param doc_id Document ID
    * @param text Normalized text content
    */
-  void AddDocument(DocId doc_id, const std::string& text);
+  void AddDocument(DocId doc_id, std::string_view text);
 
   /**
    * @brief Add multiple documents to index (batch operation, thread-safe)
@@ -87,15 +88,15 @@ class Index {
    * @param old_text Old text content
    * @param new_text New text content
    */
-  void UpdateDocument(DocId doc_id, const std::string& old_text, const std::string& new_text);
+  void UpdateDocument(DocId doc_id, std::string_view old_text, std::string_view new_text);
 
   /**
    * @brief Remove document from index
    *
    * @param doc_id Document ID
    * @param text Text content
    */
-  void RemoveDocument(DocId doc_id, const std::string& text);
+  void RemoveDocument(DocId doc_id, std::string_view text);
 
   /**
    * @brief Search for documents containing all terms (AND)
@@ -132,7 +133,7 @@ class Index {
    * @param term Search term
    * @return Document count
    */
-  [[nodiscard]] uint64_t Count(const std::string& term) const;
+  [[nodiscard]] uint64_t Count(std::string_view term) const;
 
   /**
    * @brief Get total number of unique terms
@@ -218,7 +219,7 @@ class Index {
    * @param term Search term
    * @return Pointer to posting list, or nullptr if not found
    */
-  [[nodiscard]] const PostingList* GetPostingList(const std::string& term) const;
+  [[nodiscard]] const PostingList* GetPostingList(std::string_view term) const;
 
   /**
    * @brief Get n-gram size for regular text
@@ -253,7 +254,7 @@ class Index {
   /**
    * @brief Get or create posting list for term
    */
-  PostingList* GetOrCreatePostingList(const std::string& term);
+  PostingList* GetOrCreatePostingList(std::string_view term);
 
   /**
    * @brief Internal search methods (no locking, assumes caller holds lock)

src/query/query_parser.cpp

@@ -46,8 +46,8 @@ size_t CalculateQueryExpressionLength(const Query& query) {
 /**
  * @brief Convert string to uppercase
  */
-std::string ToUpper(const std::string& str) {
-  std::string result = str;
+std::string ToUpper(std::string_view str) {
+  std::string result(str);
   std::transform(result.begin(), result.end(), result.begin(),
                  [](unsigned char character) { return std::toupper(character); });
   return result;
@@ -94,7 +94,7 @@ bool Query::IsValid() const {
   return true;
 }
 
-mygram::utils::Expected<Query, mygram::utils::Error> QueryParser::Parse(const std::string& query_str) {
+mygram::utils::Expected<Query, mygram::utils::Error> QueryParser::Parse(std::string_view query_str) {
   using mygram::utils::Error;
   using mygram::utils::ErrorCode;
   using mygram::utils::MakeError;
@@ -1089,7 +1089,7 @@ bool QueryParser::ParseSort(const std::vector<std::string>& tokens, size_t& pos,
   return true;
 }
 
-std::vector<std::string> QueryParser::Tokenize(const std::string& str) {
+std::vector<std::string> QueryParser::Tokenize(std::string_view str) {
   std::vector<std::string> tokens;
   std::string token;
   char quote_char = '\0';  // '\0' = not in quotes, '"' or '\'' = in quotes
@@ -1183,7 +1183,7 @@ std::vector<std::string> QueryParser::Tokenize(const std::string& str) {
   return tokens;
 }
 
-std::optional<FilterOp> QueryParser::ParseFilterOp(const std::string& op_str) {
+std::optional<FilterOp> QueryParser::ParseFilterOp(std::string_view op_str) {
   std::string normalized_op = ToUpper(op_str);
 
   if (normalized_op == "=" || normalized_op == "EQ") {

src/query/query_parser.h

@@ -9,6 +9,7 @@
 #include <cstdint>
 #include <optional>
 #include <string>
+#include <string_view>
 #include <unordered_map>
 #include <vector>
 
@@ -213,7 +214,7 @@ class QueryParser {
    * @param query_str Query string
    * @return Expected<Query, Error> - Parsed query or error
    */
-  mygram::utils::Expected<Query, mygram::utils::Error> Parse(const std::string& query_str);
+  mygram::utils::Expected<Query, mygram::utils::Error> Parse(std::string_view query_str);
 
   /**
    * @brief Get last error message
@@ -283,17 +284,17 @@ class QueryParser {
   /**
    * @brief Tokenize query string
    */
-  std::vector<std::string> Tokenize(const std::string& str);
+  std::vector<std::string> Tokenize(std::string_view str);
 
   /**
    * @brief Parse filter operator
    */
-  static std::optional<FilterOp> ParseFilterOp(const std::string& op_str);
+  static std::optional<FilterOp> ParseFilterOp(std::string_view op_str);
 
   /**
    * @brief Set error message
    */
-  void SetError(const std::string& msg) { error_ = msg; }
+  void SetError(std::string_view msg) { error_ = std::string(msg); }
 
   /**
    * @brief Validate query length against configured limit

src/storage/document_store.cpp

@@ -88,12 +88,13 @@ inline void ReadBinary(std::istream& input_stream, T& data) {
 
 }  // namespace
 
-Expected<DocId, Error> DocumentStore::AddDocument(const std::string& primary_key,
+Expected<DocId, Error> DocumentStore::AddDocument(std::string_view primary_key,
                                                   const std::unordered_map<std::string, FilterValue>& filters) {
   std::unique_lock lock(mutex_);
 
-  // Check if primary key already exists
-  auto iterator = pk_to_doc_id_.find(primary_key);
+  // Check if primary key already exists (convert string_view to string for unordered_map lookup)
+  std::string primary_key_str(primary_key);
+  auto iterator = pk_to_doc_id_.find(primary_key_str);
   if (iterator != pk_to_doc_id_.end()) {
     mygram::utils::StructuredLog()
         .Event("storage_warning")
@@ -114,8 +115,8 @@ Expected<DocId, Error> DocumentStore::AddDocument(const std::string& primary_key
   DocId doc_id = next_doc_id_++;
 
   // Store mappings
-  doc_id_to_pk_[doc_id] = primary_key;
-  pk_to_doc_id_[primary_key] = doc_id;
+  doc_id_to_pk_[doc_id] = primary_key_str;
+  pk_to_doc_id_[std::move(primary_key_str)] = doc_id;
 
   // Store filters
   if (!filters.empty()) {
@@ -244,9 +245,10 @@ std::optional<Document> DocumentStore::GetDocument(DocId doc_id) const {
   return doc;
 }
 
-std::optional<DocId> DocumentStore::GetDocId(const std::string& primary_key) const {
+std::optional<DocId> DocumentStore::GetDocId(std::string_view primary_key) const {
   std::shared_lock lock(mutex_);
-  auto iterator = pk_to_doc_id_.find(primary_key);
+  // C++17: unordered_map doesn't support heterogeneous lookup, convert to std::string
+  auto iterator = pk_to_doc_id_.find(std::string(primary_key));
   if (iterator == pk_to_doc_id_.end()) {
     return std::nullopt;
   }
@@ -262,27 +264,30 @@ std::optional<std::string> DocumentStore::GetPrimaryKey(DocId doc_id) const {
   return iterator->second;
 }
 
-std::optional<FilterValue> DocumentStore::GetFilterValue(DocId doc_id, const std::string& filter_name) const {
+std::optional<FilterValue> DocumentStore::GetFilterValue(DocId doc_id, std::string_view filter_name) const {
   std::shared_lock lock(mutex_);
   auto doc_it = doc_filters_.find(doc_id);
   if (doc_it == doc_filters_.end()) {
     return std::nullopt;
   }
 
-  auto filter_it = doc_it->second.find(filter_name);
+  // C++17: unordered_map doesn't support heterogeneous lookup, convert to std::string
+  auto filter_it = doc_it->second.find(std::string(filter_name));
   if (filter_it == doc_it->second.end()) {
     return std::nullopt;
   }
 
   return filter_it->second;
 }
 
-std::vector<DocId> DocumentStore::FilterByValue(const std::string& filter_name, const FilterValue& value) const {
+std::vector<DocId> DocumentStore::FilterByValue(std::string_view filter_name, const FilterValue& value) const {
   std::shared_lock lock(mutex_);
   std::vector<DocId> results;
 
+  // C++17: unordered_map doesn't support heterogeneous lookup, convert to std::string
+  std::string filter_name_str(filter_name);
   for (const auto& [doc_id, filters] : doc_filters_) {
-    auto iterator = filters.find(filter_name);
+    auto iterator = filters.find(filter_name_str);
     if (iterator != filters.end() && iterator->second == value) {
       results.push_back(doc_id);
     }
@@ -310,12 +315,14 @@ std::vector<DocId> DocumentStore::GetAllDocIds() const {
   return results;
 }
 
-bool DocumentStore::HasFilterColumn(const std::string& filter_name) const {
+bool DocumentStore::HasFilterColumn(std::string_view filter_name) const {
   std::shared_lock lock(mutex_);
 
+  // C++17: unordered_map doesn't support heterogeneous lookup, convert to std::string
+  std::string filter_name_str(filter_name);
   // Check if any document has this filter column
-  return std::any_of(doc_filters_.begin(), doc_filters_.end(), [&filter_name](const auto& doc_filter) {
-    return doc_filter.second.find(filter_name) != doc_filter.second.end();
+  return std::any_of(doc_filters_.begin(), doc_filters_.end(), [&filter_name_str](const auto& doc_filter) {
+    return doc_filter.second.find(filter_name_str) != doc_filter.second.end();
   });
 }
 

src/storage/document_store.h

@@ -10,6 +10,7 @@
 #include <optional>
 #include <shared_mutex>
 #include <string>
+#include <string_view>
 #include <unordered_map>
 #include <variant>
 #include <vector>
@@ -97,7 +98,7 @@ class DocumentStore {
    * @param filters Filter column values
    * @return Expected<DocId, Error> Assigned DocID or error (e.g., DocID exhausted)
    */
-  [[nodiscard]] Expected<DocId, Error> AddDocument(const std::string& primary_key,
+  [[nodiscard]] Expected<DocId, Error> AddDocument(std::string_view primary_key,
                                                    const std::unordered_map<std::string, FilterValue>& filters = {});
 
   /**
@@ -143,7 +144,7 @@ class DocumentStore {
    * @param primary_key Primary key
    * @return DocID if exists
    */
-  [[nodiscard]] std::optional<DocId> GetDocId(const std::string& primary_key) const;
+  [[nodiscard]] std::optional<DocId> GetDocId(std::string_view primary_key) const;
 
   /**
    * @brief Get primary key by DocID
@@ -160,7 +161,7 @@ class DocumentStore {
    * @param filter_name Filter column name
    * @return Filter value if exists
    */
-  [[nodiscard]] std::optional<FilterValue> GetFilterValue(DocId doc_id, const std::string& filter_name) const;
+  [[nodiscard]] std::optional<FilterValue> GetFilterValue(DocId doc_id, std::string_view filter_name) const;
 
   /**
    * @brief Filter documents by value
@@ -169,7 +170,7 @@ class DocumentStore {
    * @param value Filter value
    * @return Vector of matching DocIDs
    */
-  [[nodiscard]] std::vector<DocId> FilterByValue(const std::string& filter_name, const FilterValue& value) const;
+  [[nodiscard]] std::vector<DocId> FilterByValue(std::string_view filter_name, const FilterValue& value) const;
 
   /**
    * @brief Get all document IDs
@@ -187,7 +188,7 @@ class DocumentStore {
    * @param filter_name Filter column name
    * @return true if the column exists in at least one document
    */
-  [[nodiscard]] bool HasFilterColumn(const std::string& filter_name) const;
+  [[nodiscard]] bool HasFilterColumn(std::string_view filter_name) const;
 
   /**
    * @brief Get total document count (thread-safe)