Index unqualified part of the (demangled C++) symbol name

* bdx/index.py: (SymbolNameField.tokenize_value): Find
namespace-qualified parts of the symbol and index the unqualified
parts.
* tests/test_index.py (test_tokenize_symbol): Update test.

Author: mkcms

bdx/index.py

@@ -309,6 +309,9 @@ def tokenize_value(value: str) -> set[str]:
         # Find lowercase words
         lower_case_words = re.findall("[a-z]{2,}", " ".join(letters_only))
 
+        # Symbol names after '::' (C++)
+        unqualified_symbols = re.findall("::(\\w+)", value)
+
         numbers = re.findall("[0-9]+", value)
         words_with_numbers = re.findall("[a-zA-Z]+[0-9]+", value)
 
@@ -320,6 +323,7 @@ def tokenize_value(value: str) -> set[str]:
             lower_case_words,
             numbers,
             words_with_numbers,
+            unqualified_symbols,
         ]:
             tokens.update(tokenlist)
 

tests/test_index.py

@@ -791,3 +791,26 @@ def test_tokenize_symbol():
             "vv",
         ]
     )
+
+    tokens = SymbolNameField.tokenize_value(
+        "std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::_Rep::_M_destroy(std::allocator<wchar_t> const&)"
+    )
+    assert tokens == set(
+        [
+            "Rep",
+            "_M_destroy",
+            "_Rep",
+            "allocator",
+            "basic",
+            "basic_string",
+            "char",
+            "char_traits",
+            "const",
+            "destroy",
+            "ep",
+            "std",
+            "string",
+            "traits",
+            "wchar",
+        ]
+    )