fix more issues with new rapidfuzz-cpp

Author: maxbachmann

CHANGELOG.rst

@@ -15,6 +15,7 @@ Performance
 Changed
 ~~~~~~~
 * upgrade to ``Cython==3.0.7``
+* cdist for many metrics now returns a matrix of ``uint32`` instead of ``int32`` by default
 
 [3.5.2] - 2023-11-02
 ^^^^^^^^^^^^^^^^^^^^

src/rapidfuzz/_utils.py

@@ -20,22 +20,23 @@
 class ScorerFlag(IntFlag):
     RESULT_F64 = 1 << 5
     RESULT_I64 = 1 << 6
+    RESULT_SIZE_T = 1 << 7
     SYMMETRIC = 1 << 11
 
 
 def _get_scorer_flags_distance(**_kwargs: Any) -> dict[str, Any]:
     return {
         "optimal_score": 0,
         "worst_score": 2**63 - 1,
-        "flags": ScorerFlag.RESULT_I64 | ScorerFlag.SYMMETRIC,
+        "flags": ScorerFlag.RESULT_SIZE_T | ScorerFlag.SYMMETRIC,
     }
 
 
 def _get_scorer_flags_similarity(**_kwargs: Any) -> dict[str, Any]:
     return {
         "optimal_score": 2**63 - 1,
         "worst_score": 0,
-        "flags": ScorerFlag.RESULT_I64 | ScorerFlag.SYMMETRIC,
+        "flags": ScorerFlag.RESULT_SIZE_T | ScorerFlag.SYMMETRIC,
     }
 
 

src/rapidfuzz/distance/metrics_py.py

@@ -196,7 +196,7 @@
 
 
 def _get_scorer_flags_levenshtein_distance(weights: tuple[int, int, int] | None = (1, 1, 1)) -> dict[str, Any]:
-    flags = ScorerFlag.RESULT_I64
+    flags = ScorerFlag.RESULT_SIZE_T
     if weights is None or weights[0] == weights[1]:
         flags |= ScorerFlag.SYMMETRIC
 
@@ -208,7 +208,7 @@ def _get_scorer_flags_levenshtein_distance(weights: tuple[int, int, int] | None
 
 
 def _get_scorer_flags_levenshtein_similarity(weights: tuple[int, int, int] | None = (1, 1, 1)) -> dict[str, Any]:
-    flags = ScorerFlag.RESULT_I64
+    flags = ScorerFlag.RESULT_SIZE_T
     if weights is None or weights[0] == weights[1]:
         flags |= ScorerFlag.SYMMETRIC
 

src/rapidfuzz/process_cpp.hpp

@@ -74,6 +74,9 @@ struct ExtractComp {
         if (m_scorer_flags->flags & RF_SCORER_FLAG_RESULT_F64) {
             return is_first(a, b, m_scorer_flags->optimal_score.f64, m_scorer_flags->worst_score.f64);
         }
+        if (m_scorer_flags->flags & RF_SCORER_FLAG_RESULT_SIZE_T) {
+            return is_first(a, b, m_scorer_flags->optimal_score.sizet, m_scorer_flags->worst_score.sizet);
+        }
         else {
             return is_first(a, b, m_scorer_flags->optimal_score.i64, m_scorer_flags->worst_score.i64);
         }

src/rapidfuzz/process_cpp_impl.pyx

@@ -1727,6 +1727,8 @@ cdef inline MatrixType dtype_to_type_num_py(dtype, scorer, dict scorer_kwargs) e
         flags = params["get_scorer_flags"](**scorer_kwargs)
         if <int>flags["flags"] & RF_SCORER_FLAG_RESULT_I64:
             return MatrixType.INT32
+        if <int>flags["flags"] & RF_SCORER_FLAG_RESULT_SIZE_T:
+            return MatrixType.UINT32
         return MatrixType.FLOAT32
 
     return MatrixType.FLOAT32

src/rapidfuzz/process_py.py

@@ -527,6 +527,8 @@ def _dtype_to_type_num(
         flags = params["get_scorer_flags"](**scorer_kwargs)
         if flags["flags"] & ScorerFlag.RESULT_I64:
             return np.int32
+        if flags["flags"] & ScorerFlag.RESULT_SIZE_T:
+            return np.uint32
         return np.float32
 
     return np.float32

src/rapidfuzz/rapidfuzz.h

@@ -156,7 +156,7 @@ typedef bool (*RF_ScorerFuncInit)(RF_ScorerFunc* self, const RF_Kwargs* kwargs,
 #define RF_SCORER_FLAG_RESULT_I64 ((uint32_t)1 << 6)
 
 /* scorer returns result as size_t */
-#define RF_SCORER_FLAG_RESULT_SIZE_T ((uint32_t)1 << 6)
+#define RF_SCORER_FLAG_RESULT_SIZE_T ((uint32_t)1 << 7)
 
 /* scorer is symmetric: scorer(a, b) == scorer(b, a) */
 #define RF_SCORER_FLAG_SYMMETRIC ((uint32_t)1 << 11)

tests/test_process.py

@@ -510,7 +510,7 @@ def generate_choices():
 
 def test_cdist_pure_python_dtype():
     np = pytest.importorskip("numpy")
-    assert process.cdist(["test"], ["test"], scorer=Levenshtein_py.distance).dtype == np.int32
-    assert process.cdist(["test"], ["test"], scorer=Levenshtein_py.similarity).dtype == np.int32
+    assert process.cdist(["test"], ["test"], scorer=Levenshtein_py.distance).dtype == np.uint32
+    assert process.cdist(["test"], ["test"], scorer=Levenshtein_py.similarity).dtype == np.uint32
     assert process.cdist(["test"], ["test"], scorer=Levenshtein_py.normalized_distance).dtype == np.float32
     assert process.cdist(["test"], ["test"], scorer=Levenshtein_py.normalized_similarity).dtype == np.float32