implement process.extractOne in C++ (#53)

* start to simplify complexion

* start implementation

* add extractOne to C++

* fix a couple of bugs in the implementation

* start adressing performance issues

Author: maxbachmann

.github/workflows/pythonbuild.yml

@@ -9,11 +9,11 @@ on:
 
 jobs:
   test_python:
-    name: run linting, tests and benchmarks for the python module
-    runs-on: ubuntu-latest
+    name: linting and tests on Python ${{ matrix.python-version }}
+    runs-on: ubuntu-18.04
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
 
     steps:
       - uses: actions/checkout@v2
@@ -41,7 +41,7 @@ jobs:
       - name: Run Unit Tests
         run: |
           pip install .
-          pip install pytest
+          pip install pytest hypothesis
           pytest
 
 

.gitignore

@@ -15,3 +15,7 @@ site/
 
 # benchmark results
 bench_results/
+
+# Hypothesis results
+.hypothesis/
+

VERSION

@@ -1 +1 @@
-0.12.5
+0.13.0

setup.py

@@ -17,7 +17,7 @@ class BuildExt(build_ext):
     """A custom build extension for adding compiler-specific options."""
     c_opts = {
         'msvc': ['/EHsc', '/O2', '/std:c++14'],
-        'unix': ['-O3', '-std=c++14'],
+        'unix': ['-O3', '-std=c++14', '-Wextra', '-Wall'],
     }
     l_opts = {
         'msvc': [],

src/py2_utils.hpp

@@ -21,10 +21,14 @@ bool valid_str(PyObject* str, const char* name)
     Py_InitModule3(#name, methods, doc);                                                           \
   }
 
+using python_string =
+    mpark::variant<std::basic_string<uint8_t>, std::basic_string<Py_UNICODE>,
+    rapidfuzz::basic_string_view<uint8_t>, rapidfuzz::basic_string_view<Py_UNICODE>>;
+
 using python_string_view =
     mpark::variant<rapidfuzz::basic_string_view<uint8_t>, rapidfuzz::basic_string_view<Py_UNICODE>>;
 
-python_string_view decode_python_string(PyObject* py_str)
+python_string decode_python_string(PyObject* py_str)
 {
   if (PyObject_TypeCheck(py_str, &PyString_Type)) {
     Py_ssize_t len = PyString_GET_SIZE(py_str);
@@ -38,12 +42,27 @@ python_string_view decode_python_string(PyObject* py_str)
   }
 }
 
-PyObject* encode_python_string(std::basic_string<uint8_t> str)
+python_string_view decode_python_string_view(PyObject* py_str)
+{
+  if (PyObject_TypeCheck(py_str, &PyString_Type)) {
+    Py_ssize_t len = PyString_GET_SIZE(py_str);
+    uint8_t* str = reinterpret_cast<uint8_t*>(PyString_AS_STRING(py_str));
+    return rapidfuzz::basic_string_view<uint8_t>(str, len);
+  }
+  else {
+    Py_ssize_t len = PyUnicode_GET_SIZE(py_str);
+    Py_UNICODE* str = PyUnicode_AS_UNICODE(py_str);
+    return rapidfuzz::basic_string_view<Py_UNICODE>(str, len);
+  }
+}
+
+
+PyObject* encode_python_string(rapidfuzz::basic_string_view<uint8_t> str)
 {
   return PyString_FromStringAndSize(reinterpret_cast<const char*>(str.data()), str.size());
 }
 
-PyObject* encode_python_string(std::basic_string<Py_UNICODE> str)
+PyObject* encode_python_string(rapidfuzz::basic_string_view<Py_UNICODE> str)
 {
   return PyUnicode_FromUnicode(str.data(), str.size());
 }

src/py3_utils.hpp

@@ -6,20 +6,16 @@
 #include "details/types.hpp"
 #include <variant/variant.hpp>
 
-// PEP 623 deprecates legacy strings and therefor
-// deprecates e.g. PyUnicode_READY in Python 3.10
-#if PY_VERSION_HEX < 0x030A0000
-#define PY_BELOW_3_10
-#endif
-
 bool valid_str(PyObject* str, const char* name)
 {
   if (!PyUnicode_Check(str)) {
     PyErr_Format(PyExc_TypeError, "%s must be a String or None", name);
     return false;
   }
 
-#ifdef PY_BELOW_3_10
+  // PEP 623 deprecates legacy strings and therefor
+  // deprecates e.g. PyUnicode_READY in Python 3.10
+#if PY_VERSION_HEX < PYTHON_VERSION(3,10,0)
   if (PyUnicode_READY(str)) {
     return false;
   }
@@ -36,11 +32,31 @@ bool valid_str(PyObject* str, const char* name)
     return PyModule_Create(&moduledef);                                                            \
   }
 
+using python_string =
+    mpark::variant<std::basic_string<uint8_t>, std::basic_string<uint16_t>, std::basic_string<uint32_t>,
+    rapidfuzz::basic_string_view<uint8_t>, rapidfuzz::basic_string_view<uint16_t>,
+                   rapidfuzz::basic_string_view<uint32_t>>;
+
 using python_string_view =
     mpark::variant<rapidfuzz::basic_string_view<uint8_t>, rapidfuzz::basic_string_view<uint16_t>,
                    rapidfuzz::basic_string_view<uint32_t>>;
 
-python_string_view decode_python_string(PyObject* py_str)
+python_string decode_python_string(PyObject* py_str)
+{
+  Py_ssize_t len = PyUnicode_GET_LENGTH(py_str);
+  void* str = PyUnicode_DATA(py_str);
+
+  switch (PyUnicode_KIND(py_str)) {
+  case PyUnicode_1BYTE_KIND:
+    return rapidfuzz::basic_string_view<uint8_t>(static_cast<uint8_t*>(str), len);
+  case PyUnicode_2BYTE_KIND:
+    return rapidfuzz::basic_string_view<uint16_t>(static_cast<uint16_t*>(str), len);
+  default:
+    return rapidfuzz::basic_string_view<uint32_t>(static_cast<uint32_t*>(str), len);
+  }
+}
+
+python_string_view decode_python_string_view(PyObject* py_str)
 {
   Py_ssize_t len = PyUnicode_GET_LENGTH(py_str);
   void* str = PyUnicode_DATA(py_str);
@@ -55,17 +71,17 @@ python_string_view decode_python_string(PyObject* py_str)
   }
 }
 
-PyObject* encode_python_string(std::basic_string<uint8_t> str)
+PyObject* encode_python_string(rapidfuzz::basic_string_view<uint8_t> str)
 {
   return PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, str.data(), str.size());
 }
 
-PyObject* encode_python_string(std::basic_string<uint16_t> str)
+PyObject* encode_python_string(rapidfuzz::basic_string_view<uint16_t> str)
 {
   return PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, str.data(), str.size());
 }
 
-PyObject* encode_python_string(std::basic_string<uint32_t> str)
+PyObject* encode_python_string(rapidfuzz::basic_string_view<uint32_t> str)
 {
   return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, str.data(), str.size());
 }