Skip to content

Commit 0e6c997

Browse files
hendrikmuhshendrikmuhs
committed
fix msgpack unpacking
1 parent 2bf9bf0 commit 0e6c997

File tree

5 files changed

+52
-30
lines changed

5 files changed

+52
-30
lines changed

‎pyproject.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,15 @@ dependencies = [
99
"msgpack>=1.0.0",
1010
]
1111

12+
[project.optional-dependencies]
13+
lint = [
14+
'ruff>=0.5.0',
15+
]
16+
test = [
17+
'pytest>=7.4.2',
18+
]
19+
dev = ['keyvi_scikit_core[lint,test]']
20+
1221
[tool.scikit-build]
1322
wheel.expand-macos-universal-tags = true
1423
minimum-version = "build-system.requires"

‎python-pybind/src/compiler/py_dictionary_compilers.cpp‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,20 @@
2727
namespace py = pybind11;
2828
namespace kd = keyvi::dictionary;
2929

30+
template <typename Compiler>
31+
inline void py_compile(Compiler *c, std::function<void(const size_t a, const size_t b)> progress_callback) {
32+
if (progress_callback == nullptr) {
33+
c->Compile();
34+
return;
35+
}
36+
auto progress_compiler_callback = [](size_t a, size_t b, void *user_data) {
37+
auto py_callback = *reinterpret_cast<std::function<void(const size_t, const size_t)> *>(user_data);
38+
py_callback(a, b);
39+
};
40+
void *user_data = reinterpret_cast<void *>(&progress_callback);
41+
c->Compile(progress_compiler_callback, user_data);
42+
}
43+
3044
void init_keyvi_dictionary_compilers(const py::module_ &module) {
3145
#define CREATE_COMPILER(compiler, name) \
3246
py::class_<compiler>(module, name) \
@@ -35,24 +49,24 @@ void init_keyvi_dictionary_compilers(const py::module_ &module) {
3549
.def("__enter__", [](compiler &c) { return &c; }) \
3650
.def("__exit__", [](compiler &c, void *exc_type, void *exc_value, void *traceback) { c.Compile(); }) \
3751
.def("__setitem__", &compiler::Add) \
38-
.def("add", &compiler::Add) \
52+
.def("add", &compiler::Add) /* DEPRECATED */ \
53+
.def("Add", &compiler::Add) \
3954
.def( \
4055
"compile", \
4156
[](compiler &c, std::function<void(const size_t a, const size_t b)> progress_callback) { \
42-
if (progress_callback == nullptr) { \
43-
c.Compile(); \
44-
return; \
45-
} \
46-
auto progress_compiler_callback = [](size_t a, size_t b, void *user_data) { \
47-
auto py_callback = *reinterpret_cast<std::function<void(const size_t, const size_t)> *>(user_data); \
48-
py_callback(a, b); \
49-
}; \
50-
void *user_data = reinterpret_cast<void *>(&progress_callback); \
51-
c.Compile(progress_compiler_callback, user_data); \
57+
py_compile(&c, progress_callback); \
58+
}, \
59+
py::arg("progress_callback") = \
60+
static_cast<std::function<void(const size_t a, const size_t b)> *>(nullptr)) /* DEPRECATED */ \
61+
.def( \
62+
"Compile", \
63+
[](compiler &c, std::function<void(const size_t a, const size_t b)> progress_callback) { \
64+
py_compile(&c, progress_callback); \
5265
}, \
5366
py::arg("progress_callback") = static_cast<std::function<void(const size_t a, const size_t b)> *>(nullptr)) \
5467
.def("set_manifest", &compiler::SetManifest) \
55-
.def("write_to_file", &compiler::WriteToFile, py::call_guard<py::gil_scoped_release>());
68+
.def("write_to_file", &compiler::WriteToFile, py::call_guard<py::gil_scoped_release>()) /* DEPRECATED */ \
69+
.def("WriteToFile", &compiler::WriteToFile, py::call_guard<py::gil_scoped_release>());
5670
#define CREATE_SK_COMPILER(compiler, name) \
5771
py::class_<compiler>(module, name) \
5872
.def(py::init<const std::vector<std::string> &>()) \
@@ -64,18 +78,7 @@ void init_keyvi_dictionary_compilers(const py::module_ &module) {
6478
.def( \
6579
"compile", \
6680
[](compiler &c, std::function<void(const size_t a, const size_t b)> progress_callback) { \
67-
pybind11::gil_scoped_release release_gil; \
68-
if (progress_callback == nullptr) { \
69-
c.Compile(); \
70-
return; \
71-
} \
72-
auto progress_compiler_callback = [](size_t a, size_t b, void *user_data) { \
73-
auto py_callback = *reinterpret_cast<std::function<void(const size_t, const size_t)> *>(user_data); \
74-
pybind11::gil_scoped_acquire acquire_gil; \
75-
py_callback(a, b); \
76-
}; \
77-
void *user_data = reinterpret_cast<void *>(&progress_callback); \
78-
c.Compile(progress_compiler_callback, user_data); \
81+
py_compile(&c, progress_callback); \
7982
}, \
8083
py::arg("progress_callback") = static_cast<std::function<void(const size_t a, const size_t b)> *>(nullptr)) \
8184
.def("set_manifest", &compiler::SetManifest) \

‎python-pybind/src/dictionary/py_dictionary.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,9 @@ void init_keyvi_dictionary(const py::module_ &m) {
119119
)pbdoc")
120120
.def("get", &kd::Dictionary::operator[], R"pbdoc(
121121
Get an entry from the dictionary.
122+
)pbdoc")
123+
.def("__getitem__", &kd::Dictionary::operator[], R"pbdoc(
124+
Get an entry from the dictionary.
122125
)pbdoc")
123126
.def("match",
124127
[](const kd::Dictionary &d, const std::string &key) {

‎python-pybind/src/dictionary/py_match.cpp‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,13 @@
2727
namespace py = pybind11;
2828
namespace kd = keyvi::dictionary;
2929

30+
inline const py::object &get_msgpack_loads_func() {
31+
PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store<py::object> storage;
32+
return storage
33+
.call_once_and_store_result([]() -> py::object { return py::getattr(py::module_::import("msgpack"), "loads"); })
34+
.get_stored();
35+
}
36+
3037
void init_keyvi_match(const py::module_ &m) {
3138
py::module_ msgpack_ = py::module_::import("msgpack");
3239

@@ -42,7 +49,7 @@ void init_keyvi_match(const py::module_ &m) {
4249
if (packed_value.empty()) {
4350
return py::none();
4451
}
45-
return msgpack_.attr("loads")(packed_value);
52+
return get_msgpack_loads_func()(py::bytes(packed_value));
4653
})
4754
.def("value_as_string", &kd::Match::GetValueAsString)
4855
.def("raw_value_as_string", &kd::Match::GetRawValueAsString)

‎python-pybind/tests/match_object_test.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Usage: py.test tests
33

44
import keyvi_scikit_core as keyvi
5-
#from test_tools import tmp_dictionary
5+
from test_tools import tmp_dictionary
66
import warnings
77

88

@@ -114,7 +114,7 @@ def test_score():
114114
assert m.value == {"a": 3} """
115115

116116

117-
""" def test_get_value_int():
117+
def test_get_value_int():
118118
c = CompletionDictionaryCompiler({"memory_limit_mb": "10"})
119119
c.Add("abc", 42)
120120
c.Add("abd", 21)
@@ -125,7 +125,7 @@ def test_score():
125125
assert m.value == 21
126126

127127

128-
def test_get_value_key_only():
128+
""" def test_get_value_key_only():
129129
c = KeyOnlyDictionaryCompiler({"memory_limit_mb": "10"})
130130
c.Add("abc")
131131
c.Add("abd")
@@ -134,7 +134,7 @@ def test_get_value_key_only():
134134
assert m.value == ''
135135
m = d["abd"]
136136
assert m.value == ''
137-
137+
"""
138138

139139
def test_get_value_string():
140140
c = StringDictionaryCompiler({"memory_limit_mb": "10"})
@@ -145,7 +145,7 @@ def test_get_value_string():
145145
assert m.value == "aaaaa"
146146
m = d["abd"]
147147
assert m.value == "bbbbb"
148-
"""
148+
149149

150150
def test_matched_string():
151151
m = keyvi.Match()

0 commit comments

Comments
 (0)