Skip to content

Commit f4833ec

Browse files
hendrikmuhshendrikmuhs
committed
Merge branch 'master' of github.com:KeyviDev/keyvi into scikit-buildcore
2 parents fe64e1c + 207e0a2 commit f4833ec

File tree

18 files changed

+157
-386
lines changed

18 files changed

+157
-386
lines changed

‎.github/workflows/python-cibuildwheel.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ jobs:
7474
echo "MACOSX_DEPLOYMENT_TARGET=14.0" >> $GITHUB_ENV
7575
7676
- name: Build python wheels for ${{ matrix.os }}
77-
uses: pypa/cibuildwheel@v3.1.4
77+
uses: pypa/cibuildwheel@v3.2.0
7878
env:
7979
# Skip CPython 3.8
8080
CIBW_SKIP: ${{ env.CIBW_SKIP }} cp38-*

‎keyvi/include/keyvi/dictionary/dictionary.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ class Dictionary final {
324324
TRACE("GetAllKeys callback called");
325325

326326
for (;;) {
327-
if (!data->traverser.AtEnd()) {
327+
if (data->traverser) {
328328
data->traversal_stack.resize(data->traverser.GetDepth() - 1);
329329
data->traversal_stack.push_back(data->traverser.GetStateLabel());
330330
TRACE("Current depth %d (%d)", data->traverser.GetDepth() - 1, data->traversal_stack.size());

‎keyvi/include/keyvi/dictionary/fsa/comparable_state_traverser.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,6 @@ class ComparableStateTraverser final {
135135

136136
operator bool() const { return state_traverser_; }
137137

138-
bool AtEnd() const { return state_traverser_.AtEnd(); }
139-
140138
void operator++(int) {
141139
state_traverser_++;
142140
if (state_traverser_) {

‎keyvi/include/keyvi/dictionary/fsa/state_traverser.h‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ class StateTraverser final {
4848
using transition_t = TransitionT;
4949

5050
explicit StateTraverser(automata_t f)
51-
: fsa_(f), current_state_(f->GetStartState()), current_weight_(0), current_label_(0), at_end_(false), stack_() {
51+
: fsa_(f), current_state_(f->GetStartState()), current_weight_(0), current_label_(0), stack_() {
5252
TRACE("StateTraverser starting with Start state %d", current_state_);
5353
f->GetOutGoingTransitions(current_state_, &stack_.GetStates(), &stack_.traversal_stack_payload, 0);
5454

@@ -57,7 +57,7 @@ class StateTraverser final {
5757

5858
StateTraverser(automata_t f, const uint64_t start_state, traversal::TraversalPayload<TransitionT> &&payload,
5959
const bool advance = true)
60-
: fsa_(f), current_weight_(0), current_label_(0), at_end_(false), stack_(std::move(payload)) {
60+
: fsa_(f), current_weight_(0), current_label_(0), stack_(std::move(payload)) {
6161
current_state_ = start_state;
6262

6363
TRACE("StateTraverser starting with Start state %d", current_state_);
@@ -70,7 +70,7 @@ class StateTraverser final {
7070
}
7171

7272
StateTraverser(automata_t f, const uint64_t start_state, const bool advance = true)
73-
: fsa_(f), current_state_(start_state), current_weight_(0), current_label_(0), at_end_(false), stack_() {
73+
: fsa_(f), current_state_(start_state), current_weight_(0), current_label_(0), stack_() {
7474
TRACE("StateTraverser starting with Start state %d", current_state_);
7575
f->GetOutGoingTransitions(start_state, &stack_.GetStates(), &stack_.traversal_stack_payload,
7676
f->GetInnerWeight(start_state));
@@ -89,13 +89,11 @@ class StateTraverser final {
8989
current_state_(other.current_state_),
9090
current_weight_(other.current_weight_),
9191
current_label_(other.current_label_),
92-
at_end_(other.at_end_),
9392
stack_(std::move(other.stack_)) {
9493
other.fsa_ = 0;
9594
other.current_state_ = 0;
9695
other.current_weight_ = 0;
9796
other.current_label_ = 0;
98-
other.at_end_ = true;
9997
}
10098

10199
automata_t GetFsa() const { return fsa_; }
@@ -136,7 +134,7 @@ class StateTraverser final {
136134
if (stack_.GetDepth() == 0) {
137135
TRACE("traverser exhausted.");
138136
current_label_ = 0;
139-
at_end_ = true;
137+
current_state_ = 0;
140138
return;
141139
}
142140

@@ -157,7 +155,7 @@ class StateTraverser final {
157155

158156
label_t GetStateLabel() const { return current_label_; }
159157

160-
operator bool() const { return !at_end_; }
158+
operator bool() const { return current_state_ != 0; }
161159

162160
/**
163161
* Set the minimum weight states must be greater or equal to.
@@ -168,14 +166,11 @@ class StateTraverser final {
168166
*/
169167
inline void SetMinWeight(uint32_t min_weight) {}
170168

171-
bool AtEnd() const { return at_end_; }
172-
173169
private:
174170
automata_t fsa_;
175171
uint64_t current_state_;
176172
uint32_t current_weight_;
177173
label_t current_label_;
178-
bool at_end_;
179174
traversal::TraversalStack<TransitionT> stack_;
180175

181176
/**

‎keyvi/include/keyvi/dictionary/fsa/traversal/weighted_traversal.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,8 @@ inline void TraversalState<WeightedTransition>::Add(uint64_t s, uint32_t weight,
6969
template <>
7070
inline void TraversalState<WeightedTransition>::PostProcess(TraversalPayload<WeightedTransition>* payload) {
7171
if (traversal_state_payload.transitions.size() > 0) {
72-
std::sort(traversal_state_payload.transitions.begin(), traversal_state_payload.transitions.end(),
73-
WeightedTransitionCompare);
72+
std::stable_sort(traversal_state_payload.transitions.begin(), traversal_state_payload.transitions.end(),
73+
WeightedTransitionCompare);
7474
}
7575
}
7676

‎keyvi/tests/keyvi/dictionary/fsa/comparable_state_traverser_test.cpp‎

Lines changed: 46 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
//
1818

1919
#include <memory>
20+
#include <string>
2021
#include <utility>
22+
#include <vector>
2123

2224
#include <boost/test/unit_test.hpp>
2325

@@ -34,14 +36,14 @@ BOOST_AUTO_TEST_SUITE(ComparableStateTraverserTests)
3436

3537
BOOST_AUTO_TEST_CASE(StateTraverserCompatibility) {
3638
std::vector<std::string> test_data = {"aaaa", "aabb", "aabc", "aacd", "bbcd"};
37-
testing::TempDictionary dictionary(&test_data);
38-
automata_t f = dictionary.GetFsa();
39+
const testing::TempDictionary dictionary(&test_data);
40+
const automata_t f = dictionary.GetFsa();
3941

4042
ComparableStateTraverser<StateTraverser<>> s(f);
4143

4244
BOOST_CHECK_EQUAL('a', s.GetStateLabel());
4345
BOOST_CHECK_EQUAL(1, s.GetDepth());
44-
BOOST_CHECK(!s.AtEnd());
46+
BOOST_CHECK(s);
4547
BOOST_CHECK(s);
4648

4749
s++;
@@ -101,25 +103,25 @@ BOOST_AUTO_TEST_CASE(StateTraverserCompatibility) {
101103
// traverser shall be exhausted
102104
s++;
103105
BOOST_CHECK_EQUAL(0, s.GetStateLabel());
104-
BOOST_CHECK(s.AtEnd());
106+
BOOST_CHECK(!s);
105107
BOOST_CHECK(!s);
106108
BOOST_CHECK_EQUAL(0, s.GetDepth());
107109
s++;
108110
BOOST_CHECK_EQUAL(0, s.GetStateLabel());
109-
BOOST_CHECK(s.AtEnd());
111+
BOOST_CHECK(!s);
110112
BOOST_CHECK_EQUAL(0, s.GetDepth());
111113
}
112114

113115
BOOST_AUTO_TEST_CASE(StateTraverserCompatSomeTraversalWithPrune) {
114116
std::vector<std::string> test_data = {"aaaa", "aabb", "aabc", "aacd", "bbcd"};
115-
testing::TempDictionary dictionary(&test_data);
116-
automata_t f = dictionary.GetFsa();
117+
const testing::TempDictionary dictionary(&test_data);
118+
const automata_t f = dictionary.GetFsa();
117119

118120
ComparableStateTraverser<StateTraverser<>> s(f);
119121

120122
BOOST_CHECK_EQUAL('a', s.GetStateLabel());
121123
BOOST_CHECK_EQUAL(1, s.GetDepth());
122-
BOOST_CHECK(!s.AtEnd());
124+
BOOST_CHECK(s);
123125

124126
s++;
125127
BOOST_CHECK_EQUAL('a', s.GetStateLabel());
@@ -140,7 +142,7 @@ BOOST_AUTO_TEST_CASE(StateTraverserCompatSomeTraversalWithPrune) {
140142
BOOST_CHECK_EQUAL(2, s.GetDepth());
141143
BOOST_CHECK_EQUAL(2, s.GetStateLabels().size());
142144
s++;
143-
BOOST_CHECK(!s.AtEnd());
145+
BOOST_CHECK(s);
144146

145147
BOOST_CHECK_EQUAL('c', s.GetStateLabel());
146148
BOOST_CHECK_EQUAL(3, s.GetDepth());
@@ -165,65 +167,65 @@ BOOST_AUTO_TEST_CASE(StateTraverserCompatSomeTraversalWithPrune) {
165167
// traverser shall be exhausted
166168
s++;
167169
BOOST_CHECK_EQUAL(0, s.GetStateLabel());
168-
BOOST_CHECK(s.AtEnd());
170+
BOOST_CHECK(!s);
169171
BOOST_CHECK_EQUAL(0, s.GetDepth());
170172
s++;
171173
BOOST_CHECK_EQUAL(0, s.GetStateLabel());
172-
BOOST_CHECK(s.AtEnd());
174+
BOOST_CHECK(!s);
173175
BOOST_CHECK_EQUAL(0, s.GetDepth());
174176
}
175177

176178
BOOST_AUTO_TEST_CASE(StateTraverserCompatLongkeys) {
177-
std::string a(1000, 'a');
178-
std::string b(1000, 'b');
179+
const std::string a(1000, 'a');
180+
const std::string b(1000, 'b');
179181

180182
std::vector<std::string> test_data = {a, b};
181-
testing::TempDictionary dictionary(&test_data);
182-
automata_t f = dictionary.GetFsa();
183+
const testing::TempDictionary dictionary(&test_data);
184+
const automata_t f = dictionary.GetFsa();
183185

184186
ComparableStateTraverser<StateTraverser<>> s(f);
185187

186188
for (int i = 1; i <= 1000; ++i) {
187189
BOOST_CHECK_EQUAL('a', s.GetStateLabel());
188190
BOOST_CHECK_EQUAL(i, s.GetDepth());
189191
s++;
190-
BOOST_CHECK(!s.AtEnd());
192+
BOOST_CHECK(s);
191193
}
192194

193195
for (int i = 1; i <= 1000; ++i) {
194196
BOOST_CHECK_EQUAL('b', s.GetStateLabel());
195197
BOOST_CHECK_EQUAL(i, s.GetDepth());
196198
s++;
197199
if (i != 1000) {
198-
BOOST_CHECK(!s.AtEnd());
200+
BOOST_CHECK(s);
199201
} else {
200-
BOOST_CHECK(s.AtEnd());
202+
BOOST_CHECK(!s);
201203
}
202204
}
203205

204206
// traverser shall be exhausted
205207
s++;
206208
BOOST_CHECK_EQUAL(0, s.GetStateLabel());
207-
BOOST_CHECK(s.AtEnd());
209+
BOOST_CHECK(!s);
208210
BOOST_CHECK_EQUAL(0, s.GetDepth());
209211
s++;
210212
BOOST_CHECK_EQUAL(0, s.GetStateLabel());
211-
BOOST_CHECK(s.AtEnd());
213+
BOOST_CHECK(!s);
212214

213215
BOOST_CHECK_EQUAL(0, s.GetDepth());
214216
}
215217

216218
BOOST_AUTO_TEST_CASE(StateTraverserCompatZeroByte) {
217219
std::vector<std::string> test_data = {std::string("\0aaaa", 5), std::string("aa\0bb", 5), "aabc", "aacd",
218220
std::string("bbcd\0", 5)};
219-
testing::TempDictionary dictionary(&test_data);
220-
automata_t f = dictionary.GetFsa();
221+
const testing::TempDictionary dictionary(&test_data);
222+
const automata_t f = dictionary.GetFsa();
221223

222224
ComparableStateTraverser<StateTraverser<>> s(f);
223225

224226
BOOST_CHECK_EQUAL('\0', s.GetStateLabel());
225227
BOOST_CHECK_EQUAL(1, s.GetDepth());
226-
BOOST_CHECK(!s.AtEnd());
228+
BOOST_CHECK(s);
227229

228230
s++;
229231
BOOST_CHECK_EQUAL('a', s.GetStateLabel());
@@ -311,23 +313,23 @@ BOOST_AUTO_TEST_CASE(StateTraverserCompatZeroByte) {
311313
s++;
312314
BOOST_CHECK_EQUAL(0, s.GetStateLabel());
313315
BOOST_CHECK_EQUAL(0, s.GetDepth());
314-
BOOST_CHECK(s.AtEnd());
316+
BOOST_CHECK(!s);
315317

316318
s++;
317319
BOOST_CHECK_EQUAL(0, s.GetStateLabel());
318320
BOOST_CHECK_EQUAL(0, s.GetDepth());
319-
BOOST_CHECK(s.AtEnd());
321+
BOOST_CHECK(!s);
320322
}
321323

322324
BOOST_AUTO_TEST_CASE(same_data) {
323325
std::vector<std::string> test_data = {"aaaa", "aabb", "aabc", "aacd", "bbcd"};
324-
testing::TempDictionary dictionary(&test_data);
325-
automata_t f = dictionary.GetFsa();
326+
const testing::TempDictionary dictionary(&test_data);
327+
const automata_t f = dictionary.GetFsa();
326328

327329
ComparableStateTraverser<StateTraverser<>> t1(f, false, 0);
328330
ComparableStateTraverser<StateTraverser<>> t2(f, false, 0);
329331
ComparableStateTraverser<StateTraverser<>> t3(f, false, 1);
330-
ComparableStateTraverser<StateTraverser<>> t4(f, false, 1);
332+
const ComparableStateTraverser<StateTraverser<>> t4(f, false, 1);
331333

332334
BOOST_CHECK((t1 < t2) == false);
333335
BOOST_CHECK((t2 < t1) == false);
@@ -371,12 +373,12 @@ BOOST_AUTO_TEST_CASE(same_data) {
371373

372374
BOOST_AUTO_TEST_CASE(interleave1) {
373375
std::vector<std::string> test_data1 = {"aaaa", "aabb", "aabc", "aacd", "bbcd"};
374-
testing::TempDictionary dictionary1(&test_data1);
375-
automata_t f1 = dictionary1.GetFsa();
376+
const testing::TempDictionary dictionary1(&test_data1);
377+
const automata_t f1 = dictionary1.GetFsa();
376378

377379
std::vector<std::string> test_data2 = {"aabb", "aabd", "abcd", "bbcd"};
378-
testing::TempDictionary dictionary2(&test_data2);
379-
automata_t f2 = dictionary2.GetFsa();
380+
const testing::TempDictionary dictionary2(&test_data2);
381+
const automata_t f2 = dictionary2.GetFsa();
380382

381383
ComparableStateTraverser<StateTraverser<>> t1(f1, false, 0);
382384
ComparableStateTraverser<StateTraverser<>> t2(f2, false, 1);
@@ -415,12 +417,12 @@ BOOST_AUTO_TEST_CASE(interleave1) {
415417

416418
BOOST_AUTO_TEST_CASE(interleave2) {
417419
std::vector<std::string> test_data1 = {"aaaa", "aabb", "aabc", "aacd", "bbcd"};
418-
testing::TempDictionary dictionary1(&test_data1);
419-
automata_t f1 = dictionary1.GetFsa();
420+
const testing::TempDictionary dictionary1(&test_data1);
421+
const automata_t f1 = dictionary1.GetFsa();
420422

421423
std::vector<std::string> test_data2 = {"aaab", "aabd", "abcd", "bbcd"};
422-
testing::TempDictionary dictionary2(&test_data2);
423-
automata_t f2 = dictionary2.GetFsa();
424+
const testing::TempDictionary dictionary2(&test_data2);
425+
const automata_t f2 = dictionary2.GetFsa();
424426

425427
ComparableStateTraverser<StateTraverser<>> t1(f1, false, 0);
426428
ComparableStateTraverser<StateTraverser<>> t2(f2, false, 1);
@@ -449,12 +451,12 @@ BOOST_AUTO_TEST_CASE(interleave2) {
449451

450452
BOOST_AUTO_TEST_CASE(nearTraversalSpecialization) {
451453
std::vector<std::string> test_data1 = {"aaaa", "aabb", "aabc", "aacd", "bbcd", "cdefgh"};
452-
testing::TempDictionary dictionary1(&test_data1);
453-
automata_t f1 = dictionary1.GetFsa();
454+
const testing::TempDictionary dictionary1(&test_data1);
455+
const automata_t f1 = dictionary1.GetFsa();
454456

455457
std::vector<std::string> test_data2 = {"abbb", "aabc", "bbcd", "aaceh", "cdefgh"};
456-
testing::TempDictionary dictionary2(&test_data2);
457-
automata_t f2 = dictionary2.GetFsa();
458+
const testing::TempDictionary dictionary2(&test_data2);
459+
const automata_t f2 = dictionary2.GetFsa();
458460

459461
std::shared_ptr<std::string> near_key = std::make_shared<std::string>("aace");
460462

@@ -537,12 +539,12 @@ BOOST_AUTO_TEST_CASE(nearTraversalSpecialization) {
537539

538540
BOOST_AUTO_TEST_CASE(nearTraversalSpecialization2) {
539541
std::vector<std::string> test_data1 = {"aaaa", "aabb", "aabc", "aacd", "bbcd", "cdefgh"};
540-
testing::TempDictionary dictionary1(&test_data1);
541-
automata_t f1 = dictionary1.GetFsa();
542+
const testing::TempDictionary dictionary1(&test_data1);
543+
const automata_t f1 = dictionary1.GetFsa();
542544

543545
std::vector<std::string> test_data2 = {"abbb", "aabc", "bbcd", "aaceh", "aacexyz", "aacexz", "cdefgh"};
544-
testing::TempDictionary dictionary2(&test_data2);
545-
automata_t f2 = dictionary2.GetFsa();
546+
const testing::TempDictionary dictionary2(&test_data2);
547+
const automata_t f2 = dictionary2.GetFsa();
546548

547549
std::shared_ptr<std::string> near_key = std::make_shared<std::string>("aace");
548550

0 commit comments

Comments
 (0)