11#include " MPSToken.hpp"
22#include " MPSCommon.hpp"
3- #include < regex>
3+
4+ MPSToken::MPSToken ( MPSToken* parent,
5+ const std::wstring& text,
6+ const std::wstring& separators,
7+ bool discard) :
8+ m_parent(parent),
9+ m_text(text),
10+ m_separators(separators),
11+ m_discard(discard)
12+ {}
413
514MPSToken::~MPSToken ()
615{
@@ -21,14 +30,14 @@ void MPSToken::shift_subtoken (MPSToken* sub_token, EMPSDirection direction)
2130 if (!sub_token)
2231 return ;
2332
24- if (m_sub_tokens .empty ())
33+ if (m_subtokens .empty ())
2534 return ;
2635
27- MPSTokensContainer::iterator iter = m_sub_tokens .begin ();
28- for ( ; iter != m_sub_tokens .end (); ++iter) {
36+ MPSTokensContainer::iterator iter = m_subtokens .begin ();
37+ for ( ; iter != m_subtokens .end (); ++iter) {
2938 if (*iter == sub_token) {
3039 if (ELeft == direction) {
31- if (iter == m_sub_tokens .begin ())
40+ if (iter == m_subtokens .begin ())
3241 return ; // shift left requested on first subtoken, do nothing
3342
3443 MPSTokensContainer::iterator left = iter;
@@ -40,7 +49,7 @@ void MPSToken::shift_subtoken (MPSToken* sub_token, EMPSDirection direction)
4049 } else if (ERight == direction) {
4150 MPSTokensContainer::iterator right = iter;
4251 ++right;
43- if (right == m_sub_tokens .end ())
52+ if (right == m_subtokens .end ())
4453 return ; // shift right requested on last subtoken, do nothing
4554
4655 std::swap (*iter, *right);
@@ -56,9 +65,6 @@ void MPSToken::split()
5665 if (m_separators.empty ())
5766 return ;
5867
59- // check https://www.journaldev.com/37223/tokenize-string-c-plus-plus
60-
61-
6268 BoostSeparator separ (m_separators.c_str ());
6369 BoostTokenizer tokenizer (m_text, separ);
6470
@@ -82,7 +88,7 @@ void MPSToken::split()
8288 for ( ; iter != tokenizer.end (); ++iter) {
8389 // create sub-tokens
8490 MPSToken* sub_token = new MPSToken (this , *iter, L" " , m_discard);
85- m_sub_tokens .push_back (sub_token);
91+ m_subtokens .push_back (sub_token);
8692 ++count;
8793 }
8894}
@@ -96,15 +102,15 @@ void MPSToken::insert_subtoken (const std::wstring& text, size_t pos)
96102{
97103 if (pos == KLastSubtokenPosition) {
98104 // add on last position
99- m_sub_tokens .push_back (new MPSToken (this , text));
105+ m_subtokens .push_back (new MPSToken (this , text));
100106 } else {
101107 size_t idx = 0 ;
102- MPSTokensContainer::iterator iter = m_sub_tokens .begin ();
103- for ( ; (iter != m_sub_tokens .end ()) && (idx != pos); ++iter, ++idx);
104- if (iter == m_sub_tokens .end ())
105- m_sub_tokens .push_back (new MPSToken (this , text));
108+ MPSTokensContainer::iterator iter = m_subtokens .begin ();
109+ for ( ; (iter != m_subtokens .end ()) && (idx != pos); ++iter, ++idx);
110+ if (iter == m_subtokens .end ())
111+ m_subtokens .push_back (new MPSToken (this , text));
106112 else
107- m_sub_tokens .insert (iter, new MPSToken (this , text));
113+ m_subtokens .insert (iter, new MPSToken (this , text));
108114 }
109115}
110116
@@ -128,16 +134,16 @@ void MPSToken::set_discard (bool discard)
128134
129135void MPSToken::cleanup_token (MPSToken& token)
130136{
131- if (!token.m_sub_tokens .empty ()) {
137+ if (!token.m_subtokens .empty ()) {
132138 // we have some sub-tokens, need to delete them
133- for (MPSTokensContainer::iterator iter = token.m_sub_tokens .begin (); iter != token.m_sub_tokens .end (); ++iter) {
139+ for (MPSTokensContainer::iterator iter = token.m_subtokens .begin (); iter != token.m_subtokens .end (); ++iter) {
134140 if (*iter != 0 ) {
135141 cleanup_token (*(*iter));
136142 delete (*iter);
137143 *iter = 0 ;
138144 }
139145 }
140- token.m_sub_tokens .clear ();
146+ token.m_subtokens .clear ();
141147 }
142148}
143149
@@ -154,18 +160,18 @@ void MPSToken::discard_token (MPSToken& token, bool discard)
154160bool MPSToken::is_subtoken (const MPSToken* token) const
155161{
156162 MPSTokensContainer::const_iterator pos;
157- pos = std::find (m_sub_tokens .begin (), m_sub_tokens .end (), token);
158- return (pos != m_sub_tokens .end ());
163+ pos = std::find (m_subtokens .begin (), m_subtokens .end (), token);
164+ return (pos != m_subtokens .end ());
159165}
160166
161167const MPSToken* MPSToken::last_subtoken () const
162168{
163- if (m_sub_tokens .empty ())
169+ if (m_subtokens .empty ())
164170 return 0 ;
165171
166172 const MPSToken* last = 0 ;
167- MPSTokensContainer::const_iterator iter = m_sub_tokens .begin ();
168- for ( ; iter != m_sub_tokens .end ();) {
173+ MPSTokensContainer::const_iterator iter = m_subtokens .begin ();
174+ for ( ; iter != m_subtokens .end ();) {
169175 last = (*iter);
170176 ++iter;
171177 }
@@ -191,8 +197,8 @@ const MPSToken* MPSToken::find_last_leaf_subtoken (const MPSToken* token, bool i
191197 return token;
192198
193199 const MPSToken* last = 0 ;
194- MPSTokensContainer::const_iterator iter = token->sub_tokens_const_begin ();
195- for ( ; iter != token->sub_tokens_const_end (); ) {
200+ MPSTokensContainer::const_iterator iter = token->subtokens_const_begin ();
201+ for ( ; iter != token->subtokens_const_end (); ) {
196202
197203 // find only tokens which are not discarded
198204 bool discard_cond = (include_discarded) || (!include_discarded && !(*iter)->is_discard ());
@@ -203,7 +209,7 @@ const MPSToken* MPSToken::find_last_leaf_subtoken (const MPSToken* token, bool i
203209 ++iter;
204210
205211 // was it the last subttoken?
206- if (iter == token->sub_tokens_const_end ()) {
212+ if (iter == token->subtokens_const_end ()) {
207213 if (last != 0 && !last->sub_tokens_empty ()) {
208214 last = find_last_leaf_subtoken (last, include_discarded);
209215 }
@@ -223,8 +229,8 @@ const MPSToken* MPSToken::find_first_leaf_subtoken (const MPSToken* token, bool
223229 const MPSToken* first = 0 ;
224230
225231 // find first token which is not discarded
226- MPSTokensContainer::const_iterator iter = token->sub_tokens_const_begin ();
227- for ( ; iter != token->sub_tokens_const_end (); ++ iter) {
232+ MPSTokensContainer::const_iterator iter = token->subtokens_const_begin ();
233+ for ( ; iter != token->subtokens_const_end (); ++ iter) {
228234 bool discard_cond = (include_discarded) || (!include_discarded && !(*iter)->is_discard ());
229235 if (*iter && discard_cond) {
230236 first = *iter;
@@ -233,7 +239,7 @@ const MPSToken* MPSToken::find_first_leaf_subtoken (const MPSToken* token, bool
233239 }
234240
235241 // if it has subtokens, find first undiscarded token in that group
236- if (iter != token->sub_tokens_const_end () &&
242+ if (iter != token->subtokens_const_end () &&
237243 *iter &&
238244 !(*iter)->sub_tokens_empty ())
239245 {
0 commit comments