adapt to newest ucto

Author: Ko van der Sloot

configure.ac

@@ -67,7 +67,7 @@ PKG_CHECK_MODULES([folia],[folia >= 1.4])
 CXXFLAGS="$CXXFLAGS $folia_CFLAGS"
 LIBS="$folia_LIBS $LIBS"
 
-PKG_CHECK_MODULES([ucto],[ucto >= 0.9.3])
+PKG_CHECK_MODULES([ucto],[ucto >= 0.15])
 CXXFLAGS="$CXXFLAGS $ucto_CFLAGS"
 LIBS="$ucto_LIBS $LIBS"
 

src/froggen.cxx

@@ -527,14 +527,14 @@ void create_lemmatizer( const Configuration& config,
 void check_data( Tokenizer::TokenizerClass *tokenizer,
 		 const multimap<UnicodeString,map<UnicodeString,map<UnicodeString,size_t>>>& data ){
   for ( const auto& word : data ){
-    int num = tokenizer->tokenizeLine( word.first );
-    if ( num != 1 ){
+    tokenizer->tokenizeLine( word.first );
+    vector<Tokenizer::Token> v = tokenizer->popSentence();
+    if ( v.size() != 1 ){
       cerr << "the provided tokenizer doesn't handle '" << word.first
-	   << "' well (splits it into " << num << " parts.)" << endl;
-      vector<string> v = tokenizer->getSentences();
+	   << "' well (splits it into " << v.size() << " parts.)" << endl;
       cerr << "[";
       for ( const auto& w : v ){
-	cerr << w << " ";
+	cerr << w.us << " ";
       }
       cerr << "]" << endl;
     }