Process features in the CoNLLUReader. Also need to process xpos, keep the misc, and possibly build an extra semantic graph

The Reader is now reading in POS features, but the Spanish pipeline doesn't featurize, so we need to update the test to ignore those features. (Better would be to test the features!)

Author: AngledLuffa

itest/src/edu/stanford/nlp/pipeline/CoNLLUReaderITest.java

@@ -42,6 +42,9 @@ public void testReadingInCoNLLUFile() throws ClassNotFoundException, IOException
     for (CoreLabel token : goldDocument.get(CoreAnnotations.TokensAnnotation.class)) {
       token.remove(CoreAnnotations.ParentAnnotation.class);
     }
+    for (CoreLabel token : readInDocument.get(CoreAnnotations.TokensAnnotation.class)) {
+      token.remove(CoreAnnotations.CoNLLUFeats.class);
+    }
     // compare gold vs. read in
     // compare document text
     assertEquals(goldDocument.get(CoreAnnotations.TextAnnotation.class),

src/edu/stanford/nlp/pipeline/CoNLLUReader.java

@@ -4,6 +4,7 @@
 import edu.stanford.nlp.ling.*;
 import edu.stanford.nlp.semgraph.*;
 import edu.stanford.nlp.trees.*;
+import edu.stanford.nlp.trees.ud.CoNLLUFeatures;
 import edu.stanford.nlp.util.*;
 
 import java.io.*;
@@ -29,6 +30,7 @@ public class CoNLLUReader {
   public static final int CoNLLU_LemmaField = 2;
   public static final int CoNLLU_UPOSField = 3;
   public static final int CoNLLU_XPOSField = 4;
+  public static final int CoNLLU_FeaturesField = 5;
   public static final int CoNLLU_GovField = 6;
   public static final int CoNLLU_RelnField = 7;
   public static final int CoNLLU_MiscField = 9;
@@ -304,10 +306,21 @@ public CoreMap convertCoNLLUSentenceToCoreMap(CoNLLUDocument doc, CoNLLUSentence
       cl.setValue(fields.get(CoNLLU_WordField));
       cl.setOriginalText(fields.get(CoNLLU_WordField));
       cl.setIsNewline(false);
+
       if (!fields.get(CoNLLU_LemmaField).equals("_"))
         cl.setLemma(fields.get(CoNLLU_LemmaField));
+
       if (!fields.get(CoNLLU_UPOSField).equals("_"))
         cl.setTag(fields.get(CoNLLU_UPOSField));
+
+      //final String xpos = fields.get(CoNLLU_XPOSField);
+      //if (!xpos.equals("_"))
+      //  cl.setTag(xpos);
+
+      if (!fields.get(CoNLLU_FeaturesField).equals("_")) {
+        CoNLLUFeatures features = new CoNLLUFeatures(fields.get(CoNLLU_FeaturesField));
+        cl.set(CoreAnnotations.CoNLLUFeats.class, features);
+      }
       for (int extraColumnIdx = 10; extraColumnIdx < columnCount && extraColumnIdx < fields.size();
            extraColumnIdx++) {
         cl.set(extraColumns.get(extraColumnIdx), fields.get(extraColumnIdx));