Save the SpacesBefore on an MWT. Save the rest of the MWT misc annotations on the first word of the MWT. Test both operations

Author: AngledLuffa

data/edu/stanford/nlp/pipeline/en-example-misc-spaceafter.conllu

@@ -1,6 +1,6 @@
 # sent_id = reviews-231203-0002
 # text = I'm  a regular at the HH.
-1-2	I'm	_	_	_	_	_	_	_	SpacesAfter=\s\s
+1-2	I'm	_	_	_	_	_	_	_	Foo=Bar|SpacesBefore=\s\s|SpacesAfter=\s\s
 1	I	I	PRON	PRP	Case=Nom|Number=Sing|Person=1|PronType=Prs	4	nsubj	4:nsubj	_
 2	'm	be	AUX	VBP	Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin	4	cop	4:cop	_
 3	a	a	DET	DT	Definite=Ind|PronType=Art	4	det	4:det	_

itest/src/edu/stanford/nlp/pipeline/CoNLLUReaderITest.java

@@ -523,6 +523,13 @@ public void testReadingMiscSpaceAfter() throws ClassNotFoundException, IOExcepti
     // check the SpaceAfter of the second word, which is where the MWT SpaceAfter should go
     assertEquals("  ", tokens.get(1).after());
 
+    // the MWT SpacesBefore should go on the first word
+    assertEquals("  ", tokens.get(0).before());
+    assertEquals("", tokens.get(1).before());
+
+    assertEquals("Foo=Bar", tokens.get(0).get(CoreAnnotations.MWTTokenMiscAnnotation.class));
+    assertEquals(null, tokens.get(1).get(CoreAnnotations.MWTTokenMiscAnnotation.class));
+
     assertTrue(sentence.containsKey(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation.class));
     assertTrue(sentence.containsKey(SemanticGraphCoreAnnotations.EnhancedDependenciesAnnotation.class));
   }

src/edu/stanford/nlp/pipeline/CoNLLUReader.java

@@ -24,8 +24,6 @@ public class CoNLLUReader {
    * field constants
    **/
   // TODO: read sent_id?
-  // TODO: reconsider the newline as the after on the last word
-  // TODO: keep around the rest of the misc annotations
   public static final int CoNLLU_IndexField = 0;
   public static final int CoNLLU_WordField = 1;
   public static final int CoNLLU_LemmaField = 2;
@@ -369,6 +367,24 @@ public Annotation convertCoNLLUDocumentToAnnotation(CoNLLUDocument doc) {
     return finalAnnotation;
   }
 
+  public static final String rebuildMisc(Map<String, String> miscKeyValues) {
+    if (miscKeyValues.size() == 0) {
+      return null;
+    }
+
+    // rebuild the misc, since we have removed the SpaceAfter, SpacesAfter, and SpacesBefore
+    StringBuilder misc = new StringBuilder();
+    for (Map.Entry<String, String> entry : miscKeyValues.entrySet()) {
+      if (misc.length() > 0) {
+        misc.append("|");
+      }
+      misc.append(entry.getKey());
+      misc.append("=");
+      misc.append(entry.getValue());
+    }
+    return misc.toString();
+  }
+
   /**
    * Convert a single ten column CoNLLU line into a CoreLabel
    */
@@ -454,6 +470,12 @@ public CoreLabel convertLineToCoreLabel(CoNLLUSentence sentence, String line) {
         cl.setIsMWTFirst(false);
       } else {
         cl.setIsMWTFirst(true);
+
+        // if we are first, look for SpacesBefore
+        String mwtSpacesBefore = mwtKeyValues.get("SpacesBefore");
+        if (mwtSpacesBefore != null) {
+          cl.setBefore(unescapeSpacesAfter(mwtSpacesBefore));
+        }
       }
       // SpaceAfter / SpacesAfter should only apply to the last word in an MWT
       // all other words are treated as implicitly having SpaceAfter=No
@@ -467,6 +489,16 @@ public CoreLabel convertLineToCoreLabel(CoNLLUSentence sentence, String line) {
         String spaceAfter = miscToSpaceAfter(mwtKeyValues);
         cl.setAfter(spaceAfter);
       }
+      if (cl.isMWTFirst()) {
+        mwtKeyValues.remove("SpaceAfter");
+        mwtKeyValues.remove("SpacesAfter");
+        mwtKeyValues.remove("SpacesBefore");
+
+        String mwtMisc = rebuildMisc(mwtKeyValues);
+        if (mwtMisc != null) {
+          cl.set(CoreAnnotations.MWTTokenMiscAnnotation.class, mwtMisc);
+        }
+      }
     } else {
       cl.setIsMWT(false);
       cl.setIsMWTFirst(false);
@@ -476,18 +508,9 @@ public CoreLabel convertLineToCoreLabel(CoNLLUSentence sentence, String line) {
     }
     miscKeyValues.remove("SpaceAfter");
     miscKeyValues.remove("SpacesAfter");
-    if (miscKeyValues.size() > 0) {
-      // rebuild the misc, since we have removed the SpaceAfter, SpacesAfter, and SpacesBefore
-      StringBuilder misc = new StringBuilder();
-      for (Map.Entry<String, String> entry : miscKeyValues.entrySet()) {
-        if (misc.length() > 0) {
-          misc.append("|");
-        }
-        misc.append(entry.getKey());
-        misc.append("=");
-        misc.append(entry.getValue());
-      }
-      cl.set(CoreAnnotations.CoNLLUMisc.class, misc.toString());
+    String misc = rebuildMisc(miscKeyValues);
+    if (misc != null) {
+      cl.set(CoreAnnotations.CoNLLUMisc.class, misc);
     }
     return cl;
   }