Refactor the mwt misc key values - will want to keep the non-space ones separately. Check that the SpacesAfter is correctly processed

Author: AngledLuffa

itest/src/edu/stanford/nlp/pipeline/CoNLLUReaderITest.java

@@ -498,4 +498,33 @@ public void testReadingOneWord() throws ClassNotFoundException, IOException {
     assertEquals(1, graph.getRoots().size());
   }
 
+
+  public static final String mwtSpaceAfterPath = String.format("edu/stanford/nlp/pipeline/en-example-misc-spaceafter.conllu");
+
+  @Test
+  /**
+   * This test checks that the SpaceAfter on a Misc is respected.
+   */
+  public void testReadingMiscSpaceAfter() throws ClassNotFoundException, IOException {
+    Annotation readInDocument = new CoNLLUReader(new Properties()).readCoNLLUFile(mwtSpaceAfterPath).get(0);
+
+    // this document only has one sentence
+    List<CoreMap> sentences = readInDocument.get(CoreAnnotations.SentencesAnnotation.class);
+    assertEquals(1, sentences.size());
+
+    CoreMap sentence = sentences.get(0);
+
+    // cursory check of the tokens
+    List<CoreLabel> tokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
+    assertEquals(8, tokens.size());
+
+    // check that the non-last words of the MWT have no SpaceAfter
+    assertEquals("", tokens.get(0).after());
+    // check the SpaceAfter of the second word, which is where the MWT SpaceAfter should go
+    assertEquals("  ", tokens.get(1).after());
+
+    assertTrue(sentence.containsKey(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation.class));
+    assertTrue(sentence.containsKey(SemanticGraphCoreAnnotations.EnhancedDependenciesAnnotation.class));
+  }
+
 }

src/edu/stanford/nlp/pipeline/CoNLLUReader.java

@@ -437,9 +437,16 @@ public CoreLabel convertLineToCoreLabel(CoNLLUSentence sentence, String line) {
       cl.setIsMWT(false);
       cl.setIsMWTFirst(false);
     } else if (sentence.mwtData.containsKey(sentenceTokenIndex - 1)) {
+      String miscInfo = sentence.mwtMiscs.get(sentence.mwtData.get(sentenceTokenIndex - 1));
+      Map<String, String> mwtKeyValues = new HashMap<>();
+      if (miscInfo != null && !miscInfo.equals("_")) {
+        Arrays.stream(miscInfo.split("\\|")).forEach(
+          kv -> mwtKeyValues.put(kv.split("=", 2)[0], kv.split("=")[1]));
+      }
+
       // set MWT text
       cl.set(CoreAnnotations.MWTTokenTextAnnotation.class,
-          sentence.mwtTokens.get(sentence.mwtData.get(sentenceTokenIndex - 1)));
+             sentence.mwtTokens.get(sentence.mwtData.get(sentenceTokenIndex - 1)));
       cl.setIsMWT(true);
       // check if first
       if (sentence.mwtData.containsKey(sentenceTokenIndex - 2) &&
@@ -457,16 +464,8 @@ public CoreLabel convertLineToCoreLabel(CoNLLUSentence sentence, String line) {
         // then we aren't last, and SpaceAfter="" is implicitly true
         cl.setAfter("");
       } else {
-        String miscInfo = sentence.mwtMiscs.get(sentence.mwtData.get(sentenceTokenIndex - 1));
-        if (miscInfo != null && !miscInfo.equals("_")) {
-          Map<String, String> mwtKeyValues = new HashMap<>();
-          Arrays.stream(miscInfo.split("\\|")).forEach(
-            kv -> mwtKeyValues.put(kv.split("=", 2)[0], kv.split("=")[1]));
-          String spaceAfter = miscToSpaceAfter(mwtKeyValues);
-          cl.setAfter(spaceAfter);
-        } else {
-          cl.setAfter(" ");
-        }
+        String spaceAfter = miscToSpaceAfter(mwtKeyValues);
+        cl.setAfter(spaceAfter);
       }
     } else {
       cl.setIsMWT(false);