Process SpacesAfter as well as SpaceAfter in the CoNLLUReader

Author: AngledLuffa

data/edu/stanford/nlp/pipeline/es-example.conllu

@@ -1,8 +1,9 @@
 # sent_id = 3LB-CAST-a22-0-s10
-# text = Pero la existencia de dos recién nacidos en la misma caja sólo podía deberse a un descuido de fábrica.
+# text = Pero la  existencia de dos recién nacidos en la misma caja sólo podía deberse a un descuido de fábrica.
 # orig_file_sentence 001#48
+# Note = This sentence is from the dev set, but we manually updated word 2 to have SpacesAfter=\s\s
 1	Pero	pero	CCONJ	cc	_	14	advmod	14:advmod	_
-2	la	el	DET	da0fs0	Definite=Def|Gender=Fem|Number=Sing|PronType=Art	3	det	3:det	_
+2	la	el	DET	da0fs0	Definite=Def|Gender=Fem|Number=Sing|PronType=Art	3	det	3:det	SpacesAfter=\s\s
 3	existencia	existencia	NOUN	ncfs000	Gender=Fem|Number=Sing	14	nsubj	14:nsubj	ArgTem=arg1:tem
 4	de	de	ADP	sps00	_	7	case	7:case	_
 5	dos	dos	NUM	dn0cp0	Number=Plur|NumForm=Word|NumType=Card	7	nummod	7:nummod	_

itest/src/edu/stanford/nlp/pipeline/CoNLLUReaderITest.java

@@ -22,15 +22,13 @@
  */
 public class CoNLLUReaderITest {
 
-  public String exampleDocument = "Pero la existencia de dos recién nacidos en la misma caja sólo podía deberse a un " +
-      "descuido de fábrica.\nDe allí las rebajas.\n";
   public String examplePath = String.format("edu/stanford/nlp/pipeline/es-example.conllu");
   public StanfordCoreNLP pipeline;
   public Annotation goldDocument;
   public Annotation readInDocument;
 
   static final String[] EXPECTED_SENTENCE_TEXT = {
-    "Pero la existencia de dos recién nacidos en la misma caja sólo podía deberse a un descuido de fábrica.",
+    "Pero la  existencia de dos recién nacidos en la misma caja sólo podía deberse a un descuido de fábrica.",
     "De allí las rebajas."
   };
   static final String EXPECTED_TEXT = String.join(System.lineSeparator(), EXPECTED_SENTENCE_TEXT) + System.lineSeparator();
@@ -189,7 +187,9 @@ public void testReadingInCoNLLUFile() throws ClassNotFoundException, IOException
       List<CoreLabel> tokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
       for (int j = 0; j < tokens.size(); ++j) {
         CoreLabel token = tokens.get(j);
-        if (j == tokens.size() - 1) {
+        if (i == 0 && j == 1) {
+          assertEquals("  ", token.after());
+        } else if (j == tokens.size() - 1) {
           assertEquals("\n", token.after());
         } else if (j == tokens.size() - 2) {
           assertEquals("", token.after());
@@ -199,7 +199,9 @@ public void testReadingInCoNLLUFile() throws ClassNotFoundException, IOException
           assertEquals(" ", token.after());
         }
 
-        if (i == 0 && j == 0) {
+        if (i == 0 && j == 2) {
+          assertEquals("  ", token.before());
+        } else if (i == 0 && j == 0) {
           assertEquals("", token.before());
         } else if (j == 0) {
           assertEquals("\n", token.before());

src/edu/stanford/nlp/pipeline/CoNLLUReader.java

@@ -108,6 +108,79 @@ public CoNLLUReader(Properties props) throws ClassNotFoundException {
     columnCount += extraColumns.size();
   }
 
+  // TODO: is there a better place for this?
+  public static String unescapeSpacesAfter(String escaped) {
+    int idx = 0;
+    StringBuilder unescaped = new StringBuilder();
+    while (idx < escaped.length()) {
+      if (escaped.charAt(idx) != '\\') {
+        unescaped.append(escaped.charAt(idx));
+        ++idx;
+        continue;
+      }
+      if (idx + 2 <= escaped.length()) {
+        String piece = escaped.substring(idx, idx + 2);
+        if (piece.equals("\\s")) {
+          unescaped.append(' ');
+          idx += 2;
+          continue;
+        } else if (piece.equals("\\t")) {
+          unescaped.append('\t');
+          idx += 2;
+          continue;
+        } else if (piece.equals("\\r")) {
+          unescaped.append('\r');
+          idx += 2;
+          continue;
+        } else if (piece.equals("\\n")) {
+          unescaped.append('\n');
+          idx += 2;
+          continue;
+        } else if (piece.equals("\\p")) {
+          unescaped.append('|');
+          idx += 2;
+          continue;
+        } else if (piece.equals("\\\\")) {
+          unescaped.append('\\');
+          idx += 2;
+          continue;
+        }
+      }
+      if (idx + 6 <= escaped.length()) {
+        String piece = escaped.substring(idx, idx + 6);
+        if (piece.equals("\\u00A0")) {
+          unescaped.append(' ');
+          idx += 6;
+          continue;
+        }
+      }
+      unescaped.append(escaped.charAt(idx));
+      ++idx;
+    }
+    return unescaped.toString();
+  }
+
+  public static String miscToSpaceAfter(Map<String, String> miscKeyValues) {
+    String spaceAfter = miscKeyValues.get("SpaceAfter");
+    if (spaceAfter != null) {
+      if (spaceAfter.equals("No") || spaceAfter.equals("no")) {
+        return "";
+      } else if (spaceAfter.equals("No~")) {
+        // a random data bug in UD 2.11 Russian-Taiga
+        return "";
+      } else {
+        return " ";
+      }
+    }
+
+    String spacesAfter = miscKeyValues.get("SpacesAfter");
+    if (spacesAfter != null) {
+      return unescapeSpacesAfter(spacesAfter);
+    }
+
+    return " ";
+  }
+
   /**
    * class to store info for a CoNLL-U document
    **/
@@ -345,15 +418,11 @@ public CoreMap convertCoNLLUSentenceToCoreMap(CoNLLUDocument doc, CoNLLUSentence
        *
        */
       if (!fields.get(CoNLLU_MiscField).equals("_")) {
-        HashMap<String, String> miscKeyValues = new HashMap<>();
+        Map<String, String> miscKeyValues = new HashMap<>();
         Arrays.stream(fields.get(CoNLLU_MiscField).split("\\|")).forEach(
-            kv -> miscKeyValues.put(kv.split("=")[0], kv.split("=")[1]));
-        // unless SpaceAfter=No, add a space after this token
-        if (!miscKeyValues.getOrDefault("SpaceAfter", "Yes").equals("No")) {
-          cl.setAfter(" ");
-        } else {
-          cl.setAfter("");
-        }
+          kv -> miscKeyValues.put(kv.split("=", 2)[0], kv.split("=")[1]));
+        String spaceAfter = miscToSpaceAfter(miscKeyValues);
+        cl.setAfter(spaceAfter);
       } else {
         cl.setAfter(" ");
       }
@@ -372,11 +441,17 @@ public CoreMap convertCoNLLUSentenceToCoreMap(CoNLLUDocument doc, CoNLLUSentence
           cl.setIsMWTFirst(true);
         }
         // handle MISC info
+        // TODO: only do SpaceAfter/SpacesAfter for the last one
+        // other MWT words should have after==""
         String miscInfo = sentence.mwtMiscs.get(sentence.mwtData.get(sentenceTokenIndex - 1));
-        for (String miscKV : miscInfo.split("\\|")) {
-          if (miscKV.startsWith("SpaceAfter")) {
-            cl.setAfter(miscKV.split("=")[1].equals("No") ? "" : " ");
-          }
+        if (miscInfo != null && !miscInfo.equals("_")) {
+          Map<String, String> miscKeyValues = new HashMap<>();
+          Arrays.stream(miscInfo.split("\\|")).forEach(
+            kv -> miscKeyValues.put(kv.split("=", 2)[0], kv.split("=")[1]));
+          String spaceAfter = miscToSpaceAfter(miscKeyValues);
+          cl.setAfter(spaceAfter);
+        } else {
+          cl.setAfter(" ");
         }
       } else {
         cl.setIsMWT(false);