Oops, need to set the SentenceIndexAnnotation on the empty tokens

Author: AngledLuffa

itest/src/edu/stanford/nlp/pipeline/CoNLLUReaderITest.java

@@ -353,6 +353,7 @@ public void testReadingInEmpties() throws ClassNotFoundException, IOException {
     CoreLabel empty = emptyTokens.get(0);
     assertEquals(8, empty.index());
     assertEquals(Integer.valueOf(1), empty.get(CoreAnnotations.EmptyIndexAnnotation.class));
+    assertEquals(0, empty.sentIndex());
     assertEquals("reported", empty.value());
   }
 

src/edu/stanford/nlp/pipeline/CoNLLUReader.java

@@ -348,6 +348,11 @@ public Annotation convertCoNLLUDocumentToAnnotation(CoNLLUDocument doc) {
         tokens.add(token);
         documentIdx++;
       }
+      if (sentence.containsKey(CoreAnnotations.EmptyTokensAnnotation.class)) {
+        for (CoreLabel token : sentence.get(CoreAnnotations.EmptyTokensAnnotation.class)) {
+          token.set(CoreAnnotations.SentenceIndexAnnotation.class, sentenceIdx);
+        }
+      }
       sentenceIdx++;
     }
     // make sure to set docText AFTER all the above processing