Add 'uniq' as a keyword to SemgrexParser.jj

Add a UniqPattern which removes duplicates based on the node names given (using the values of those nodes)

Add some unit testing of the uniq search parsing functionality -
SemgrexParser should fail if a requested node is not in the pattern.
Also, uniq should be usable as a node name

Test a couple varieties of this operation

To allow uniq for a ProcessSemgrexRequest, need to decode all sentences from the request first, then turn that into a response.
Flip the order of matching in ProcessSemgrexRequest so that for each pattern, it matches all of the sentences at once.  Allows for operations on the complete batch of matches, such as the new uniq operator

We also refactor the ProcessSemgrexRequest and make the CoreNLPServer use the refactored method as well

Add a test of uniq for ProcessSemgrexRequest as well (it should only produce one result now for two graphs, not two)

Author: AngledLuffa

src/edu/stanford/nlp/pipeline/StanfordCoreNLPServer.java

@@ -1348,18 +1348,12 @@ public void handle(HttpExchange httpExchange) throws IOException {
               return Pair.makePair("".getBytes(), null);
             }
 
-            CoreNLPProtos.SemgrexResponse.Builder responseBuilder = CoreNLPProtos.SemgrexResponse.newBuilder();
-            int sentenceIdx = 0;
-            for (CoreMap sentence : doc.get(CoreAnnotations.SentencesAnnotation.class)) {
-              SemanticGraph graph = sentence.get(dependenciesType.annotation());
-              CoreNLPProtos.SemgrexResponse.GraphResult.Builder graphResultBuilder = CoreNLPProtos.SemgrexResponse.GraphResult.newBuilder();
-              graphResultBuilder.addResult(ProcessSemgrexRequest.matchSentence(regex, graph, 0, sentenceIdx));
-              responseBuilder.addResult(graphResultBuilder.build());
-              ++sentenceIdx;
-            }
+            List<CoreMap> sentences = doc.get(CoreAnnotations.SentencesAnnotation.class);
+            List<SemgrexPattern> patterns = Collections.singletonList(regex);
+            CoreNLPProtos.SemgrexResponse semgrexResponse = ProcessSemgrexRequest.processRequest(sentences, patterns);
 
             ByteArrayOutputStream os = new ByteArrayOutputStream();
-            responseBuilder.build().writeTo(os);
+            semgrexResponse.writeTo(os);
             os.close();
 
             return Pair.makePair(os.toByteArray(), doc);

src/edu/stanford/nlp/semgraph/semgrex/ProcessSemgrexRequest.java

@@ -10,26 +10,31 @@
 import java.io.InputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 
+import edu.stanford.nlp.ling.CoreAnnotations;
 import edu.stanford.nlp.ling.CoreLabel;
 import edu.stanford.nlp.pipeline.ProtobufAnnotationSerializer;
 import edu.stanford.nlp.pipeline.CoreNLPProtos;
 import edu.stanford.nlp.semgraph.SemanticGraph;
+import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations;
 import edu.stanford.nlp.semgraph.SemanticGraphEdge;
 import edu.stanford.nlp.semgraph.semgrex.SemgrexMatcher;
 import edu.stanford.nlp.semgraph.semgrex.SemgrexPattern;
+import edu.stanford.nlp.util.ArrayCoreMap;
+import edu.stanford.nlp.util.CoreMap;
+import edu.stanford.nlp.util.Pair;
 import edu.stanford.nlp.util.ProcessProtobufRequest;
 
 public class ProcessSemgrexRequest extends ProcessProtobufRequest {
   /**
    * Builds a single inner SemgrexResult structure from the pair of a SemgrexPattern and a SemanticGraph
    */
-  public static CoreNLPProtos.SemgrexResponse.SemgrexResult matchSentence(SemgrexPattern pattern, SemanticGraph graph, int patternIdx, int graphIdx) {
+  public static CoreNLPProtos.SemgrexResponse.SemgrexResult matchSentence(SemgrexPattern pattern, SemanticGraph graph, List<SemgrexMatch> matches, int patternIdx, int graphIdx) {
     CoreNLPProtos.SemgrexResponse.SemgrexResult.Builder semgrexResultBuilder = CoreNLPProtos.SemgrexResponse.SemgrexResult.newBuilder();
-    SemgrexMatcher matcher = pattern.matcher(graph);
-    while (matcher.find()) {
+    for (SemgrexMatch matcher : matches) {
       CoreNLPProtos.SemgrexResponse.Match.Builder matchBuilder = CoreNLPProtos.SemgrexResponse.Match.newBuilder();
       matchBuilder.setMatchIndex(matcher.getMatch().index());
       matchBuilder.setSemgrexIndex(patternIdx);
@@ -74,37 +79,63 @@ public static CoreNLPProtos.SemgrexResponse.SemgrexResult matchSentence(SemgrexP
     return semgrexResultBuilder.build();
   }
 
+  public static CoreNLPProtos.SemgrexResponse processRequest(List<CoreMap> sentences, List<SemgrexPattern> patterns) {
+    CoreNLPProtos.SemgrexResponse.Builder responseBuilder = CoreNLPProtos.SemgrexResponse.newBuilder();
+    List<Pair<CoreMap, List<Pair<SemgrexPattern, List<SemgrexMatch>>>>> allMatches = new ArrayList<>();
+    for (CoreMap sentence : sentences) {
+      allMatches.add(new Pair<>(sentence, new ArrayList<>()));
+    }
+    for (SemgrexPattern pattern : patterns) {
+      List<Pair<CoreMap, List<SemgrexMatch>>> patternMatches = pattern.matchSentences(sentences, true);
+      for (int i = 0; i < sentences.size(); ++i) {
+        Pair<CoreMap, List<SemgrexMatch>> sentenceMatches = patternMatches.get(i);
+        allMatches.get(i).second().add(new Pair<>(pattern, sentenceMatches.second()));
+      }
+    }
+
+    int graphIdx = 0;
+    for (Pair<CoreMap, List<Pair<SemgrexPattern, List<SemgrexMatch>>>> sentenceMatches : allMatches) {
+      CoreNLPProtos.SemgrexResponse.GraphResult.Builder graphResultBuilder = CoreNLPProtos.SemgrexResponse.GraphResult.newBuilder();
+
+      int patternIdx = 0;
+      SemanticGraph graph = sentenceMatches.first().get(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation.class);
+      for (Pair<SemgrexPattern, List<SemgrexMatch>> patternMatches : sentenceMatches.second()) {
+        SemgrexPattern pattern = patternMatches.first();
+        graphResultBuilder.addResult(matchSentence(pattern, graph, patternMatches.second(), patternIdx, graphIdx));
+        ++patternIdx;
+      }
+
+      responseBuilder.addResult(graphResultBuilder.build());
+      ++graphIdx;
+    }
+    return responseBuilder.build();
+  }
+
   /**
    * For a single request, iterate through the SemanticGraphs it
    * includes, and add the results of each Semgrex operation included
    * in the request.
    */
   public static CoreNLPProtos.SemgrexResponse processRequest(CoreNLPProtos.SemgrexRequest request) {
     ProtobufAnnotationSerializer serializer = new ProtobufAnnotationSerializer();
-    CoreNLPProtos.SemgrexResponse.Builder responseBuilder = CoreNLPProtos.SemgrexResponse.newBuilder();
 
-    List<SemgrexPattern> patterns = request.getSemgrexList().stream().map(SemgrexPattern::compile).collect(Collectors.toList());
-    int graphIdx = 0;
+    List<CoreMap> sentences = new ArrayList<>();
     for (CoreNLPProtos.SemgrexRequest.Dependencies sentence : request.getQueryList()) {
-      CoreNLPProtos.SemgrexResponse.GraphResult.Builder graphResultBuilder = CoreNLPProtos.SemgrexResponse.GraphResult.newBuilder();
-
       final List<CoreLabel> tokens;
       if (sentence.getGraph().getTokenList().size() > 0) {
         tokens = sentence.getGraph().getTokenList().stream().map(serializer::fromProto).collect(Collectors.toList());
       } else {
         tokens = sentence.getTokenList().stream().map(serializer::fromProto).collect(Collectors.toList());
       }
       SemanticGraph graph = ProtobufAnnotationSerializer.fromProto(sentence.getGraph(), tokens, "semgrex");
-      int patternIdx = 0;
-      for (SemgrexPattern pattern : patterns) {
-        graphResultBuilder.addResult(matchSentence(pattern, graph, patternIdx, graphIdx));
-        ++patternIdx;
-      }
-
-      responseBuilder.addResult(graphResultBuilder.build());
-      ++graphIdx;
+      CoreMap coremap = new ArrayCoreMap();
+      coremap.set(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation.class, graph);
+      coremap.set(CoreAnnotations.TokensAnnotation.class, tokens);
+      sentences.add(coremap);
     }
-    return responseBuilder.build();
+
+    List<SemgrexPattern> patterns = request.getSemgrexList().stream().map(SemgrexPattern::compile).collect(Collectors.toList());
+    return processRequest(sentences, patterns);
   }
 
   /**