added support for phrases occurrences in text processor, some more refactoring

Author: alenegro81

src/main/java/com/graphaware/nlp/domain/Labels.java

@@ -23,5 +23,6 @@
 public enum Labels implements Label {
     AnnotatedText,
     Sentence,
-    Tag
+    Tag,
+    Phrase
 }

…aphaware/nlp/domain/CypherStatement.java …are/nlp/domain/PartOfTextOccurrence.javasrc/main/java/com/graphaware/nlp/domain/CypherStatement.java renamed to src/main/java/com/graphaware/nlp/domain/PartOfTextOccurrence.java src/main/java/com/graphaware/nlp/domain/CypherStatement.java renamed to src/main/java/com/graphaware/nlp/domain/PartOfTextOccurrence.java

@@ -15,34 +15,23 @@
  */
 package com.graphaware.nlp.domain;
 
-import java.util.Map;
+import com.graphaware.common.util.Pair;
 
-/**
- *
- * @author ale
- */
-public class CypherStatement {
-    private String query;
-    private Map<String, Object> params;
+class PartOfTextOccurrence<T> {
 
-    public CypherStatement(String query, Map<String, Object> params) {
-        this.query = query;
-        this.params = params;
-    }
+    private final T element;
+    private final Pair<Integer, Integer> span;
 
-    public String getQuery() {
-        return query;
+    public PartOfTextOccurrence(T element, int begin, int end) {
+        this.element = element;
+        this.span = new Pair<>(begin, end);
     }
 
-    public void setQuery(String query) {
-        this.query = query;
+    public T getElement() {
+        return element;
     }
 
-    public Map<String, Object> getParams() {
-        return params;
+    public Pair<Integer, Integer> getSpan() {
+        return span;
     }
-
-    public void setParams(Map<String, Object> params) {
-        this.params = params;
-    }    
 }

src/main/java/com/graphaware/nlp/domain/Phrase.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2013-2016 GraphAware
+ *
+ * This file is part of the GraphAware Framework.
+ *
+ * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms of
+ * the GNU General Public License as published by the Free Software Foundation, either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details. You should have received a copy of
+ * the GNU General Public License along with this program.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+package com.graphaware.nlp.domain;
+
+public class Phrase {
+    private final String content;
+
+    public Phrase(String content) {
+        this.content = content.trim();
+    }    
+
+    public String getContent() {
+        return content;
+    }
+    
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || !(o instanceof Phrase))
+            return false;
+        return this.content.equalsIgnoreCase(((Phrase)o).content);
+    }
+    
+}

src/main/java/com/graphaware/nlp/domain/Sentence.java

@@ -15,16 +15,17 @@
  */
 package com.graphaware.nlp.domain;
 
-import com.graphaware.common.util.Pair;
 import static com.graphaware.nlp.domain.SentimentLabels.*;
 import static com.graphaware.nlp.domain.Labels.Sentence;
 import static com.graphaware.nlp.domain.Properties.HASH;
 import static com.graphaware.nlp.domain.Properties.PROPERTY_ID;
 import static com.graphaware.nlp.domain.Properties.TEXT;
 import static com.graphaware.nlp.domain.Relationships.HAS_TAG;
 import static com.graphaware.nlp.util.HashFunctions.MD5;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Node;
@@ -33,11 +34,15 @@
 
 public class Sentence implements Persistable {
 
+    public static final int NO_SENTIMENT = -1;
+    
     private final Map<String, Tag> tags;
-    private Map<Integer, TagOccurrence> tagOccurrences;
+    private Map<Integer, PartOfTextOccurrence<Tag>> tagOccurrences;
+    private Map<Integer, Map<Integer, PartOfTextOccurrence<Phrase>>> phraseOccurrences;
 
     private final String sentence;
-    private int sentiment = -1;
+    private int sentiment = NO_SENTIMENT;
+    
     private boolean store = false;
     private String id;
 
@@ -76,29 +81,61 @@ public String getId() {
         return id;
     }
 
-    public void addOccurrence(int begin, int end, Tag tag) {
+    public void addTagOccurrence(int begin, int end, Tag tag) {
         if (begin < 0) {
             throw new RuntimeException("Begin cannot be negative (for tag: " + tag.getLemma() + ")" );
         }
         if (tagOccurrences == null) {
             tagOccurrences = new HashMap<>();
         }
         //Will update end if already exist
-        tagOccurrences.put(begin, new TagOccurrence(tag, begin, end));
+        tagOccurrences.put(begin, new PartOfTextOccurrence<>(tag, begin, end));
     }
 
     //Currently used only for testing purpose
     public Tag getTagOccurrence(int begin) {
         if (begin < 0) {
             throw new RuntimeException("Begin cannot be negative");
         }
-        TagOccurrence occurrence = tagOccurrences.get(begin);
+        PartOfTextOccurrence<Tag> occurrence = tagOccurrences.get(begin);
         if (occurrence != null) {
-            return occurrence.getTag();
+            return occurrence.getElement();
         } else {
           return null;  
         }
     }
+    
+    public void addPhraseOccurrence(int begin, int end, Phrase phrase) {
+        if (begin < 0) {
+            throw new RuntimeException("Begin cannot be negative (for phrase: " + phrase.getContent()+ ")" );
+        }
+        if (phraseOccurrences == null) {
+            phraseOccurrences = new HashMap<>();
+        }
+        if (!phraseOccurrences.containsKey(begin)) {
+            phraseOccurrences.put(begin, new HashMap<>());
+        }
+        //Will update end if already exist
+        phraseOccurrences.get(begin).put(end, new PartOfTextOccurrence<>(phrase, begin, end));
+    }
+
+    //Currently used only for testing purpose
+    public List<Phrase> getPhraseOccurrence(int begin) {
+        if (begin < 0) {
+            throw new RuntimeException("Begin cannot be negative");
+        }
+        Map<Integer, PartOfTextOccurrence<Phrase>> occurrence = phraseOccurrences.get(begin);
+        
+        if (occurrence != null) {
+            List<Phrase> result = new ArrayList<>();
+            occurrence.values().stream().forEach((item) -> {
+                result.add(item.getElement());
+            });
+            return result;
+        } else {
+          return new ArrayList<>();  
+        }
+    }
 
     @Override
     public Node storeOnGraph(GraphDatabaseService database) {
@@ -165,23 +202,4 @@ private Node checkIfExist(GraphDatabaseService database, Object id) {
         }
         return null;
     }
-
-    class TagOccurrence {
-
-        private final Tag tag;
-        private final Pair<Integer, Integer> span;
-
-        public TagOccurrence(Tag tag, int begin, int end) {
-            this.tag = tag;
-            this.span = new Pair<>(begin, end);
-        }
-
-        public Tag getTag() {
-            return tag;
-        }
-
-        public Pair<Integer, Integer> getSpan() {
-            return span;
-        }
-    }
 }

src/main/java/com/graphaware/nlp/processor/PipelineBuilder.java

@@ -0,0 +1,77 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package com.graphaware.nlp.processor;
+
+import edu.stanford.nlp.pipeline.StanfordCoreNLP;
+import java.util.Properties;
+
+public class PipelineBuilder {
+    private static final String CUSTOM_STOP_WORD_LIST = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,o,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with";
+
+    private final Properties properties = new Properties();
+    private final StringBuilder annotattors = new StringBuilder(); //basics annotators
+    private int threadsNumber = 4;
+
+    public PipelineBuilder tokenize() {
+        checkForExistingAnnotators();
+        annotattors.append("tokenize, ssplit, pos, lemma, ner");
+        return this;
+    }
+
+    private void checkForExistingAnnotators() {
+        if (annotattors.toString().length() > 0) {
+            annotattors.append(", ");
+        }
+    }
+
+    public PipelineBuilder extractSentiment() {
+        checkForExistingAnnotators();
+        annotattors.append("parse, sentiment");
+        return this;
+    }
+
+    public PipelineBuilder extractRelations() {
+        checkForExistingAnnotators();
+        annotattors.append("relation");
+        return this;
+    }
+
+    public PipelineBuilder extractCoref() {
+        checkForExistingAnnotators();
+        annotattors.append("mention, coref");
+        properties.setProperty("coref.doClustering", "true");
+        properties.setProperty("coref.md.type", "rule");
+        properties.setProperty("coref.mode", "statistical");
+        return this;
+    }
+
+    public PipelineBuilder defaultStopWordAnnotator() {
+        checkForExistingAnnotators();
+        annotattors.append("stopword");
+        properties.setProperty("customAnnotatorClass.stopword", "com.graphaware.nlp.processor.StopwordAnnotator");
+        properties.setProperty(StopwordAnnotator.STOPWORDS_LIST, CUSTOM_STOP_WORD_LIST);
+        return this;
+    }
+
+    public PipelineBuilder stopWordAnnotator(Properties properties) {
+        properties.entrySet().stream().forEach((entry) -> {
+            this.properties.setProperty((String) entry.getKey(), (String) entry.getValue());
+        });
+        return this;
+    }
+
+    public PipelineBuilder threadNumber(int threads) {
+        this.threadsNumber = threads;
+        return this;
+    }
+
+    public StanfordCoreNLP build() {
+        properties.setProperty("annotators", annotattors.toString());
+        properties.setProperty("threads", String.valueOf(threadsNumber));
+        StanfordCoreNLP pipeline = new StanfordCoreNLP(properties);
+        return pipeline;
+    }
+}