Add possibility to retrieve popularity data

Author: ahornace

src/org/opensolaris/opengrok/web/api/v1/controller/SuggesterController.java

@@ -46,21 +46,27 @@
 import javax.validation.constraints.Min;
 import javax.ws.rs.BeanParam;
 import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.AbstractMap.SimpleEntry;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
  * Endpoint for suggester related REST queries.
@@ -71,6 +77,8 @@ public final class SuggesterController {
 
     public static final String PATH = "suggest";
 
+    private static final int POPULARITY_DEFAULT_PAGE_SIZE = 100;
+
     private static final Logger logger = LoggerFactory.getLogger(SuggesterController.class);
 
     private final RuntimeEnvironment env = RuntimeEnvironment.getInstance();
@@ -245,6 +253,21 @@ public void addSearchCountsRaw(@Valid final List<TermIncrementData> termIncremen
         }
     }
 
+    @GET
+    @Path("/popularity/{project}")
+    @Localhost
+    @Produces(MediaType.APPLICATION_JSON)
+    public List<Entry<String, Integer>> getPopularityData(
+            @PathParam("project") final String project,
+            @QueryParam("field") @DefaultValue(QueryBuilder.FULL) final String field,
+            @QueryParam("page") @DefaultValue("" + 0) final int page,
+            @QueryParam("pageSize") @DefaultValue("" + POPULARITY_DEFAULT_PAGE_SIZE) final int pageSize
+    ) {
+        return suggester.getPopularityData(project, field, page, pageSize).stream()
+                .map(e -> new SimpleEntry<>(e.getKey().utf8ToString(), e.getValue()))
+                .collect(Collectors.toList());
+    }
+
     private static class Result {
 
         private long time;

src/org/opensolaris/opengrok/web/api/v1/suggester/provider/service/SuggesterService.java

@@ -24,11 +24,13 @@
 
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
 import org.opengrok.suggest.LookupResultItem;
 import org.opengrok.suggest.query.SuggesterQuery;
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Map.Entry;
 
 public interface SuggesterService {
 
@@ -75,6 +77,16 @@ public interface SuggesterService {
      */
     void increaseSearchCount(String project, Term term, int value);
 
+    /**
+     * Returns the searched terms sorted according to their popularity.
+     * @param project project for which to return the data
+     * @param field field for which to return the data
+     * @param page which page of data to retrieve
+     * @param pageSize number of results to return
+     * @return list of terms with their popularity
+     */
+    List<Entry<BytesRef, Integer>> getPopularityData(String project, String field, int page, int pageSize);
+
     /**
      * Closes the underlying service explicitly.
      */

src/org/opensolaris/opengrok/web/api/v1/suggester/provider/service/impl/SuggesterServiceImpl.java

@@ -29,6 +29,7 @@
 import com.cronutils.parser.CronParser;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
 import org.opengrok.suggest.LookupResultItem;
 import org.opengrok.suggest.Suggester;
 import org.opengrok.suggest.Suggester.NamedIndexReader;
@@ -52,6 +53,7 @@
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.Executors;
@@ -238,6 +240,26 @@ public void increaseSearchCount(final String project, final Term term, final int
         }
     }
 
+    /** {@inheritDoc} */
+    @Override
+    public List<Entry<BytesRef, Integer>> getPopularityData(
+            final String project,
+            final String field,
+            final int page,
+            final int pageSize
+    ) {
+        lock.readLock().lock();
+        try {
+            if (suggester == null) {
+                logger.log(Level.FINE, "Cannot retrieve popularity data because suggester is not initialized");
+                return Collections.emptyList();
+            }
+            return suggester.getSearchCounts(project, field, page, pageSize);
+        } finally {
+            lock.readLock().unlock();
+        }
+    }
+
     private void initSuggester() {
         Configuration config = env.getConfiguration();
 

suggester/src/main/java/org/opengrok/suggest/FieldWFSTCollection.java

@@ -52,6 +52,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -455,6 +456,28 @@ public void unlock() {
         lock.readLock().unlock();
     }
 
+    /**
+     * Returns the searched terms sorted according to their popularity.
+     * @param field field for which to return the data
+     * @param page which page of data to retrieve
+     * @param pageSize number of results to return
+     * @return list of terms with their popularity
+     */
+    public List<Entry<BytesRef, Integer>> getSearchCountsSorted(final String field, int page, int pageSize) {
+        lock.readLock().lock();
+        try {
+            PopularityMap map = searchCountMaps.get(field);
+            if (map == null) {
+                logger.log(Level.FINE, "No search count map initialized for field {0}", field);
+                return Collections.emptyList();
+            }
+
+            return map.getPopularityData(page, pageSize);
+        } finally {
+            lock.readLock().unlock();
+        }
+    }
+
     @Override
     public String toString() {
         return "FieldWFSTCollection{" +

suggester/src/main/java/org/opengrok/suggest/Suggester.java

@@ -28,6 +28,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
 import org.opengrok.suggest.query.SuggesterPrefixQuery;
 import org.opengrok.suggest.query.SuggesterQuery;
 
@@ -41,6 +42,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -384,6 +386,30 @@ public void increaseSearchCount(final String project, final Term term, final int
         data.incrementSearchCount(term, value);
     }
 
+    /**
+     * Returns the searched terms sorted according to their popularity.
+     * @param project project for which to return the data
+     * @param field field for which to return the data
+     * @param page which page of data to retrieve
+     * @param pageSize number of results to return
+     * @return list of terms with their popularity
+     */
+    public List<Entry<BytesRef, Integer>> getSearchCounts(
+            final String project,
+            final String field,
+            final int page,
+            final int pageSize
+    ) {
+        FieldWFSTCollection data = projectData.get(project);
+        if (data == null) {
+            logger.log(Level.FINE, "Cannot retrieve search counts because data for project {0} were not found",
+                    project);
+            return Collections.emptyList();
+        }
+
+        return data.getSearchCountsSorted(field, page, pageSize);
+    }
+
     /**
      * Closes opened resources.
      */

suggester/src/main/java/org/opengrok/suggest/popular/PopularityMap.java

@@ -24,6 +24,9 @@
 
 import org.apache.lucene.util.BytesRef;
 
+import java.util.List;
+import java.util.Map.Entry;
+
 /**
  * Abstraction for the map to store the data for most popular completion.
  */
@@ -36,4 +39,12 @@ public interface PopularityMap extends PopularityCounter, AutoCloseable {
      */
     void increment(BytesRef key, int value);
 
+    /**
+     * Returns the popularity data sorted according to their value.
+     * @param page which page of data to retrieve
+     * @param pageSize number of results to return
+     * @return popularity data sorted according to their value
+     */
+    List<Entry<BytesRef, Integer>> getPopularityData(int page, int pageSize);
+
 }

suggester/src/main/java/org/opengrok/suggest/popular/impl/chronicle/ChronicleMapAdapter.java

@@ -30,6 +30,10 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
 import java.util.function.Predicate;
 
 /**
@@ -67,6 +71,31 @@ public void increment(final BytesRef key, final int value) {
         map.merge(key, value, (a, b) -> a + b);
     }
 
+    /** {@inheritDoc} */
+    @Override
+    public List<Entry<BytesRef, Integer>> getPopularityData(final int page, final int pageSize) {
+        if (page < 0) {
+            throw new IllegalArgumentException("Cannot retrieve popularity data for negative page: " + page);
+        }
+        if (pageSize < 0) {
+            throw new IllegalArgumentException("Cannot retrieve negative number of results: " + pageSize);
+        }
+
+        List<Entry<BytesRef, Integer>> list = new ArrayList<>(map.entrySet());
+        list.sort(Entry.<BytesRef, Integer>comparingByValue().reversed());
+
+        int startIndex = page * pageSize;
+        if (startIndex >= list.size()) {
+            return Collections.emptyList();
+        }
+        int endIndex = startIndex + pageSize;
+        if (endIndex > list.size()) {
+            endIndex = list.size();
+        }
+
+        return list.subList(startIndex, endIndex);
+    }
+
     /**
      * Removes the entries with key that meets the predicate.
      * @param predicate predicate which tests which entries should be removed

suggester/src/test/java/org/opengrok/suggest/popular/impl/ChronicleMapAdapterTest.java

@@ -31,8 +31,13 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.AbstractMap.SimpleEntry;
+import java.util.List;
+import java.util.Map.Entry;
 
+import static org.hamcrest.collection.IsIterableContainingInOrder.contains;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
 
 public class ChronicleMapAdapterTest {
 
@@ -86,4 +91,36 @@ public void testResize() throws IOException {
         checkData(500, map);
     }
 
+    @Test
+    public void testGetPopularityData() {
+        Entry<BytesRef, Integer> e1 = new SimpleEntry<>(new BytesRef("test"), 1);
+        Entry<BytesRef, Integer> e2 = new SimpleEntry<>(new BytesRef("test2"), 2);
+
+        map.increment(e1.getKey(), e1.getValue());
+        map.increment(e2.getKey(), e2.getValue());
+
+        List<Entry<BytesRef, Integer>> data = map.getPopularityData(0, 10);
+
+        assertThat(data, contains(e2, e1));
+    }
+
+    @Test
+    public void testGetPopularityPaging() {
+        Entry<BytesRef, Integer> e1 = new SimpleEntry<>(new BytesRef("test"), 1);
+        Entry<BytesRef, Integer> e2 = new SimpleEntry<>(new BytesRef("test2"), 2);
+        Entry<BytesRef, Integer> e3 = new SimpleEntry<>(new BytesRef("test3"), 3);
+
+        map.increment(e1.getKey(), e1.getValue());
+        map.increment(e2.getKey(), e2.getValue());
+        map.increment(e3.getKey(), e3.getValue());
+
+        List<Entry<BytesRef, Integer>> data = map.getPopularityData(0, 2);
+
+        assertThat(data, contains(e3, e2));
+
+        data = map.getPopularityData(1, 2);
+
+        assertThat(data, contains(e1));
+    }
+
 }