use selected fields for csv export

limit number of messages to 10000 to avoid running out of heap.

fixes #766

Author: kroepke

app/controllers/SearchController.java

@@ -131,7 +131,7 @@ protected Set<String> getSelectedFields(String fields) {
         return selectedFields;
     }
 
-    public Result exportAsCsv(String q, String filter, String rangeType, int relative, String from, String to, String keyword) {
+    public Result exportAsCsv(String q, String filter, String rangeType, int relative, String from, String to, String keyword, String fields) {
         UniversalSearch search;
         try {
             search = getSearch(q, filter.isEmpty() ? null : filter, rangeType, relative, from, to, keyword, 0, UniversalSearch.DEFAULT_SORT);
@@ -143,7 +143,8 @@ public Result exportAsCsv(String q, String filter, String rangeType, int relativ
 
         final String s;
         try {
-            s = search.searchAsCsv();
+            Set<String> selectedFields = getSelectedFields(fields);
+            s = search.searchAsCsv(selectedFields);
         } catch (IOException e) {
             return status(504, views.html.errors.error.render(ApiClient.ERROR_MSG_IO, e, request()));
         } catch (APIException e) {

app/controllers/StreamSearchController.java

@@ -102,7 +102,7 @@ public Result index(String streamId,
     }
 
     @Override
-    public Result exportAsCsv(String q, String streamId, String rangeType, int relative, String from, String to, String keyword) {
-        return super.exportAsCsv(q, "streams:"+streamId, rangeType, relative, from, to, keyword);
+    public Result exportAsCsv(String q, String streamId, String rangeType, int relative, String from, String to, String keyword, String fields) {
+        return super.exportAsCsv(q, "streams:"+streamId, rangeType, relative, from, to, keyword, fields);
     }
 }

app/models/UniversalSearch.java

@@ -18,12 +18,14 @@
  */
 package models;
 
+import com.google.common.base.Joiner;
 import com.google.common.net.MediaType;
 import com.google.inject.assistedinject.Assisted;
 import com.google.inject.assistedinject.AssistedInject;
 import controllers.routes;
 import lib.APIException;
 import lib.ApiClient;
+import lib.ApiRequestBuilder;
 import lib.Tools;
 import lib.timeranges.TimeRange;
 import models.api.responses.*;
@@ -36,6 +38,7 @@
 
 import javax.annotation.Nullable;
 import java.io.IOException;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static lib.Configuration.apiTimeout;
@@ -102,23 +105,28 @@ private UniversalSearch(ApiClient api, FieldMapper fieldMapper, @Assisted TimeRa
         }
     }
 
-    private <T> T doSearch(Class<T> clazz, MediaType mediaType, int pageSize) throws APIException, IOException {
-        return api.get(clazz)
+    private <T> T doSearch(Class<T> clazz, MediaType mediaType, int pageSize, Set<String> selectedFields) throws APIException, IOException {
+        final ApiRequestBuilder<T> builder = api.get(clazz)
                 .path("/search/universal/{0}", timeRange.getType().toString().toLowerCase())
                 .queryParams(timeRange.getQueryParams())
                 .queryParam("query", query)
                 .queryParam("limit", pageSize)
                 .queryParam("offset", page * pageSize)
                 .queryParam("filter", (filter == null ? "*" : filter))
-                .queryParam("sort", order.toApiParam())
+                .queryParam("sort", order.toApiParam());
+        if (selectedFields != null && !selectedFields.isEmpty()) {
+            builder.queryParam("fields", Joiner.on(',').skipNulls().join(selectedFields));
+        }
+        final T result = builder
                 .accept(mediaType)
                 .timeout(KEITH, TimeUnit.SECONDS)
                 .expect(200, 400)
                 .execute();
+        return result;
     }
 
     public SearchResult search() throws IOException, APIException {
-        SearchResultResponse response = doSearch(SearchResultResponse.class, MediaType.JSON_UTF_8, PER_PAGE);
+        SearchResultResponse response = doSearch(SearchResultResponse.class, MediaType.JSON_UTF_8, PER_PAGE, null);
         if (response == null) {
             log.error("We should never get an empty result without throwing an IOException.");
             throw new APIException(null, null, new RuntimeException("Empty search response, this is likely a bug in exception handling."));
@@ -140,8 +148,8 @@ public SearchResult search() throws IOException, APIException {
         return result;
     }
 
-    public String searchAsCsv() throws IOException, APIException {
-        return doSearch(String.class, MediaType.CSV_UTF_8, 100000);  // TODO fix huge page size by using scroll searches and streaming results
+    public String searchAsCsv(Set<String> selectedFields) throws IOException, APIException {
+        return doSearch(String.class, MediaType.CSV_UTF_8, 10_000, selectedFields);  // TODO make use of streaming support in the server.
     }
 
     public DateHistogramResult dateHistogram(String interval) throws IOException, APIException {
@@ -243,6 +251,7 @@ public Call getCsvRoute(Request request, Stream stream) {
         String from = Tools.stringSearchParamOrEmpty(request, "from");
         String to = Tools.stringSearchParamOrEmpty(request, "to");
         String keyword = Tools.stringSearchParamOrEmpty(request, "keyword");
+        String fields = Tools.stringSearchParamOrEmpty(request, "fields");
         if (stream == null) {
             return routes.SearchController.exportAsCsv(
                     query,
@@ -251,7 +260,8 @@ public Call getCsvRoute(Request request, Stream stream) {
                     relative,
                     from,
                     to,
-                    keyword
+                    keyword,
+                    fields
             );
         } else {
             return routes.StreamSearchController.exportAsCsv(
@@ -261,7 +271,8 @@ public Call getCsvRoute(Request request, Stream stream) {
                     relative,
                     from,
                     to,
-                    keyword
+                    keyword,
+                    fields
             );
         }
     }

app/views/search/results.scala.html

@@ -61,7 +61,7 @@
             <span class="caret"></span>
         </a>
         <ul class="dropdown-menu">
-            <li><a href="@search.getCsvRoute(request(), stream)">Download as CSV</a></li>
+            <li><a class="search-view-state" href="@search.getCsvRoute(request(), stream)">Download as CSV</a></li>
         </ul>
     </div>
 

conf/routes

@@ -18,7 +18,7 @@ GET         /startpage
 
 # Search
 GET         /search                                                                  @controllers.SearchController.index(q ?= "", rangetype ?= "", relative:Integer ?= -1, from ?= "", to ?= "", keyword ?= "", interval ?= "", page:Integer ?= 1, saved:String ?= "", sortField:String ?= "", sortOrder:String ?= "", fields:String ?= "")
-GET         /search/csv                                                              @controllers.SearchController.exportAsCsv(q ?= "", filter ?= "", rangetype ?= "", relative:Integer ?= -1, from ?= "", to ?= "", keyword ?= "")
+GET         /search/csv                                                              @controllers.SearchController.exportAsCsv(q ?= "", filter ?= "", rangetype ?= "", relative:Integer ?= -1, from ?= "", to ?= "", keyword ?= "", fields:String ?= "")
 
 # Messages
 GET         /messages/:index/:id                                                     @controllers.MessagesController.show(index: String, id: String)
@@ -40,7 +40,7 @@ GET         /streams/:stream_id/edit
 POST        /streams/:stream_id/update                                               @controllers.StreamsController.update(stream_id: String)
 GET         /streams/:stream_id/clone                                                @controllers.StreamsController.cloneStreamForm(stream_id: String)
 POST        /streams/:stream_id/clone                                                @controllers.StreamsController.cloneStream(stream_id: String)
-GET         /streams/:stream_id/csv                                                  @controllers.StreamSearchController.exportAsCsv(q ?= "", stream_id: String, rangetype ?= "", relative:Integer ?= -1, from ?= "", to ?= "", keyword ?= "")
+GET         /streams/:stream_id/csv                                                  @controllers.StreamSearchController.exportAsCsv(q ?= "", stream_id: String, rangetype ?= "", relative:Integer ?= -1, from ?= "", to ?= "", keyword ?= "", fields:String ?="")
 
 # Alerts
 GET         /streams/:stream_id/alerts                                               @controllers.AlertsController.index(stream_id: String)