importing exported extractors. #726

Lennart Koopmann · Lennart Koopmann · commit a807ffba1cc8 · 2014-05-04T19:56:08.000+02:00
diff --git a/app/controllers/ExtractorsController.java b/app/controllers/ExtractorsController.java
@@ -21,15 +21,20 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.google.inject.Inject;
 import lib.APIException;
 import lib.ApiClient;
 import lib.BreadcrumbList;
+import lib.Version;
 import models.*;
+import models.api.requests.ExtractorImportRequest;
+import models.api.requests.ExtractorListImportRequest;
 import play.Logger;
 import play.mvc.Result;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
@@ -166,23 +171,26 @@ public Result delete(String nodeId, String inputId, String extractorId) {
         }
     }
 
-    public Result export(String nodeId, String inputId) {
+    public Result exportExtractors(String nodeId, String inputId) {
         try {
             Node node = nodeService.loadNode(nodeId);
             Input input = node.getInput(inputId);
 
             BreadcrumbList bc = standardBreadcrumbs(node, input);
-            bc.addCrumb("Export", routes.ExtractorsController.export(nodeId, inputId));
+            bc.addCrumb("Export", routes.ExtractorsController.exportExtractors(nodeId, inputId));
 
-            List<Map<String, Object>> exports = Lists.newArrayList();
+            Map<String, Object> result = Maps.newHashMap();
+            List<Map<String, Object>> extractors = Lists.newArrayList();
             for (Extractor extractor : extractorService.all(node, input)) {
-                exports.add(extractor.export());
+                extractors.add(extractor.export());
             }
+            result.put("extractors", extractors);
+            result.put("version", Version.VERSION.toString());
 
             String extractorExport = "[]";
             try {
                 ObjectMapper om = new ObjectMapper();
-                extractorExport = om.writeValueAsString(exports);
+                extractorExport = om.writeValueAsString(result);
             } catch(JsonProcessingException e) {
                 Logger.error("Could not generate extractor export.", e);
             }
@@ -204,6 +212,88 @@ public Result export(String nodeId, String inputId) {
         }
     }
 
+    public Result importExtractorsPage(String nodeId, String inputId) {
+        try {
+            Node node = nodeService.loadNode(nodeId);
+            Input input = node.getInput(inputId);
+
+            BreadcrumbList bc = standardBreadcrumbs(node, input);
+            bc.addCrumb("Import", routes.ExtractorsController.importExtractorsPage(nodeId, inputId));
+
+            return ok(views.html.system.inputs.extractors.importPage.render(
+                    currentUser(),
+                    bc,
+                    node,
+                    input
+            ));
+        } catch (IOException e) {
+            return status(500, views.html.errors.error.render(ApiClient.ERROR_MSG_IO, e, request()));
+        } catch (APIException e) {
+            String message = "Could not fetch system information. We expected HTTP 200, but got a HTTP " + e.getHttpCode() + ".";
+            return status(500, views.html.errors.error.render(message, e, request()));
+        } catch (NodeService.NodeNotFoundException e) {
+            return status(404, views.html.errors.error.render(ApiClient.ERROR_MSG_NODE_NOT_FOUND, e, request()));
+        }
+    }
+
+    public Result importExtractors(String nodeId, String inputId) {
+        Map<String, String> form = flattenFormUrlEncoded(request().body().asFormUrlEncoded());
+
+        if(!form.containsKey("extractors") || form.get("extractors").isEmpty()) {
+            flash("error", "No JSON provided. Please fill out the import definition field.");
+            return redirect(routes.ExtractorsController.importExtractorsPage(nodeId, inputId));
+        }
+
+        ExtractorListImportRequest elir;
+        try {
+            ObjectMapper om = new ObjectMapper();
+            elir = om.readValue(form.get("extractors"), ExtractorListImportRequest.class);
+        } catch(Exception e) {
+            Logger.error("Could not read JSON.", e);
+            flash("error", "Could not read JSON.");
+            return redirect(routes.ExtractorsController.importExtractorsPage(nodeId, inputId));
+        }
+
+        /*
+         * For future versions with breaking changes: check the "version" field in the ExtractorListImportRequest.
+         *
+         * Thank me later.
+         */
+
+        int successes = 0;
+        for (ExtractorImportRequest importRequest : elir.extractors) {
+            try {
+                Node node = nodeService.loadNode(nodeId);
+
+                Extractor.Type type = Extractor.Type.valueOf(importRequest.extractorType.toUpperCase());
+
+                Extractor extractor = extractorFactory.forCreate(
+                        Extractor.CursorStrategy.valueOf(importRequest.cursorStrategy.toUpperCase()),
+                        importRequest.title,
+                        importRequest.sourceField,
+                        importRequest.targetField,
+                        type,
+                        currentUser(),
+                        Extractor.ConditionType.valueOf(importRequest.conditionType.toUpperCase()),
+                        importRequest.conditionValue
+                );
+
+                extractor.loadConfigFromImport(type, importRequest.extractorConfig);
+                extractor.loadConvertersFromImport(importRequest.converters);
+                extractor.setOrder(importRequest.order);
+                extractor.create(node, node.getInput(inputId));
+            } catch (Exception e) {
+                Logger.error("Could not import extractor. Continuing.", e);
+                continue;
+            }
+
+            successes++;
+        }
+
+        flash("success", "Successfully imported " + successes + " of " + elir.extractors.size() + " extractors.");
+        return redirect(routes.ExtractorsController.manage(nodeId, inputId));
+    }
+
     private static BreadcrumbList standardBreadcrumbs(Node node, Input input) {
         BreadcrumbList bc = new BreadcrumbList();
         bc.addCrumb("System", routes.SystemController.index(0));
diff --git a/app/models/Extractor.java b/app/models/Extractor.java
@@ -24,6 +24,7 @@
 import com.google.inject.assistedinject.AssistedInject;
 import lib.APIException;
 import lib.ApiClient;
+import lib.Version;
 import models.api.requests.CreateExtractorRequest;
 import models.api.responses.system.ExtractorSummaryResponse;
 import org.slf4j.Logger;
@@ -96,7 +97,6 @@ public static ConditionType fromString(String name) {
 
     private String id;
     private final String title;
-    private final int order;
     private final CursorStrategy cursorStrategy;
     private final Type extractorType;
     private final String sourceField;
@@ -110,6 +110,8 @@ public static ConditionType fromString(String name) {
     private final long exceptions;
     private final long converterExceptions;
 
+    private int order;
+
     @AssistedInject
     private Extractor(ApiClient api, UserService userService, @Assisted ExtractorSummaryResponse esr) {
         this.api = api;
@@ -180,7 +182,8 @@ public Extractor create(Node node, Input input) throws IOException, APIException
         request.extractorConfig = extractorConfig;
         request.converters = converterList;
         request.conditionType = conditionType.toString().toLowerCase();
-        request.conditionValue =  conditionValue;
+        request.conditionValue = conditionValue;
+        request.order = order;
 
         final Map response = api.post(Map.class)
                 .path("/system/inputs/{0}/extractors", input.getId())
@@ -206,6 +209,17 @@ public void loadConfigFromForm(Type extractorType, Map<String,String[]> form) {
         }
     }
 
+    public void loadConfigFromImport(Type type, Map<String, Object> extractorConfig) {
+        // we go the really easy way here.
+        Map<String, String[]> looksLikeForm = Maps.newHashMap();
+
+        for (Map.Entry<String, Object> e : extractorConfig.entrySet()) {
+            looksLikeForm.put(e.getKey(), new String[]{ e.getValue().toString() });
+        }
+
+        loadConfigFromForm(type, looksLikeForm);
+    }
+
     public void loadConvertersFromForm(Map<String,String[]> form) {
         for(String name : extractSelectedConverters(form)) {
             Converter.Type converterType = Converter.Type.valueOf(name.toUpperCase());
@@ -215,6 +229,17 @@ public void loadConvertersFromForm(Map<String,String[]> form) {
         }
     }
 
+    public void loadConvertersFromImport(List<Map<String, Object>> imports) {
+        for (Map<String, Object> imp : imports) {
+            Converter converter = new Converter(
+                    Converter.Type.valueOf(((String) imp.get("type")).toUpperCase()),
+                    (Map<String, Object>) imp.get("config")
+            );
+
+            converters.add(converter);
+        }
+    }
+
     private Map<String, Object> extractConverterConfig(Converter.Type converterType, Map<String,String[]> form) {
         Map<String, Object> config = Maps.newHashMap();
         switch (converterType) {
@@ -394,12 +419,21 @@ public int getOrder() {
         return order;
     }
 
+    public void setOrder(int order) {
+        this.order = order;
+    }
+
     public Map<String, Object> export() {
         Map<String, Object> export = Maps.newTreeMap();
 
         List<Map<String, Object>> converterConfigList = Lists.newArrayList();
         for (Converter converter : converters) {
-            converterConfigList.add(converter.getConfig());
+            Map<String, Object> converterExport = Maps.newHashMap();
+
+            converterExport.put("config", converter.getConfig());
+            converterExport.put("type", converter.getType());
+
+            converterConfigList.add(converterExport);
         }
 
         export.put("title", title);
diff --git a/app/models/api/requests/CreateExtractorRequest.java b/app/models/api/requests/CreateExtractorRequest.java
@@ -55,4 +55,6 @@ public class CreateExtractorRequest extends ApiRequest {
     @SerializedName("condition_value")
     public String conditionValue;
 
+    public int order;
+
 }
diff --git a/app/models/api/requests/ExtractorImportRequest.java b/app/models/api/requests/ExtractorImportRequest.java
@@ -0,0 +1,59 @@
+/**
+ * Copyright 2014 Lennart Koopmann <lennart@torch.sh>
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+package models.api.requests;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author Lennart Koopmann <lennart@torch.sh>
+ */
+public class ExtractorImportRequest {
+
+    public String title;
+
+    public int order;
+
+    @JsonProperty("condition_type")
+    public String conditionType;
+
+    @JsonProperty("condition_value")
+    public String conditionValue;
+
+    @JsonProperty("cursor_strategy")
+    public String cursorStrategy;
+
+    @JsonProperty("extractor_type")
+    public String extractorType;
+
+    @JsonProperty("source_field")
+    public String sourceField;
+
+    @JsonProperty("target_field")
+    public String targetField;
+
+    @JsonProperty("extractor_config")
+    public Map<String, Object> extractorConfig;
+
+    public List<Map<String, Object>> converters;
+
+}
diff --git a/app/models/api/requests/ExtractorListImportRequest.java b/app/models/api/requests/ExtractorListImportRequest.java
@@ -0,0 +1,33 @@
+/**
+ * Copyright 2014 Lennart Koopmann <lennart@torch.sh>
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+package models.api.requests;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author Lennart Koopmann <lennart@torch.sh>
+ */
+public class ExtractorListImportRequest {
+
+    public List<ExtractorImportRequest> extractors;
+    public String version;
+
+}
diff --git a/app/views/system/inputs/extractors/export.scala.html b/app/views/system/inputs/extractors/export.scala.html
@@ -6,16 +6,16 @@
 
     <div class="row-fluid">
         <h1>
-            <i class="icon icon-cloud-download"></i>
+            <i class="icon icon-cloud-upload"></i>
             Export extractors of <em>@input.getTitle</em>
         </h1>
 
         The extractors of an input can be exported to JSON for importing into other setups
-        or sharing in the extractor directory on <a href="http://www.graylog2.org/">graylog2.org</a>.
+        or sharing in the extractor directory on <a href="http://www.graylog2.org/" target="_blank">graylog2.org</a>.
     </div>
 
     <div class="row-fluid">
-<textarea id="extractor-export-json">@extractorExport</textarea>
+<textarea class="extractor-json">@extractorExport</textarea>
     </div>
 
 }
diff --git a/app/views/system/inputs/extractors/importPage.scala.html b/app/views/system/inputs/extractors/importPage.scala.html
@@ -0,0 +1,28 @@
+@(currentUser: User, breadcrumbs: lib.BreadcrumbList, node: Node, input: Input)
+
+@main("Import extractors", views.html.system.sidebar(), "", currentUser) {
+
+    @views.html.partials.breadcrumbs(breadcrumbs)
+
+    <div class="row-fluid">
+        <h1>
+            <i class="icon icon-cloud-download"></i>
+            Import extractors to input <em>@input.getTitle</em>
+        </h1>
+
+        Exported extractors can be imported to an input. All you need is the JSON export of extractors from any other
+        Graylog2 setup or from the extractor directory on <a href="http://www.graylog2.org/" target="_blank">graylog2.org</a>.
+    </div>
+
+    <form action="@routes.ExtractorsController.importExtractors(node.getNodeId, input.getId)" method="POST">
+        <div class="row-fluid">
+            <textarea class="extractor-json" name="extractors"></textarea>
+        </div>
+
+        <button type="submit" class="btn btn-success" data-confirm="Really add all extractors to this input?">
+            <i class="icon icon-cloud-download"></i>
+            Add extractors to input
+        </button>
+    </form>
+
+}
diff --git a/app/views/system/inputs/extractors/manage.scala.html b/app/views/system/inputs/extractors/manage.scala.html
@@ -9,8 +9,8 @@
                 <span class="caret"></span>
             </a>
             <ul class="dropdown-menu">
-                <li><a href="#">Import extractors</a></li>
-                <li><a href="@routes.ExtractorsController.export(node.getNodeId, input.getId)">Export extractors</a></li>
+                <li><a href="@routes.ExtractorsController.importExtractorsPage(node.getNodeId, input.getId)">Import extractors</a></li>
+                <li><a href="@routes.ExtractorsController.exportExtractors(node.getNodeId, input.getId)">Export extractors</a></li>
             </ul>
         </div>
     </div>
diff --git a/conf/routes b/conf/routes
diff --git a/public/javascripts/extractors.js b/public/javascripts/extractors.js
diff --git a/public/stylesheets/main.css b/public/stylesheets/main.css

Original file line number	Diff line number	Diff line change
`@@ -55,4 +55,6 @@ public class CreateExtractorRequest extends ApiRequest {`
`55`	`55`	`@SerializedName("condition_value")`
`56`	`56`	`public String conditionValue;`
`57`	`57`
	`58`	`+ public int order;`
	`59`	`+`
`58`	`60`	`}`