Skip to content
This repository was archived by the owner on Apr 23, 2020. It is now read-only.

Commit 6e4848f

Browse files
JohnWarnerEFJohnWarnerEF
committed
Added import/export support
1 parent 48f37d8 commit 6e4848f

12 files changed

Lines changed: 605 additions & 7 deletions

File tree

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ project(':exhibitor-core') {
5353
compile 'org.codehaus.jackson:jackson-mapper-asl:1.8.3'
5454
compile 'org.apache.lucene:lucene-core:3.6.0'
5555
compile 'com.sun.jersey:jersey-client:1.11'
56+
compile 'com.sun.jersey.contribs:jersey-multipart:1.11'
5657

5758
// if you are using Java 7 you can remove this and switch to the JDK version
5859
compile 'org.codehaus.jsr166-mirror:jsr166y:1.7.0'

‎exhibitor-core/src/main/java/com/netflix/exhibitor/core/entities/ExportRequest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package com.netflix.exhibitor.core.entities;
2+
3+
import javax.xml.bind.annotation.XmlRootElement;
4+
5+
@XmlRootElement
6+
public class ExportRequest {
7+
private String startPath;
8+
9+
public ExportRequest() {
10+
this("/");
11+
}
12+
13+
public ExportRequest(String startPath) {
14+
this.startPath = startPath;
15+
}
16+
17+
public String getStartPath() {
18+
return startPath;
19+
}
20+
21+
public void setStartPath(String startPath) {
22+
this.startPath = startPath;
23+
}
24+
}

‎exhibitor-core/src/main/java/com/netflix/exhibitor/core/importandexport/Exporter.java‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
package com.netflix.exhibitor.core.importandexport;
2+
3+
import com.google.common.base.Strings;
4+
import com.netflix.exhibitor.core.Exhibitor;
5+
import com.netflix.exhibitor.core.rest.UIContext;
6+
import com.sun.jersey.core.util.Base64;
7+
import org.apache.curator.utils.ZKPaths;
8+
import org.apache.zookeeper.data.ACL;
9+
import org.codehaus.jackson.JsonNode;
10+
import org.codehaus.jackson.node.ArrayNode;
11+
import org.codehaus.jackson.node.JsonNodeFactory;
12+
import org.codehaus.jackson.node.ObjectNode;
13+
14+
import java.util.Iterator;
15+
import java.util.List;
16+
17+
public class Exporter {
18+
private String startPath;
19+
private Exhibitor exhibitor;
20+
private UIContext context;
21+
22+
public Exporter(UIContext context, String startPath)
23+
{
24+
this.context = context;
25+
this.exhibitor = context.getExhibitor();
26+
27+
if (Strings.isNullOrEmpty(startPath)) {
28+
this.startPath = "/";
29+
} else {
30+
if (startPath.startsWith("/")) {
31+
this.startPath = startPath;
32+
} else {
33+
this.startPath = "/" + startPath;
34+
}
35+
}
36+
}
37+
38+
public String generate() throws Exception
39+
{
40+
ArrayNode jsonArray = JsonNodeFactory.instance.arrayNode();
41+
42+
return convertToExportFormat(getChildren(startPath, jsonArray));
43+
}
44+
45+
private String convertToExportFormat(ArrayNode jsonArray)
46+
{
47+
StringBuilder sb = new StringBuilder();
48+
sb.append("[\r\n");
49+
50+
Iterator<JsonNode> iterator = jsonArray.iterator();
51+
while (iterator.hasNext()) {
52+
JsonNode jsonNode = iterator.next();
53+
54+
sb.append(convertToFlatJsonAsString(jsonNode.get("path").getTextValue(), jsonNode.get("data").getTextValue(),
55+
jsonNode.get("acls").getElements()));
56+
57+
if (iterator.hasNext()) {
58+
sb.append(",\r\n");
59+
}
60+
}
61+
62+
sb.append("\r\n]");
63+
64+
return sb.toString();
65+
}
66+
67+
private String convertToFlatJsonAsString(String path, String data, Iterator<JsonNode> acls)
68+
{
69+
StringBuilder node = new StringBuilder();
70+
node.append("{\"path\": \"");
71+
node.append(path);
72+
node.append("\", \"data\": \"");
73+
node.append(data);
74+
node.append("\", \"acls\": [");
75+
76+
77+
while(acls.hasNext()) {
78+
JsonNode aclNode = acls.next();
79+
node.append("{\"scheme\": \"");
80+
node.append(aclNode.get("scheme").getTextValue());
81+
node.append("\", \"id\": \"");
82+
node.append(aclNode.get("id").getTextValue());
83+
node.append("\", \"perms\": ");
84+
node.append(aclNode.get("perms").getIntValue());
85+
node.append("}");
86+
87+
if (acls.hasNext()) {
88+
node.append(", ");
89+
}
90+
}
91+
92+
node.append("]}");
93+
94+
return node.toString();
95+
}
96+
97+
/**
98+
* Gets the list of children for a specific path. Then for each result it calls this method again. Eventually
99+
* we will have traversed the entire tree.
100+
*
101+
* @param path
102+
* @return ArrayNode
103+
* @throws Exception
104+
*/
105+
private ArrayNode getChildren(String path, ArrayNode jsonArray) throws Exception
106+
{
107+
List<String> children = exhibitor.getLocalConnection().getChildren().forPath(path);
108+
jsonArray.add(getNodeDetails(path));
109+
110+
for (String child : children) {
111+
getChildren(ZKPaths.makePath(path, child), jsonArray);
112+
}
113+
114+
return jsonArray;
115+
}
116+
117+
private JsonNode getNodeDetails(String path) throws Exception
118+
{
119+
byte[] data = exhibitor.getLocalConnection().getData().forPath(path);
120+
List<ACL> acls = exhibitor.getLocalConnection().getACL().forPath(path);
121+
122+
ObjectNode node = JsonNodeFactory.instance.objectNode();
123+
node.put("path", path);
124+
node.put("data", new String(Base64.encode(data)));
125+
126+
ArrayNode aclsArray = JsonNodeFactory.instance.arrayNode();
127+
for (ACL acl : acls) {
128+
ObjectNode aclNode = JsonNodeFactory.instance.objectNode();
129+
130+
aclNode.put("scheme", acl.getId().getScheme());
131+
aclNode.put("id", acl.getId().getId());
132+
aclNode.put("perms", acl.getPerms());
133+
134+
aclsArray.add(aclNode);
135+
}
136+
137+
node.put("acls", aclsArray);
138+
139+
return node;
140+
}
141+
}

‎exhibitor-core/src/main/java/com/netflix/exhibitor/core/importandexport/Importer.java‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package com.netflix.exhibitor.core.importandexport;
2+
3+
import com.netflix.exhibitor.core.activity.ActivityLog;
4+
import com.netflix.exhibitor.core.rest.UIContext;
5+
import com.sun.jersey.core.util.Base64;
6+
import org.apache.curator.framework.api.transaction.CuratorTransaction;
7+
import org.apache.curator.framework.api.transaction.CuratorTransactionFinal;
8+
import org.apache.curator.utils.ZKPaths;
9+
import org.apache.zookeeper.data.ACL;
10+
import org.apache.zookeeper.data.Id;
11+
import org.codehaus.jackson.JsonNode;
12+
13+
import javax.ws.rs.WebApplicationException;
14+
import javax.ws.rs.core.Response;
15+
import java.util.*;
16+
17+
public class Importer {
18+
19+
private final UIContext context;
20+
21+
public Importer(UIContext context)
22+
{
23+
this.context = context;
24+
}
25+
26+
/**
27+
* Imports all of the supplied nodes, starting at the prescribed base node. Because it isn't possible to set ACLs
28+
* on set commands, only create, and because we're using a transaction, the flow of this method is as follows.
29+
*
30+
* 1. If the node already exists and we're not overwriting, ignore this node and leave it alone.
31+
* 2. If the node already exists and we are overwriting, add the set command to the transaction and save the ACL
32+
* details. Once the transaction has been successfully committed, we then apply all the ACLs that we saved.
33+
* 3. If the node does not exist, add the create command to the transaction including the ACLs needed.
34+
*
35+
* This does leave us open to a couple of possible issues. It might be possible that a node doesn't exist when we
36+
* add it to the transaction, but it has been created by a 3rd party before we commit.
37+
*
38+
* It is also possible that something could go wrong when applying the ACLs to a node. Because the transaction has
39+
* already been committed at that point, we will be left in a position where the node data has been updated but not
40+
* the ACL.
41+
*/
42+
public void doImport(String basePath, boolean overwrite, Iterator<JsonNode> nodesToImport) throws Exception {
43+
CuratorTransaction transaction = context.getExhibitor().getLocalConnection().inTransaction();
44+
CuratorTransactionFinal curatorTransactionFinal = null;
45+
Map<String, List<ACL>> savedACLs = new HashMap<String, List<ACL>>();
46+
Set<String> toBeCreated = new HashSet<String>();
47+
48+
while (nodesToImport.hasNext()) {
49+
JsonNode jsonNode = nodesToImport.next();
50+
JsonNode pathNode = jsonNode.get("path");
51+
JsonNode dataNode = jsonNode.get("data");
52+
JsonNode aclsNode = jsonNode.get("acls");
53+
54+
if (pathNode == null || dataNode == null) throw new WebApplicationException(Response.Status.BAD_REQUEST);
55+
56+
String path = ZKPaths.makePath(basePath, pathNode.getTextValue());
57+
byte[] data = Base64.decode(dataNode.getTextValue());
58+
List<ACL> aclList = createACLList(aclsNode);
59+
60+
boolean alreadyExists = nodeAlreadyExists(path);
61+
62+
if (overwrite || !alreadyExists) {
63+
if (alreadyExists) {
64+
curatorTransactionFinal = transaction.setData().forPath(path, data).and();
65+
savedACLs.put(path, aclList);
66+
} else {
67+
createParentsIfNeeded(transaction, path, aclList, toBeCreated);
68+
curatorTransactionFinal = transaction.create().withACL(aclList).forPath(path, data).and();
69+
toBeCreated.add(path);
70+
}
71+
}
72+
}
73+
74+
if (curatorTransactionFinal == null) {
75+
context.getExhibitor().getLog().add(ActivityLog.Type.INFO, "There was nothing to import");
76+
return;
77+
}
78+
79+
curatorTransactionFinal.commit();
80+
81+
// Finally we apply those ACLs we saved
82+
for (Map.Entry<String, List<ACL>> entry : savedACLs.entrySet()) {
83+
context.getExhibitor().getLocalConnection().setACL().withACL(entry.getValue()).forPath(entry.getKey());
84+
}
85+
}
86+
87+
private void createParentsIfNeeded(CuratorTransaction transaction, String path, List<ACL> acls, Set<String> toBeCreated) throws Exception {
88+
String[] parts = path.substring(1).split("/");
89+
String builtUpPath = "";
90+
91+
// We do this to parts.length - 1, because we don't want to create the final path, as that's being done in the
92+
// calling method.
93+
94+
for (int i = 0; i < (parts.length - 1); i++) {
95+
builtUpPath += "/" + parts[i];
96+
97+
if (!toBeCreated.contains(builtUpPath) && context.getExhibitor().getLocalConnection().checkExists().forPath(builtUpPath) == null) {
98+
transaction.create().withACL(acls).forPath(builtUpPath, new byte[0]);
99+
toBeCreated.add(builtUpPath);
100+
}
101+
}
102+
}
103+
104+
private List<ACL> createACLList(JsonNode aclsNode) {
105+
List<ACL> aclList = new ArrayList<ACL>();
106+
if (aclsNode == null) return aclList;
107+
108+
Iterator<JsonNode> acls = aclsNode.getElements();
109+
110+
while (acls.hasNext()) {
111+
JsonNode aclNode = acls.next();
112+
113+
String scheme = aclNode.get("scheme").getTextValue();
114+
String id = aclNode.get("id").getTextValue();
115+
int perms = aclNode.get("perms").getIntValue();
116+
117+
aclList.add(new ACL(perms, new Id(scheme, id)));
118+
}
119+
120+
return aclList;
121+
}
122+
123+
private boolean nodeAlreadyExists(String path) throws Exception {
124+
return (context.getExhibitor().getLocalConnection().checkExists().forPath(path) != null);
125+
}
126+
127+
}

‎exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/ExplorerResource.java‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,26 +20,23 @@
2020
import com.google.common.collect.Iterables;
2121
import com.google.common.collect.Lists;
2222
import com.google.common.io.Closeables;
23+
import com.netflix.exhibitor.core.entities.*;
24+
import com.netflix.exhibitor.core.importandexport.Exporter;
2325
import org.apache.curator.utils.ZKPaths;
2426
import com.netflix.exhibitor.core.activity.ActivityLog;
2527
import com.netflix.exhibitor.core.analyze.Analysis;
2628
import com.netflix.exhibitor.core.analyze.PathAnalyzer;
2729
import com.netflix.exhibitor.core.analyze.PathAndMax;
2830
import com.netflix.exhibitor.core.analyze.PathComplete;
2931
import com.netflix.exhibitor.core.analyze.UsageListing;
30-
import com.netflix.exhibitor.core.entities.IdList;
31-
import com.netflix.exhibitor.core.entities.PathAnalysis;
32-
import com.netflix.exhibitor.core.entities.PathAnalysisNode;
33-
import com.netflix.exhibitor.core.entities.PathAnalysisRequest;
34-
import com.netflix.exhibitor.core.entities.Result;
35-
import com.netflix.exhibitor.core.entities.UsageListingRequest;
3632
import org.apache.zookeeper.KeeperException;
3733
import org.apache.zookeeper.data.Stat;
3834
import org.codehaus.jackson.map.ObjectMapper;
3935
import org.codehaus.jackson.node.ArrayNode;
4036
import org.codehaus.jackson.node.JsonNodeFactory;
4137
import org.codehaus.jackson.node.ObjectNode;
4238
import org.codehaus.jackson.type.TypeReference;
39+
4340
import javax.ws.rs.*;
4441
import javax.ws.rs.core.Context;
4542
import javax.ws.rs.core.Response;
@@ -257,6 +254,32 @@ public String getNode(@QueryParam("key") String key) throws Exception
257254
return children.toString();
258255
}
259256

257+
@GET
258+
@Path("export")
259+
@Produces("application/json")
260+
public Response getExport(@QueryParam("request") String json) throws Exception
261+
{
262+
ObjectMapper mapper = new ObjectMapper();
263+
ExportRequest exportRequest = mapper.getJsonFactory().createJsonParser(json).readValueAs(ExportRequest.class);
264+
265+
return getExport(exportRequest);
266+
}
267+
268+
@POST
269+
@Path("export")
270+
@Consumes("application/json")
271+
@Produces("application/json")
272+
public Response getExport(ExportRequest exportRequest) throws Exception
273+
{
274+
context.getExhibitor().getLog().add(ActivityLog.Type.INFO, "Starting export");
275+
276+
Exporter exporter = new Exporter(context, exportRequest.getStartPath());
277+
278+
return Response.ok(exporter.generate().toString())
279+
.header("content-disposition", "attachment; filename=exhibitor_export.json")
280+
.build();
281+
}
282+
260283
@GET
261284
@Path("usage-listing")
262285
@Produces("text/plain")

0 commit comments

Comments
 (0)