Skip to content

Commit 57f8e72

Browse files
fm3fm3
authored
Merge pull request #14 from scalableminds/copy-all-script
Handy client scripts
2 parents 9bdefac + de5bab0 commit 57f8e72

File tree

3 files changed

+155
-0
lines changed

3 files changed

+155
-0
lines changed

‎client/copy-all-script‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import grpc
5+
import sys
6+
7+
import fossildbapi_pb2 as proto
8+
import fossildbapi_pb2_grpc as proto_rpc
9+
10+
MAX_MESSAGE_LENGTH = 1073741824
11+
12+
def main():
13+
verbose = True
14+
15+
collections = ['skeletons', 'volumes', 'volumeData', 'skeletonUpdates']
16+
17+
listKeysBatchSize = 300
18+
19+
srcPort = 2000
20+
dstPort = 7155
21+
22+
srcChannel = grpc.insecure_channel('localhost:{}'.format(srcPort), options=[('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), (
23+
'grpc.max_receive_message_length', MAX_MESSAGE_LENGTH)])
24+
srcStub = proto_rpc.FossilDBStub(srcChannel)
25+
26+
dstChannel = grpc.insecure_channel('localhost:{}'.format(dstPort), options=[('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), (
27+
'grpc.max_receive_message_length', MAX_MESSAGE_LENGTH)])
28+
dstStub = proto_rpc.FossilDBStub(dstChannel)
29+
30+
testHealth(srcStub, 'source fossildb at {}'.format(srcPort))
31+
testHealth(dstStub, 'destination fossildb at {}'.format(dstPort))
32+
33+
putCount = 0
34+
35+
for collection in collections:
36+
print('copying collection ' + collection)
37+
lastKey = None
38+
while True:
39+
listKeysReply = srcStub.ListKeys(proto.ListKeysRequest(collection=collection, limit=listKeysBatchSize, startAfterKey=lastKey))
40+
assertSuccess(listKeysReply)
41+
if len(listKeysReply.keys) == 0:
42+
break
43+
if verbose:
44+
print(' copying key batch ', listKeysReply.keys)
45+
for key in listKeysReply.keys:
46+
if verbose:
47+
print(' copying key ', key)
48+
getMultipleVersionsReply = srcStub.GetMultipleVersions(proto.GetMultipleVersionsRequest(collection=collection, key=key))
49+
assertSuccess(getMultipleVersionsReply)
50+
for versionValueTuple in zip(getMultipleVersionsReply.versions, getMultipleVersionsReply.values):
51+
if verbose:
52+
print(' copying version ', versionValueTuple[0])
53+
putReply = dstStub.Put(proto.PutRequest(collection=collection, key=key, version=versionValueTuple[0], value=versionValueTuple[1]))
54+
assertSuccess(putReply)
55+
putCount += 1
56+
if (verbose and putCount % 10 == 0) or putCount % 10000 == 0:
57+
print("total put count:", putCount)
58+
59+
lastKey = listKeysReply.keys[-1]
60+
print("Done. total put count:", putCount)
61+
62+
def testHealth(stub, label):
63+
try:
64+
reply = stub.Health(proto.HealthRequest())
65+
assertSuccess(reply)
66+
print('successfully connected to ' + label)
67+
except Exception as e:
68+
print('failed to connect to ' + label + ': ' + str(e))
69+
sys.exit(1)
70+
71+
def assertSuccess(reply):
72+
if not reply.success:
73+
raise Exception("reply.success failed: " + reply.errorMessage)
74+
75+
if __name__ == '__main__':
76+
main()

‎client/copy-some-script‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env python3
2+
3+
import json
4+
import grpc
5+
import sys
6+
7+
import fossildbapi_pb2 as proto
8+
import fossildbapi_pb2_grpc as proto_rpc
9+
10+
MAX_MESSAGE_LENGTH = 1073741824
11+
12+
def main():
13+
verbose = True
14+
15+
collectionsByTyp = {
16+
'skeleton': ['skeletons', 'skeletonUpdates'],
17+
'volume': ['volumes', 'volumeData']
18+
}
19+
20+
srcPort = 2000
21+
dstPort = 7155
22+
23+
tracingReferences = json.load(open('tracingReferences.json'))
24+
25+
srcChannel = grpc.insecure_channel('localhost:{}'.format(srcPort), options=[('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), (
26+
'grpc.max_receive_message_length', MAX_MESSAGE_LENGTH)])
27+
srcStub = proto_rpc.FossilDBStub(srcChannel)
28+
29+
dstChannel = grpc.insecure_channel('localhost:{}'.format(dstPort), options=[('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), (
30+
'grpc.max_receive_message_length', MAX_MESSAGE_LENGTH)])
31+
dstStub = proto_rpc.FossilDBStub(dstChannel)
32+
33+
testHealth(srcStub, 'source fossildb at {}'.format(srcPort))
34+
testHealth(dstStub, 'destination fossildb at {}'.format(dstPort))
35+
36+
putCount = 0
37+
38+
for tracingReference in tracingReferences:
39+
key = tracingReference['id']
40+
if verbose:
41+
print(' copying key ', key)
42+
for collection in collectionsByTyp[tracingReference['typ']]:
43+
getMultipleVersionsReply = srcStub.GetMultipleVersions(proto.GetMultipleVersionsRequest(collection=collection, key=key))
44+
assertSuccess(getMultipleVersionsReply)
45+
if len(getMultipleVersionsReply.versions) == 0:
46+
print('[warn] no data for', key, 'in', collection)
47+
for versionValueTuple in zip(getMultipleVersionsReply.versions, getMultipleVersionsReply.values):
48+
if verbose:
49+
print(' copying version ', versionValueTuple[0])
50+
putReply = dstStub.Put(proto.PutRequest(collection=collection, key=key, version=versionValueTuple[0], value=versionValueTuple[1]))
51+
assertSuccess(putReply)
52+
putCount += 1
53+
if (verbose and putCount % 10 == 0) or putCount % 10000 == 0:
54+
print("total put count:", putCount)
55+
print("Done. total put count:", putCount)
56+
57+
def testHealth(stub, label):
58+
try:
59+
reply = stub.Health(proto.HealthRequest())
60+
assertSuccess(reply)
61+
print('successfully connected to ' + label)
62+
except Exception as e:
63+
print('failed to connect to ' + label + ': ' + str(e))
64+
sys.exit(1)
65+
66+
def assertSuccess(reply):
67+
if not reply.success:
68+
raise Exception("reply.success failed: " + reply.errorMessage)
69+
70+
if __name__ == '__main__':
71+
main()

‎docker-compose.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@ services:
2121
links:
2222
- fossildb
2323

24+
client:
25+
image: scalableminds/fossildb-client:${FOSSILDB_CLIENT_TAG:-master}
26+
volumes:
27+
- ".:/app"
28+
working_dir: /app
29+
entrypoint: /bin/bash
30+
network_mode: host
31+
2432
sbt:
2533
image: scalableminds/sbt:${SBT_VERSION_TAG:-sbt-0.13.15_mongo-3.2.17_node-8.x_jdk-8}
2634
environment:

0 commit comments

Comments
 (0)