Skip to content

Commit 8a42a62

Browse files
author
Meir Shpilraien (Spielrein)
authored
Merge pull request #9 from RedisGears/add_circle_tests
added circle tests
2 parents 4134b59 + dd1c1f0 commit 8a42a62

File tree

3 files changed

+145
-73
lines changed

3 files changed

+145
-73
lines changed

‎.circleci/config.yml‎

Lines changed: 22 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -3,83 +3,39 @@
33
# Check https://circleci.com/docs/2.0/language-python/ for more details
44
#
55
version: 2.1
6-
commands:
7-
early_return_for_forked_pull_requests:
8-
description: >-
9-
If this build is from a fork, stop executing the current job and return success.
10-
This is useful to avoid steps that will fail due to missing credentials.
11-
steps:
12-
- run:
13-
name: Early return if this build is from a forked PR
14-
command: |
15-
if [ -n "$CIRCLE_PR_NUMBER" ]; then
16-
echo "Nothing to do for forked PRs, so marking this step successful"
17-
circleci step halt
18-
fi
196
jobs:
20-
build:
7+
build_and_test:
218
docker:
22-
- image: circleci/python:3.6.1
23-
- image: redislabs/redisgears:edge
24-
25-
working_directory: ~/repo
26-
9+
- image: 'ubuntu:bionic'
2710
steps:
11+
- run:
12+
name: installations
13+
command: apt-get -qq update; apt-get install -y ca-certificates wget build-essential git python-pip
14+
- run:
15+
name: Redis_5_upgrade
16+
command: cd ..; git clone https://github.com/antirez/redis.git; cd ./redis; git fetch; git checkout 6.0.1; make; make install
17+
- run:
18+
name: download_RedisGears
19+
command: cd ..; wget http://redismodules.s3.amazonaws.com/redisgears/snapshots/redisgears.linux-bionic-x64.master.zip; apt-get install -y unzip; unzip redisgears.linux-bionic-x64.master.zip
20+
- run:
21+
name: download_RedisGears_deps
22+
command: cd ..; wget http://redismodules.s3.amazonaws.com/redisgears/snapshots/redisgears-dependencies.linux-bionic-x64.master.tgz; mkdir -p /var/opt/redislabs/modules/rg/; cd /var/opt/redislabs/modules/rg/; tar -xvf /root/redisgears-dependencies.linux-bionic-x64.master.tgz
2823
- checkout
29-
30-
# Download and cache dependencies
31-
- restore_cache:
32-
keys:
33-
- v1-dependencies-{{ checksum "requirements.txt" }}
34-
# fallback to using the latest cache if no exact match is found
35-
- v1-dependencies-
36-
3724
- run:
38-
name: install dependencies
39-
command: |
40-
python -m venv venv
41-
. venv/bin/activate
42-
pip install -r requirements.txt
43-
pip install codecov
44-
45-
- save_cache:
46-
paths:
47-
- ./venv
48-
key: v1-dependencies-{{ checksum "requirements.txt" }}
49-
25+
name: install_redisgears_py_to_RedisGears_virtual_env
26+
command: /var/opt/redislabs/modules/rg/python3_99.99.99/bin/python3 setup.py install
5027
- run:
51-
name: test dist
52-
command: python setup.py sdist
53-
28+
name: install RLTest
29+
command: /var/opt/redislabs/modules/rg/python3_99.99.99/bin/python3 -m pip install git+https://github.com/Grokzen/redis-py-cluster.git@master git+https://github.com/RedisLabsModules/RLTest.git
5430
- run:
55-
name: test example
56-
command: |
57-
. venv/bin/activate
58-
python example.py
59-
60-
# - run:
61-
# name: run tests
62-
# command: |
63-
# . venv/bin/activate
64-
# coverage run test.py
65-
66-
- early_return_for_forked_pull_requests
67-
68-
# - run:
69-
# name: codecove
70-
# command: |
71-
# . venv/bin/activate
72-
# codecov
31+
name: run_tests
32+
command: /var/opt/redislabs/modules/rg/python3_99.99.99/bin/python3 -m RLTest --module ../redisgears.so
7333

74-
# - store_artifacts:
75-
# path: test-reports
76-
# destination: test-reports
77-
7834
workflows:
7935
version: 2
8036
commit:
8137
jobs:
82-
- build
38+
- build_and_test
8339
nightly:
8440
triggers:
8541
- schedule:
@@ -89,4 +45,4 @@ workflows:
8945
only:
9046
- master
9147
jobs:
92-
- build
48+
- build_and_test

‎gearsclient/redisgears_builder.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def __init__(self, callback):
5959
def AddToGB(self, gb):
6060
gb.countby(self.callback)
6161

62-
class GearsRemoteAvgByStep():
62+
class GearsRemoteAvgStep():
6363
def __init__(self, callback):
6464
self.callback = callback
6565

@@ -68,14 +68,14 @@ def AddToGB(self, gb):
6868

6969
class GearsRemoteCountStep():
7070
def __init__(self):
71-
self.callback = callback
71+
pass
7272

7373
def AddToGB(self, gb):
7474
gb.count()
7575

7676
class GearsRemoteDistinctStep():
7777
def __init__(self):
78-
self.callback = callback
78+
pass
7979

8080
def AddToGB(self, gb):
8181
gb.distinct()
@@ -97,7 +97,7 @@ def __init__(self, extractor, zero, seqOp, combOp):
9797
self.combOp = combOp
9898

9999
def AddToGB(self, gb):
100-
gb.aggregate(self.extractor, self.zero, self.seqOp, self.combOp)
100+
gb.aggregateby(self.extractor, self.zero, self.seqOp, self.combOp)
101101

102102
class GearsRemoteSortStep():
103103
def __init__(self, reverse):
@@ -215,11 +215,16 @@ def createAndRun(self, GB):
215215

216216

217217
class GearsRemoteBuilder():
218-
def __init__(self, reader='KeysReader', defaultArg='*', r=None):
218+
def __init__(self, reader='KeysReader', defaultArg='*', r=None, requirements=[], addClientToRequirements=True):
219219
if r is None:
220220
r = redis.Redis()
221221
self.r = r
222222
self.pipe = GearsPipe(reader, defaultArg)
223+
self.requirements = requirements
224+
if addClientToRequirements:
225+
self.requirements += ['git+https://github.com/RedisGears/redisgears-py.git']
226+
if len(self.requirements) > 0:
227+
self.requirements = ['REQUIREMENTS'] + self.requirements
223228

224229
def localgroupby(self, extractor, reducer):
225230
self.pipe.localgroupby(extractor, reducer)
@@ -290,7 +295,7 @@ def run(self, arg=None, collect=True, **kargs):
290295
p = cloudpickle.loads(%s)
291296
p.createAndRun(GB)
292297
''' % selfBytes
293-
results = self.r.execute_command('RG.PYEXECUTE', serverCode)
298+
results = self.r.execute_command('RG.PYEXECUTE', serverCode, *self.requirements)
294299
res, errs = results
295300
res = [cloudpickle.loads(record) for record in res]
296301
return res, errs
@@ -303,7 +308,7 @@ def register(self, prefix='*', convertToStr=True, collect=True, **kargs):
303308
p = cloudpickle.loads(%s)
304309
p.createAndRun(GB)
305310
''' % selfBytes
306-
res = self.r.execute_command('RG.PYEXECUTE', serverCode)
311+
res = self.r.execute_command('RG.PYEXECUTE', serverCode, *self.requirements)
307312
return res
308313

309314
def log(msg, level='notice'):

‎test.py‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
from gearsclient import GearsRemoteBuilder as GRB
2+
from gearsclient import log, hashtag, execute, atomic
3+
4+
counter = 0
5+
6+
def getGB(env, reader='KeysReader'):
7+
return GRB(reader, r=env.getConnection(), addClientToRequirements=False)
8+
9+
def test_map(env):
10+
env.cmd('set', 'x', '1')
11+
env.cmd('set', 'y', '2')
12+
env.cmd('set', 'z', '3')
13+
res = getGB(env).map(lambda x: x['value']).sort().run()
14+
env.assertEqual(res, (['1', '2', '3'], []))
15+
16+
def test_filter(env):
17+
env.cmd('set', 'x', '1')
18+
env.cmd('set', 'y', '2')
19+
env.cmd('set', 'z', '3')
20+
res = getGB(env).map(lambda x: x['value']).filter(lambda x: x=='1').run()
21+
env.assertEqual(res, (['1'], []))
22+
23+
def test_foreach(env):
24+
env.cmd('set', 'x', '1')
25+
env.cmd('set', 'y', '2')
26+
env.cmd('set', 'z', '3')
27+
def increase(x):
28+
global counter
29+
counter += 1
30+
31+
# important to notice, the counte will increased on the server size and not on client side!!
32+
res = getGB(env).foreach(increase).map(lambda x: counter).run()
33+
env.assertEqual(res, ([1, 2, 3], []))
34+
35+
def test_flatmap(env):
36+
env.cmd('lpush', 'l', '1', '2', '3')
37+
res = getGB(env, 'KeysOnlyReader').map(lambda x: execute('lrange', x, '0', '-1')).flatmap(lambda x: x).run()
38+
env.assertEqual(res, (['1', '2', '3'], []))
39+
40+
def test_countby(env):
41+
env.cmd('set', 'x', '1')
42+
env.cmd('set', 'y', '1')
43+
env.cmd('set', 'z', '2')
44+
env.cmd('set', 't', '2')
45+
res = getGB(env).map(lambda x: x['value']).countby().map(lambda x: (x['key'], x['value'])).sort().run()
46+
env.assertEqual(res, ([('1', 2), ('2', 2)], []))
47+
48+
def test_avg(env):
49+
env.cmd('set', 'x', '1')
50+
env.cmd('set', 'y', '1')
51+
env.cmd('set', 'z', '2')
52+
env.cmd('set', 't', '2')
53+
res = getGB(env).map(lambda x: x['value']).avg().run()
54+
env.assertEqual(res, ([1.5], []))
55+
56+
def test_count(env):
57+
env.cmd('set', 'x', '1')
58+
env.cmd('set', 'y', '1')
59+
env.cmd('set', 'z', '2')
60+
env.cmd('set', 't', '2')
61+
res = getGB(env).count().run()
62+
env.assertEqual(res, ([4], []))
63+
64+
def test_distinct(env):
65+
env.cmd('set', 'x', '1')
66+
env.cmd('set', 'y', '1')
67+
env.cmd('set', 'z', '2')
68+
env.cmd('set', 't', '2')
69+
res = getGB(env).map(lambda x: x['value']).distinct().count().run()
70+
env.assertEqual(res, ([2], []))
71+
72+
def test_aggregate(env):
73+
env.cmd('set', 'x', '1')
74+
env.cmd('set', 'y', '1')
75+
env.cmd('set', 'z', '2')
76+
env.cmd('set', 't', '2')
77+
res = getGB(env).map(lambda x: x['value']).aggregate(0, lambda a, r: a + int(r), lambda a, r: a + r).run()
78+
env.assertEqual(res, ([6], []))
79+
80+
def test_aggregateby(env):
81+
env.cmd('set', 'x', '1')
82+
env.cmd('set', 'y', '1')
83+
env.cmd('set', 'z', '2')
84+
env.cmd('set', 't', '2')
85+
res = getGB(env).map(lambda x: x['value']).aggregateby(lambda x: x, 0, lambda k, a, r: a + int(r), lambda k, a, r: a + r).map(lambda x: (x['key'], x['value'])).sort().run()
86+
env.assertEqual(res, ([('1', 2), ('2', 4)], []))
87+
88+
def test_limit(env):
89+
env.cmd('set', 'x', '1')
90+
env.cmd('set', 'y', '1')
91+
env.cmd('set', 'z', '2')
92+
env.cmd('set', 't', '2')
93+
res = getGB(env).map(lambda x: x['value']).sort().limit(1).run()
94+
env.assertEqual(res, (['1'], []))
95+
96+
def test_sort(env):
97+
env.cmd('set', 'x', '1')
98+
env.cmd('set', 'y', '1')
99+
env.cmd('set', 'z', '2')
100+
env.cmd('set', 't', '2')
101+
res = getGB(env).map(lambda x: x['key']).sort().run()
102+
env.assertEqual(res, (['t', 'x', 'y', 'z'], []))
103+
104+
def test_hashtag(env):
105+
res = getGB(env, 'ShardsIDReader').map(lambda x: hashtag()).run()
106+
env.assertEqual(res, (['06S'], []))
107+
108+
def test_register(env):
109+
res = getGB(env, 'CommandReader').register(trigger='test')
110+
env.assertEqual(res, b'OK')
111+
env.expect('RG.TRIGGER', 'test', 'this', 'is', 'a', 'test').equal([b"['test', 'this', 'is', 'a', 'test']"])

0 commit comments

Comments
 (0)