Skip to content

Commit a132dab

Browse files
nwilbertnwilbert
committed
added async MongoDB benchmarks
1 parent cfebfff commit a132dab

File tree

5 files changed

+291
-10
lines changed

5 files changed

+291
-10
lines changed

notebook/aioloop34.ipynb renamed to notebook/aio34.ipynb

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,14 +95,18 @@
9595
},
9696
{
9797
"cell_type": "code",
98-
"execution_count": 4,
98+
"execution_count": 1,
9999
"metadata": {},
100100
"outputs": [
101101
{
102-
"name": "stdout",
103-
"output_type": "stream",
104-
"text": [
105-
"Future is done!\n"
102+
"ename": "NameError",
103+
"evalue": "name 'asyncio' is not defined",
104+
"output_type": "error",
105+
"traceback": [
106+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
107+
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
108+
"\u001b[0;32m<ipython-input-1-02a280871274>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;34m@\u001b[0m\u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcoroutine\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mslow_operation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Future is done!'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
109+
"\u001b[0;31mNameError\u001b[0m: name 'asyncio' is not defined"
106110
]
107111
}
108112
],

notebook/aioloop35.ipynb renamed to notebook/aio35.ipynb

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@
6262
},
6363
{
6464
"cell_type": "code",
65-
"execution_count": 9,
65+
"execution_count": 15,
6666
"metadata": {},
6767
"outputs": [
6868
{
@@ -71,12 +71,23 @@
7171
"text": [
7272
"Hello World!\n"
7373
]
74+
},
75+
{
76+
"data": {
77+
"text/plain": [
78+
"42"
79+
]
80+
},
81+
"execution_count": 15,
82+
"output_type": "execute_result",
83+
"metadata": {}
7484
}
7585
],
7686
"source": [
7787
"async def hello_world():\n",
7888
" await asyncio.sleep(1.0)\n",
7989
" print('Hello World!')\n",
90+
" return 42\n",
8091
"\n",
8192
"loop.run_until_complete(hello_world())"
8293
]
@@ -90,22 +101,31 @@
90101
},
91102
{
92103
"cell_type": "code",
93-
"execution_count": 10,
104+
"execution_count": 16,
94105
"metadata": {},
95106
"outputs": [
107+
{
108+
"name": "stdout",
109+
"output_type": "stream",
110+
"text": [
111+
"Hello World!\nHello World!\n"
112+
]
113+
},
96114
{
97115
"data": {
98116
"text/plain": [
99-
"<Task pending coro=<hello_world() running at <ipython-input-9-c98f2874b726>:1>>"
117+
"[42, 42]"
100118
]
101119
},
102-
"execution_count": 10,
120+
"execution_count": 16,
103121
"output_type": "execute_result",
104122
"metadata": {}
105123
}
106124
],
107125
"source": [
108-
"asyncio.ensure_future(hello_world())"
126+
"task1 = loop.create_task(hello_world())\n",
127+
"task2 = loop.create_task(hello_world())\n",
128+
"loop.run_until_complete(asyncio.gather(task1, task2))"
109129
]
110130
},
111131
{

‎notebook/aio_mongo.ipynb‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 13,
6+
"metadata": {},
7+
"outputs": [],
8+
"source": [
9+
"import time\n",
10+
"import asyncio\n",
11+
"import contextlib\n",
12+
"from pymongo import MongoClient\n",
13+
"from motor.motor_asyncio import AsyncIOMotorClient\n",
14+
"\n",
15+
"@contextlib.contextmanager\n",
16+
"def timer(description):\n",
17+
" start_time = time.time()\n",
18+
" yield\n",
19+
" end_time = time.time()\n",
20+
" print('{} took {} seconds'.format(description, end_time - start_time))\n",
21+
" \n",
22+
"sync_client = MongoClient().aiotest.testcol\n",
23+
"aio_client = AsyncIOMotorClient(max_pool_size=10).aiotest.testcol\n",
24+
"loop = asyncio.get_event_loop()"
25+
]
26+
},
27+
{
28+
"cell_type": "code",
29+
"execution_count": 6,
30+
"metadata": {},
31+
"outputs": [],
32+
"source": [
33+
"N = 1000"
34+
]
35+
},
36+
{
37+
"cell_type": "code",
38+
"execution_count": 22,
39+
"metadata": {},
40+
"outputs": [],
41+
"source": [
42+
"_ = sync_client.insert({'abc': i} for i in range(N))"
43+
]
44+
},
45+
{
46+
"cell_type": "code",
47+
"execution_count": 25,
48+
"metadata": {},
49+
"outputs": [
50+
{
51+
"name": "stdout",
52+
"output_type": "stream",
53+
"text": [
54+
"sequential sync read took 0.34887003898620605 seconds\n"
55+
]
56+
}
57+
],
58+
"source": [
59+
"with timer('sequential sync read'):\n",
60+
" for i in range(N):\n",
61+
" sync_client.find_one({'abc': i})"
62+
]
63+
},
64+
{
65+
"cell_type": "code",
66+
"execution_count": 9,
67+
"metadata": {},
68+
"outputs": [
69+
{
70+
"name": "stdout",
71+
"output_type": "stream",
72+
"text": [
73+
"sequential async read took 0.5591599941253662 seconds\n"
74+
]
75+
}
76+
],
77+
"source": [
78+
"with timer('sequential async read'):\n",
79+
" for i in range(N):\n",
80+
" loop.run_until_complete(aio_client.find_one({'abc': i}))"
81+
]
82+
},
83+
{
84+
"cell_type": "code",
85+
"execution_count": 14,
86+
"metadata": {},
87+
"outputs": [
88+
{
89+
"name": "stdout",
90+
"output_type": "stream",
91+
"text": [
92+
"parallel async read with 1 parallel requests took 0.616163969039917 seconds\nparallel async read with 2 parallel requests took 0.35162997245788574 seconds"
93+
]
94+
},
95+
{
96+
"name": "stdout",
97+
"output_type": "stream",
98+
"text": [
99+
"\nparallel async read with 3 parallel requests took 0.2948617935180664 seconds"
100+
]
101+
},
102+
{
103+
"name": "stdout",
104+
"output_type": "stream",
105+
"text": [
106+
"\nparallel async read with 4 parallel requests took 0.26516294479370117 seconds"
107+
]
108+
},
109+
{
110+
"name": "stdout",
111+
"output_type": "stream",
112+
"text": [
113+
"\nparallel async read with 5 parallel requests took 0.24994611740112305 seconds"
114+
]
115+
},
116+
{
117+
"name": "stdout",
118+
"output_type": "stream",
119+
"text": [
120+
"\nparallel async read with 6 parallel requests took 0.24745702743530273 seconds"
121+
]
122+
},
123+
{
124+
"name": "stdout",
125+
"output_type": "stream",
126+
"text": [
127+
"\nparallel async read with 7 parallel requests took 0.23832011222839355 seconds"
128+
]
129+
},
130+
{
131+
"name": "stdout",
132+
"output_type": "stream",
133+
"text": [
134+
"\nparallel async read with 8 parallel requests took 0.23779797554016113 seconds"
135+
]
136+
},
137+
{
138+
"name": "stdout",
139+
"output_type": "stream",
140+
"text": [
141+
"\nparallel async read with 9 parallel requests took 0.23258399963378906 seconds"
142+
]
143+
},
144+
{
145+
"name": "stdout",
146+
"output_type": "stream",
147+
"text": [
148+
"\nparallel async read with 10 parallel requests took 0.231536865234375 seconds"
149+
]
150+
},
151+
{
152+
"name": "stdout",
153+
"output_type": "stream",
154+
"text": [
155+
"\nparallel async read with 11 parallel requests took 0.22452688217163086 seconds"
156+
]
157+
},
158+
{
159+
"name": "stdout",
160+
"output_type": "stream",
161+
"text": [
162+
"\nparallel async read with 12 parallel requests took 0.2237100601196289 seconds"
163+
]
164+
},
165+
{
166+
"name": "stdout",
167+
"output_type": "stream",
168+
"text": [
169+
"\nparallel async read with 13 parallel requests took 0.21818184852600098 seconds"
170+
]
171+
},
172+
{
173+
"name": "stdout",
174+
"output_type": "stream",
175+
"text": [
176+
"\nparallel async read with 14 parallel requests took 0.21832585334777832 seconds"
177+
]
178+
},
179+
{
180+
"name": "stdout",
181+
"output_type": "stream",
182+
"text": [
183+
"\nparallel async read with 15 parallel requests took 0.22688078880310059 seconds"
184+
]
185+
},
186+
{
187+
"name": "stdout",
188+
"output_type": "stream",
189+
"text": [
190+
"\nparallel async read with 16 parallel requests took 0.22031497955322266 seconds"
191+
]
192+
},
193+
{
194+
"name": "stdout",
195+
"output_type": "stream",
196+
"text": [
197+
"\nparallel async read with 17 parallel requests took 0.22269797325134277 seconds"
198+
]
199+
},
200+
{
201+
"name": "stdout",
202+
"output_type": "stream",
203+
"text": [
204+
"\nparallel async read with 18 parallel requests took 0.22491192817687988 seconds"
205+
]
206+
},
207+
{
208+
"name": "stdout",
209+
"output_type": "stream",
210+
"text": [
211+
"\nparallel async read with 19 parallel requests took 0.21470904350280762 seconds"
212+
]
213+
},
214+
{
215+
"name": "stdout",
216+
"output_type": "stream",
217+
"text": [
218+
"\n"
219+
]
220+
}
221+
],
222+
"source": [
223+
"for chunk_size in range(1, 20):\n",
224+
" with timer('parallel async read with {} parallel requests'.format(chunk_size)):\n",
225+
" futures = [col.find_one({'abc': i})]\n",
226+
" for i in range(N // chunk_size):\n",
227+
" # interesting part:\n",
228+
" futures = [col.find_one({'abc': (i * chunk_size) + j})\n",
229+
" for j in range(chunk_size)]\n",
230+
" loop.run_until_complete(asyncio.gather(*futures))"
231+
]
232+
}
233+
],
234+
"metadata": {
235+
"kernelspec": {
236+
"display_name": "Python 3",
237+
"language": "python",
238+
"name": "python3"
239+
},
240+
"language_info": {
241+
"codemirror_mode": {
242+
"name": "ipython",
243+
"version": 3.0
244+
},
245+
"file_extension": ".py",
246+
"mimetype": "text/x-python",
247+
"name": "python",
248+
"nbconvert_exporter": "python",
249+
"pygments_lexer": "ipython3",
250+
"version": "3.5.0"
251+
}
252+
},
253+
"nbformat": 4,
254+
"nbformat_minor": 0
255+
}

notebook/overhead.ipynb renamed to notebook/aio_overhead.ipynb

File renamed without changes.

‎requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
aiohttp == 0.18.3
22
netifaces == 0.10.4
33
jupyter == 1.0.0
4+
motor == 0.5b0
5+
matplotlib == 1.5.0

0 commit comments

Comments
 (0)