Skip to content

Commit ea9e9d7

Browse files
alimanfooalimanfoo
committed
bench results
1 parent b30b5a6 commit ea9e9d7

File tree

1 file changed

+161
-0
lines changed

1 file changed

+161
-0
lines changed

‎bench/compress_normal.txt

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
zarr.core.Array((200000000,), uint16, chunks=(1000000,), order=C)
2+
compression: blosc; compression_opts: {'clevel': 5, 'cname': 'lz4', 'shuffle': 2}
3+
nbytes: 381.5M; nbytes_stored: 294; ratio: 1360544.2; initialized: 0/200
4+
store: builtins.dict
5+
*******************************************************************************
6+
[0.27119584499996563, 0.2855067059999783, 0.2887747180002407, 0.3058794240005227, 0.3139041080003153, 0.3021271820007314, 0.31543190899992624, 0.31403100900024583, 0.3272544129995367, 0.31834129100025166]
7+
0.27119584499996563
8+
zarr.core.Array((200000000,), uint16, chunks=(1000000,), order=C)
9+
compression: blosc; compression_opts: {'clevel': 5, 'cname': 'lz4', 'shuffle': 2}
10+
nbytes: 381.5M; nbytes_stored: 314.1M; ratio: 1.2; initialized: 200/200
11+
store: builtins.dict
12+
Timer unit: 1e-06 s
13+
14+
Total time: 0.297223 s
15+
File: /home/aliman/code/github/alimanfoo/zarr/zarr/blosc.pyx
16+
Function: compress at line 137
17+
18+
Line # Hits Time Per Hit % Time Line Contents
19+
==============================================================
20+
137 def compress(source, char* cname, int clevel, int shuffle):
21+
138 """Compress data in a numpy array.
22+
139
23+
140 Parameters
24+
141 ----------
25+
142 source : array-like
26+
143 Data to be compressed.
27+
144 cname : bytes
28+
145 Name of compression library to use.
29+
146 clevel : int
30+
147 Compression level.
31+
148 shuffle : int
32+
149 Shuffle filter.
33+
150
34+
151 Returns
35+
152 -------
36+
153 dest : bytes-like
37+
154 Compressed data.
38+
155
39+
156 """
40+
157
41+
158 cdef:
42+
159 char *source_ptr
43+
160 char *dest_ptr
44+
161 Py_buffer source_buffer
45+
162 size_t nbytes, cbytes, itemsize
46+
163 200 506 2.5 0.2 array.array char_array_template = array.array('b', [])
47+
164 array.array dest
48+
165
49+
166 # setup source buffer
50+
167 200 458 2.3 0.2 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS)
51+
168 200 119 0.6 0.0 source_ptr = <char *> source_buffer.buf
52+
169
53+
170 # setup destination
54+
171 200 239 1.2 0.1 nbytes = source_buffer.len
55+
172 200 103 0.5 0.0 itemsize = source_buffer.itemsize
56+
173 200 2286 11.4 0.8 dest = array.clone(char_array_template, nbytes + BLOSC_MAX_OVERHEAD,
57+
174 zero=False)
58+
175 200 129 0.6 0.0 dest_ptr = <char *> dest.data.as_voidptr
59+
176
60+
177 # perform compression
61+
178 200 1734 8.7 0.6 if _get_use_threads():
62+
179 # allow blosc to use threads internally
63+
180 200 167 0.8 0.1 compressor_set = blosc_set_compressor(cname)
64+
181 200 94 0.5 0.0 if compressor_set < 0:
65+
182 raise ValueError('compressor not supported: %r' % cname)
66+
183 200 288570 1442.8 97.1 with nogil:
67+
184 cbytes = blosc_compress(clevel, shuffle, itemsize, nbytes,
68+
185 source_ptr, dest_ptr,
69+
186 nbytes + BLOSC_MAX_OVERHEAD)
70+
187
71+
188 else:
72+
189 with nogil:
73+
190 cbytes = blosc_compress_ctx(clevel, shuffle, itemsize, nbytes,
74+
191 source_ptr, dest_ptr,
75+
192 nbytes + BLOSC_MAX_OVERHEAD, cname,
76+
193 0, 1)
77+
194
78+
195 # release source buffer
79+
196 200 616 3.1 0.2 PyBuffer_Release(&source_buffer)
80+
197
81+
198 # check compression was successful
82+
199 200 120 0.6 0.0 if cbytes <= 0:
83+
200 raise RuntimeError('error during blosc compression: %d' % cbytes)
84+
201
85+
202 # resize after compression
86+
203 200 1896 9.5 0.6 array.resize(dest, cbytes)
87+
204
88+
205 200 186 0.9 0.1 return dest
89+
90+
*******************************************************************************
91+
[0.24293352799941204, 0.2324290420001489, 0.24935673900017719, 0.25716222699975333, 0.24246313799994823, 0.23272456500035332, 0.2636815870000646, 0.2576046349995522, 0.2781278639995435, 0.23824110699933954]
92+
0.2324290420001489
93+
Timer unit: 1e-06 s
94+
95+
Total time: 0.240178 s
96+
File: /home/aliman/code/github/alimanfoo/zarr/zarr/blosc.pyx
97+
Function: decompress at line 75
98+
99+
Line # Hits Time Per Hit % Time Line Contents
100+
==============================================================
101+
75 def decompress(source, dest):
102+
76 """Decompress data.
103+
77
104+
78 Parameters
105+
79 ----------
106+
80 source : bytes-like
107+
81 Compressed data, including blosc header.
108+
82 dest : array-like
109+
83 Object to decompress into.
110+
84
111+
85 Notes
112+
86 -----
113+
87 Assumes that the size of the destination buffer is correct for the size of
114+
88 the uncompressed data.
115+
89
116+
90 """
117+
91 cdef:
118+
92 int ret
119+
93 char *source_ptr
120+
94 char *dest_ptr
121+
95 Py_buffer source_buffer
122+
96 array.array source_array
123+
97 Py_buffer dest_buffer
124+
98 size_t nbytes
125+
99
126+
100 # setup source buffer
127+
101 200 573 2.9 0.2 if PY2 and isinstance(source, array.array):
128+
102 # workaround fact that array.array does not support new-style buffer
129+
103 # interface in PY2
130+
104 release_source_buffer = False
131+
105 source_array = source
132+
106 source_ptr = <char *> source_array.data.as_voidptr
133+
107 else:
134+
108 200 112 0.6 0.0 release_source_buffer = True
135+
109 200 144 0.7 0.1 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS)
136+
110 200 98 0.5 0.0 source_ptr = <char *> source_buffer.buf
137+
111
138+
112 # setup destination buffer
139+
113 200 552 2.8 0.2 PyObject_GetBuffer(dest, &dest_buffer,
140+
114 PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE)
141+
115 200 100 0.5 0.0 dest_ptr = <char *> dest_buffer.buf
142+
116 200 84 0.4 0.0 nbytes = dest_buffer.len
143+
117
144+
118 # perform decompression
145+
119 200 1856 9.3 0.8 if _get_use_threads():
146+
120 # allow blosc to use threads internally
147+
121 200 235286 1176.4 98.0 with nogil:
148+
122 ret = blosc_decompress(source_ptr, dest_ptr, nbytes)
149+
123 else:
150+
124 with nogil:
151+
125 ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1)
152+
126
153+
127 # release buffers
154+
128 200 754 3.8 0.3 if release_source_buffer:
155+
129 200 326 1.6 0.1 PyBuffer_Release(&source_buffer)
156+
130 200 165 0.8 0.1 PyBuffer_Release(&dest_buffer)
157+
131
158+
132 # handle errors
159+
133 200 128 0.6 0.1 if ret <= 0:
160+
134 raise RuntimeError('error during blosc decompression: %d' % ret)
161+

0 commit comments

Comments
 (0)