@@ -80,67 +80,6 @@ def __init__(self, iterable=None, buffer_size=4):
8080
8181 coroutine .close () # Terminate coroutine.
8282
83- def _extend_using_coroutine (self , buffer_size = 4 ):
84- """ Creates a coroutine allowing to append elements.
85-
86- Parameters
87- ----------
88- buffer_size : float, optional
89- Size (in Mb) for memory pre-allocation.
90-
91- Returns
92- -------
93- coroutine
94- Coroutine object which expects the values to be appended to this
95- array sequence.
96-
97- Notes
98- -----
99- This method is essential for
100- :func:`create_arraysequences_from_generator` as it allows for an
101- efficient way of creating multiple array sequences in a hyperthreaded
102- fashion and still benefit from the memory buffering. Whitout this
103- method the alternative would be to use :meth:`append` which does
104- not have such buffering mechanism and thus is at least one order of
105- magnitude slower.
106- """
107- offsets = []
108- lengths = []
109-
110- offset = 0 if len (self ) == 0 else self ._offsets [- 1 ] + self ._lengths [- 1 ]
111- try :
112- first_element = True
113- while True :
114- e = (yield )
115- e = np .asarray (e )
116- if first_element :
117- first_element = False
118- n_rows_buffer = int (buffer_size * 1024 ** 2 // e .nbytes )
119- new_shape = (n_rows_buffer ,) + e .shape [1 :]
120- if len (self ) == 0 :
121- self ._data = np .empty (new_shape , dtype = e .dtype )
122-
123- end = offset + len (e )
124- if end > len (self ._data ):
125- # Resize needed, adding `len(e)` items plus some buffer.
126- nb_points = len (self ._data )
127- nb_points += len (e ) + n_rows_buffer
128- self ._data .resize ((nb_points ,) + self .common_shape )
129-
130- offsets .append (offset )
131- lengths .append (len (e ))
132- self ._data [offset :offset + len (e )] = e
133- offset += len (e )
134-
135- except GeneratorExit :
136- pass
137-
138- self ._offsets = np .concatenate ([self ._offsets , offsets ], axis = 0 )
139- self ._lengths = np .concatenate ([self ._lengths , lengths ], axis = 0 )
140-
141- # Clear unused memory.
142- self ._data .resize ((offset ,) + self .common_shape )
143-
14483 @property
14584 def is_array_sequence (self ):
14685 return True
@@ -238,6 +177,67 @@ def extend(self, elements):
238177 self ._lengths = np .r_ [self ._lengths , elements ._lengths ]
239178 self ._offsets = np .r_ [self ._offsets , offsets ]
240179
180+ def _extend_using_coroutine (self , buffer_size = 4 ):
181+ """ Creates a coroutine allowing to append elements.
182+
183+ Parameters
184+ ----------
185+ buffer_size : float, optional
186+ Size (in Mb) for memory pre-allocation.
187+
188+ Returns
189+ -------
190+ coroutine
191+ Coroutine object which expects the values to be appended to this
192+ array sequence.
193+
194+ Notes
195+ -----
196+ This method is essential for
197+ :func:`create_arraysequences_from_generator` as it allows for an
198+ efficient way of creating multiple array sequences in a hyperthreaded
199+ fashion and still benefit from the memory buffering. Whitout this
200+ method the alternative would be to use :meth:`append` which does
201+ not have such buffering mechanism and thus is at least one order of
202+ magnitude slower.
203+ """
204+ offsets = []
205+ lengths = []
206+
207+ offset = 0 if len (self ) == 0 else self ._offsets [- 1 ] + self ._lengths [- 1 ]
208+ try :
209+ first_element = True
210+ while True :
211+ e = (yield )
212+ e = np .asarray (e )
213+ if first_element :
214+ first_element = False
215+ n_rows_buffer = int (buffer_size * 1024 ** 2 // e .nbytes )
216+ new_shape = (n_rows_buffer ,) + e .shape [1 :]
217+ if len (self ) == 0 :
218+ self ._data = np .empty (new_shape , dtype = e .dtype )
219+
220+ end = offset + len (e )
221+ if end > len (self ._data ):
222+ # Resize needed, adding `len(e)` items plus some buffer.
223+ nb_points = len (self ._data )
224+ nb_points += len (e ) + n_rows_buffer
225+ self ._data .resize ((nb_points ,) + self .common_shape )
226+
227+ offsets .append (offset )
228+ lengths .append (len (e ))
229+ self ._data [offset :offset + len (e )] = e
230+ offset += len (e )
231+
232+ except GeneratorExit :
233+ pass
234+
235+ self ._offsets = np .r_ [self ._offsets , offsets ].astype (np .intp )
236+ self ._lengths = np .r_ [self ._lengths , lengths ].astype (np .intp )
237+
238+ # Clear unused memory.
239+ self ._data .resize ((offset ,) + self .common_shape )
240+
241241 def copy (self ):
242242 """ Creates a copy of this :class:`ArraySequence` object.
243243
0 commit comments