Skip to content

Commit 201f5e8

Browse files
stamparmstamparm
committed
Couple of bug fixes for BigArray
1 parent f256077 commit 201f5e8

File tree

3 files changed

+122
-39
lines changed

3 files changed

+122
-39
lines changed

‎data/txt/sha256sums.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ eed1db5da17eca4c65a8f999166e2246eef84397687ae820bbe4984ef65a09df extra/vulnserv
165165
49bcd74281297c79a6ae5d4b0d1479ddace4476fddaf4383ca682a6977b553e3 lib/controller/handler.py
166166
4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811 lib/controller/__init__.py
167167
ac44a343947162532dbf17bd1f9ab424f8008f677367c5ad3f9f7b715a679818 lib/core/agent.py
168-
fbba89420acafcdb9ba1a95428cf2161b13cfa2d1a7ad7d5e70c14b0e04861f0 lib/core/bigarray.py
168+
fdb5e3828e14f5bca76bed85747c111a861c972bac3892ac789d0285c1b0e8b3 lib/core/bigarray.py
169169
f6062e324fdeaacf9df0a289fc3f12f755143e3876a70cb65b38aa2e690f73c1 lib/core/common.py
170170
11c748cc96ea2bc507bc6c1930a17fe4bc6fdd2dd2a80430df971cb21428eb00 lib/core/compat.py
171171
39ea62d4224be860befeffb3843c150f2343b64555ad8c438a400222056f6cc0 lib/core/convert.py
@@ -188,7 +188,7 @@ c4bfb493a03caf84dd362aec7c248097841de804b7413d0e1ecb8a90c8550bc0 lib/core/readl
188188
d1bd70c1a55858495c727fbec91e30af267459c8f64d50fabf9e4ee2c007e920 lib/core/replication.py
189189
1d0f80b0193ac5204527bfab4bde1a7aee0f693fd008e86b4b29f606d1ef94f3 lib/core/revision.py
190190
d2eb8e4b05ac93551272b3d4abfaf5b9f2d3ac92499a7704c16ed0b4f200db38 lib/core/session.py
191-
be3e2b9a16441137d97d366f09dfb14a39d89a258945d70413456fd42e85ca22 lib/core/settings.py
191+
d9a017b535ac6566d750ec4315e155cbe056fc39538b932071c1c546ec26d535 lib/core/settings.py
192192
1c5eab9494eb969bc9ce118a2ea6954690c6851cbe54c18373c723b99734bf09 lib/core/shell.py
193193
4eea6dcf023e41e3c64b210cb5c2efc7ca893b727f5e49d9c924f076bb224053 lib/core/subprocessng.py
194194
cdd352e1331c6b535e780f6edea79465cb55af53aa2114dcea0e8bf382e56d1a lib/core/target.py

‎lib/core/bigarray.py‎

Lines changed: 118 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
import itertools
1414
import os
15+
import shutil
1516
import sys
1617
import tempfile
1718
import threading
@@ -28,14 +29,23 @@
2829
except TypeError:
2930
DEFAULT_SIZE_OF = 16
3031

32+
try:
33+
# Python 2: basestring covers str and unicode
34+
STRING_TYPES = (basestring,)
35+
except NameError:
36+
# Python 3: str and bytes are separate
37+
STRING_TYPES = (str, bytes)
38+
3139
def _size_of(instance):
3240
"""
3341
Returns total size of a given instance / object (in bytes)
3442
"""
3543

3644
retval = sys.getsizeof(instance, DEFAULT_SIZE_OF)
3745

38-
if isinstance(instance, dict):
46+
if isinstance(instance, STRING_TYPES):
47+
return retval
48+
elif isinstance(instance, dict):
3949
retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(instance.items()))
4050
elif hasattr(instance, "__iter__"):
4151
retval += sum(_size_of(_) for _ in instance if _ != instance)
@@ -58,12 +68,29 @@ class BigArray(list):
5868
5969
>>> _ = BigArray(xrange(100000))
6070
>>> _[20] = 0
71+
>>> _[-1] = 999
6172
>>> _[99999]
62-
99999
73+
999
74+
>>> _[100000]
75+
Traceback (most recent call last):
76+
...
77+
IndexError: BigArray index out of range
6378
>>> _ += [0]
79+
>>> sum(_)
80+
4999850980
81+
>>> _[len(_) // 2] = 17
82+
>>> sum(_)
83+
4999800997
6484
>>> _[100000]
6585
0
66-
>>> _ = _ + [1]
86+
>>> _[0] = [None]
87+
>>> _.index(0)
88+
20
89+
>>> import pickle; __ = pickle.loads(pickle.dumps(_))
90+
>>> __.append(1)
91+
>>> len(_)
92+
100001
93+
>>> _ = __
6794
>>> _[-1]
6895
1
6996
>>> len([_ for _ in BigArray(xrange(100000))])
@@ -134,15 +161,23 @@ def index(self, value):
134161
if self[index] == value:
135162
return index
136163

137-
return ValueError, "%s is not in list" % value
164+
raise ValueError("%s is not in list" % value)
165+
166+
def __reduce__(self):
167+
return (self.__class__, (), self.__getstate__())
138168

139169
def close(self):
140-
while self.filenames:
141-
filename = self.filenames.pop()
142-
try:
143-
self._os_remove(filename)
144-
except OSError:
145-
pass
170+
with self._lock:
171+
while self.filenames:
172+
filename = self.filenames.pop()
173+
try:
174+
self._os_remove(filename)
175+
except OSError:
176+
pass
177+
self.chunks = [[]]
178+
self.cache = None
179+
self.chunk_length = getattr(sys, "maxsize", None)
180+
self._size_counter = 0
146181

147182
def __del__(self):
148183
self.close()
@@ -181,41 +216,89 @@ def _checkcache(self, index):
181216
raise SqlmapSystemException(errMsg)
182217

183218
def __getstate__(self):
184-
return self.chunks, self.filenames
219+
if self.cache and self.cache.dirty:
220+
filename = self._dump(self.cache.data)
221+
self.chunks[self.cache.index] = filename
222+
self.cache.dirty = False
223+
224+
return self.chunks, self.filenames, self.chunk_length
185225

186226
def __setstate__(self, state):
187227
self.__init__()
188-
self.chunks, self.filenames = state
228+
chunks, filenames, self.chunk_length = state
229+
230+
file_mapping = {}
231+
self.filenames = set()
232+
self.chunks = []
233+
234+
for filename in filenames:
235+
if not os.path.exists(filename):
236+
continue
237+
238+
try:
239+
handle, new_filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.BIG_ARRAY)
240+
os.close(handle)
241+
shutil.copyfile(filename, new_filename)
242+
self.filenames.add(new_filename)
243+
file_mapping[filename] = new_filename
244+
except (OSError, IOError):
245+
pass
246+
247+
for chunk in chunks:
248+
if isinstance(chunk, STRING_TYPES):
249+
if chunk in file_mapping:
250+
self.chunks.append(file_mapping[chunk])
251+
else:
252+
errMsg = "exception occurred while restoring BigArray chunk "
253+
errMsg += "from file '%s'" % chunk
254+
raise SqlmapSystemException(errMsg)
255+
else:
256+
self.chunks.append(chunk)
189257

190258
def __getitem__(self, y):
191-
length = len(self)
192-
if length == 0:
193-
raise IndexError("BigArray index out of range")
259+
with self._lock:
260+
length = len(self)
261+
if length == 0:
262+
raise IndexError("BigArray index out of range")
263+
264+
if y < 0:
265+
y += length
194266

195-
while y < 0:
196-
y += length
267+
if y < 0 or y >= length:
268+
raise IndexError("BigArray index out of range")
197269

198-
index = y // self.chunk_length
199-
offset = y % self.chunk_length
200-
chunk = self.chunks[index]
270+
index = y // self.chunk_length
271+
offset = y % self.chunk_length
272+
chunk = self.chunks[index]
201273

202-
if isinstance(chunk, list):
203-
return chunk[offset]
204-
else:
205-
self._checkcache(index)
206-
return self.cache.data[offset]
274+
if isinstance(chunk, list):
275+
return chunk[offset]
276+
else:
277+
self._checkcache(index)
278+
return self.cache.data[offset]
207279

208280
def __setitem__(self, y, value):
209-
index = y // self.chunk_length
210-
offset = y % self.chunk_length
211-
chunk = self.chunks[index]
212-
213-
if isinstance(chunk, list):
214-
chunk[offset] = value
215-
else:
216-
self._checkcache(index)
217-
self.cache.data[offset] = value
218-
self.cache.dirty = True
281+
with self._lock:
282+
length = len(self)
283+
if length == 0:
284+
raise IndexError("BigArray index out of range")
285+
286+
if y < 0:
287+
y += length
288+
289+
if y < 0 or y >= length:
290+
raise IndexError("BigArray index out of range")
291+
292+
index = y // self.chunk_length
293+
offset = y % self.chunk_length
294+
chunk = self.chunks[index]
295+
296+
if isinstance(chunk, list):
297+
chunk[offset] = value
298+
else:
299+
self._checkcache(index)
300+
self.cache.data[offset] = value
301+
self.cache.dirty = True
219302

220303
def __repr__(self):
221304
return "%s%s" % ("..." if len(self.chunks) > 1 else "", self.chunks[-1].__repr__())

‎lib/core/settings.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from thirdparty import six
2020

2121
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
22-
VERSION = "1.9.12.29"
22+
VERSION = "1.9.12.30"
2323
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
2424
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
2525
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -658,7 +658,7 @@
658658
BIGARRAY_CHUNK_SIZE = 1024 * 1024
659659

660660
# Compress level used for storing BigArray chunks to disk (0-9)
661-
BIGARRAY_COMPRESS_LEVEL = 9
661+
BIGARRAY_COMPRESS_LEVEL = 4
662662

663663
# Maximum number of socket pre-connects
664664
SOCKET_PRE_CONNECT_QUEUE_SIZE = 3

0 commit comments

Comments
 (0)