@@ -19,9 +19,9 @@ def open(filename, mode="rb", compresslevel=igzip._COMPRESS_LEVEL_TRADEOFF,
19
19
20
20
21
21
class ThreadedGzipReader (io .RawIOBase ):
22
- def __init__ (self , fp , queue_size = 8 , block_size = 128 * 1024 ):
22
+ def __init__ (self , fp , queue_size = 4 , block_size = 8 * 1024 * 1024 ):
23
23
self .raw = fp
24
- self .fileobj = igzip ._IGzipReader (fp , 512 * 1024 )
24
+ self .fileobj = igzip ._IGzipReader (fp , buffersize = 8 * 1024 * 1024 )
25
25
self .pos = 0
26
26
self .read_file = False
27
27
self .queue = queue .Queue (queue_size )
@@ -46,11 +46,9 @@ def _decompress(self):
46
46
return
47
47
block_queue .put (data )
48
48
49
- def read (self , size : int = - 1 ) -> bytes :
50
- if size < 0 :
51
- return self .readall ()
52
- data = self .buffer .read (size )
53
- if not data :
49
+ def readinto (self , b ):
50
+ result = self .buffer .readinto (b )
51
+ if result == 0 :
54
52
while True :
55
53
try :
56
54
data_from_queue = self .queue .get (timeout = 0.01 )
@@ -60,17 +58,10 @@ def read(self, size: int = -1) -> bytes:
60
58
if self .exception :
61
59
raise self .exception
62
60
# EOF reached
63
- return b""
61
+ return 0
64
62
self .buffer = io .BytesIO (data_from_queue )
65
- data = self .buffer .read (size )
66
- self .pos += len (data )
67
- return data
68
-
69
- def readinto (self , b ):
70
- with memoryview (b ) as view , view .cast ("B" ) as byte_view :
71
- data = self .read (len (byte_view ))
72
- byte_view [:len (data )] = data
73
- return len (data )
63
+ result = self .buffer .readinto (b )
64
+ return result
74
65
75
66
def readable (self ) -> bool :
76
67
return True
0 commit comments