10
10
"""
11
11
12
12
import sys
13
- if sys .version_info [0 ] < 3 :
14
- from bz2file import BZ2File
15
- else :
16
- from bz2 import BZ2File
13
+ from bz2 import BZ2File
17
14
import gzip
18
15
import sys
19
16
import warnings
43
40
HAVE_INDEXED_GZIP = False
44
41
45
42
46
- # The largest memory chunk that gzip can use for reads
47
- GZIP_MAX_READ_CHUNK = 100 * 1024 * 1024 # 100Mb
48
-
49
-
50
- class BufferedGzipFile (gzip .GzipFile ):
51
- """GzipFile able to readinto buffer >= 2**32 bytes.
52
-
53
- This class only differs from gzip.GzipFile
54
- in Python 3.5.0.
55
-
56
- This works around a known issue in Python 3.5.
57
- See https://bugs.python.org/issue25626
58
- """
59
-
60
- # This helps avoid defining readinto in Python 2.6,
61
- # where it is undefined on gzip.GzipFile.
62
- # It also helps limit the exposure to this code.
63
- if sys .version_info [:3 ] == (3 , 5 , 0 ):
64
- def __init__ (self , fileish , mode = 'rb' , compresslevel = 9 ,
65
- buffer_size = 2 ** 32 - 1 ):
66
- super (BufferedGzipFile , self ).__init__ (fileish , mode = mode ,
67
- compresslevel = compresslevel )
68
- self .buffer_size = buffer_size
69
-
70
- def readinto (self , buf ):
71
- """Uses self.buffer_size to do a buffered read."""
72
- n_bytes = len (buf )
73
- if n_bytes < 2 ** 32 :
74
- return super (BufferedGzipFile , self ).readinto (buf )
75
-
76
- # This works around a known issue in Python 3.5.
77
- # See https://bugs.python.org/issue25626
78
- mv = memoryview (buf )
79
- n_read = 0
80
- max_read = 2 ** 32 - 1 # Max for unsigned 32-bit integer
81
- while (n_read < n_bytes ):
82
- n_wanted = min (n_bytes - n_read , max_read )
83
- n_got = super (BufferedGzipFile , self ).readinto (
84
- mv [n_read :n_read + n_wanted ])
85
- n_read += n_got
86
- if n_got != n_wanted :
87
- break
88
- return n_read
89
-
90
-
91
43
def _gzip_open (filename , mode = 'rb' , compresslevel = 9 , keep_open = False ):
92
44
93
45
# use indexed_gzip if possible for faster read access. If keep_open ==
@@ -96,16 +48,9 @@ def _gzip_open(filename, mode='rb', compresslevel=9, keep_open=False):
96
48
if HAVE_INDEXED_GZIP and mode == 'rb' :
97
49
gzip_file = IndexedGzipFile (filename , drop_handles = not keep_open )
98
50
99
- # Fall-back to built-in GzipFile (wrapped with the BufferedGzipFile class
100
- # defined above)
51
+ # Fall-back to built-in GzipFile
101
52
else :
102
- gzip_file = BufferedGzipFile (filename , mode , compresslevel )
103
-
104
- # Speedup for #209, for versions of python < 3.5. Open gzip files with
105
- # faster reads on large files using a larger read buffer. See
106
- # https://github.com/nipy/nibabel/pull/210 for discussion
107
- if hasattr (gzip_file , 'max_read_chunk' ):
108
- gzip_file .max_read_chunk = GZIP_MAX_READ_CHUNK
53
+ gzip_file = gzip .GzipFile (filename , mode , compresslevel )
109
54
110
55
return gzip_file
111
56
0 commit comments