Skip to content
This repository was archived by the owner on Oct 2, 2024. It is now read-only.

Commit 51bfa64

Browse files
wiktornwiktorn
committed
Introduce FileSlice
New class providing "windowed" view over the file objects. It gives the possibility to have a file-alike object that returns only some part of the file. Useful for multipart upload.
1 parent 2e590a6 commit 51bfa64

File tree

2 files changed

+301
-0
lines changed

2 files changed

+301
-0
lines changed

‎src/onedrivesdk/helpers/file_slice.py‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# -*- coding: utf-8 -*-
2+
'''
3+
The MIT License (MIT)
4+
5+
Copyright (c) 2015 Wiktor Niesiobędzki
6+
7+
Permission is hereby granted, free of charge, to any person obtaining a copy of
8+
this software and associated documentation files (the "Software"), to deal in
9+
the Software without restriction, including without limitation the rights to
10+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
11+
the Software, and to permit persons to whom the Software is furnished to do so,
12+
subject to the following conditions:
13+
14+
The above copyright notice and this permission notice shall be included in all
15+
copies or substantial portions of the Software.
16+
17+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
19+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
20+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
21+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
22+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23+
'''
24+
25+
import io
26+
import os
27+
28+
class FileSlice(io.RawIOBase):
29+
'''
30+
This class represents a window over a file handle. It will allow only access
31+
and read of bytes above start and no further than end/length bytes
32+
'''
33+
def __init__(self, handle, start, end=None, length=None):
34+
'''
35+
Creates new instance of FileSlice on file-like object handle. It behaves
36+
like normal file object, but only allow reading of bytes above start and
37+
no further than end/lenght byte
38+
39+
Args:
40+
handle (file-like object): file handle object to create a view over.
41+
File should be open in binary mode
42+
start (int): start byte number, makring the first byte that can be
43+
read from FileSlice
44+
end (int): Optional. Last byte of the file that can be read.
45+
length (int): Optional. Number of the bytes, starting from the
46+
start, that can be read from FileSlice
47+
48+
One of end or length must be provided
49+
'''
50+
assert end or length, "You need to provide one of end or length parameter"
51+
assert not (end and length), "You need to proivde only one parameter: end or length, not both"
52+
if start < 0:
53+
raise ValueError("Start of the file smaller than 0")
54+
if end and end < start:
55+
raise ValueError("End of the tile smaller than start")
56+
if length and length < 0:
57+
raise ValueError("Length smaller than 0")
58+
59+
self._handle = handle
60+
self._start = start
61+
if end:
62+
self._end = end
63+
else:
64+
self._end = start + length
65+
self._end = min(self._end, os.fstat(handle.fileno()).st_size)
66+
self.seek(0)
67+
68+
@property
69+
def _bytes_left(self):
70+
current_pos = self._handle.tell()
71+
return self._end - current_pos
72+
73+
def close(self):
74+
# do nothing, someone else might want to process this file
75+
return
76+
77+
@property
78+
def closed(self):
79+
return self._handle.closed
80+
81+
def fileno(self):
82+
return self._handle.fileno()
83+
84+
def flush(self):
85+
return self._handle.flush()
86+
87+
def len(self):
88+
# this is provided for requests, so it will properly recognize the size of the file
89+
return self._bytes_left
90+
91+
def __len__(self):
92+
# this is provided for requests, so it will properly recognize the size of the file
93+
return self.len()
94+
95+
def isatty(self):
96+
return self._handle.isatty()
97+
98+
def readable(self):
99+
return self._handle.readable()
100+
101+
def read(self, size=-1):
102+
if size == -1:
103+
read_size = self._bytes_left
104+
else:
105+
read_size = min(size, self._bytes_left)
106+
return self._handle.read(read_size)
107+
108+
def readall(self):
109+
return self._handle.read(self._bytes_left)
110+
111+
def readinto(self, b):
112+
if len(b) > self._bytes_left:
113+
r = self._handle.read(self._bytes_left)
114+
b[:len(r)] = r
115+
return len(r)
116+
return self._handle.readinto(b)
117+
118+
def readline(self, size=-1):
119+
return self._handle.readline(max(size, self._bytes_left))
120+
121+
def readlines(self, hint=-1):
122+
return self._handle.readlines(max(hint, self._bytes_left))
123+
124+
def seek(self, offset, whence=io.SEEK_SET):
125+
if whence == io.SEEK_SET:
126+
desired_pos = self._start + offset
127+
if whence == io.SEEK_CUR:
128+
desired_pos = self._handle.tell() + offset
129+
if whence == io.SEEK_END:
130+
desired_pos = self._end + offset
131+
132+
if desired_pos < self._start:
133+
raise ValueError("Seeking before the file slice")
134+
if desired_pos > self._end:
135+
raise ValueError("Seekeing past the end of file slice")
136+
137+
ret = self._handle.seek(desired_pos, io.SEEK_SET)
138+
if ret:
139+
return ret - self._start
140+
else:
141+
return ret
142+
143+
def seekable(self):
144+
return self._handle.seekable()
145+
146+
def tell(self):
147+
return self._handle.tell() - self._start
148+
149+
def truncate(self, size=None):
150+
raise IOError("Operation not supported")
151+
152+
def writable(self):
153+
return False
154+
155+
def write(self, b):
156+
raise IOError("Operation not supported")
157+
158+
def writelines(self, lines):
159+
raise IOError("Operation not supported")
160+

‎testonedrivesdk/test_file_slice.py‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import unittest
2+
try:
3+
from unittest.mock import patch, mock_open
4+
except ImportError:
5+
from mock import patch, mock_open
6+
7+
import io
8+
import os
9+
import math
10+
import tempfile
11+
12+
from onedrivesdk.helpers.file_slice import FileSlice
13+
14+
class TestFileSlice(unittest.TestCase):
15+
def testSliceFileStartEnd(self):
16+
with tempfile.TemporaryFile() as f:
17+
f.write(b'123456789')
18+
f.flush()
19+
part = FileSlice(f, 0, 5)
20+
self.assertEqual(len(part), 5)
21+
self.assertEqual(part.read(), b'12345')
22+
self.assertEqual(part.read(3), b'')
23+
part.seek(0, io.SEEK_SET)
24+
self.assertEqual(part.read(3), b'123')
25+
self.assertEqual(part.tell(), 3)
26+
part.seek(-3, io.SEEK_CUR)
27+
self.assertEqual(part.tell(), 0)
28+
part.seek(-2, io.SEEK_END)
29+
self.assertEqual(part.tell(), 3)
30+
self.assertEqual(part.readall(), b'45')
31+
with self.assertRaises(IOError):
32+
part.write('abc')
33+
with self.assertRaises(IOError):
34+
part.writelines(['foo', 'bar'])
35+
36+
def testSliceFileStartLength(self):
37+
with tempfile.TemporaryFile() as f:
38+
f.write(b'123456789')
39+
f.flush()
40+
part = FileSlice(f, 0, length=5)
41+
self.assertEqual(len(part), 5)
42+
self.assertEqual(part.read(), b'12345')
43+
self.assertEqual(part.read(3), b'')
44+
part.seek(0)
45+
self.assertEqual(part.read(3), b'123')
46+
self.assertEqual(part.tell(), 3)
47+
part.seek(-3, io.SEEK_CUR)
48+
self.assertEqual(part.readall(), b'12345')
49+
with self.assertRaises(IOError):
50+
part.write('abc')
51+
with self.assertRaises(IOError):
52+
part.writelines(['foo', 'bar'])
53+
54+
def testSliceFileMiddleStartEnd(self):
55+
with tempfile.TemporaryFile() as f:
56+
f.write(b'123456789')
57+
f.flush()
58+
part = FileSlice(f, 1, 5)
59+
self.assertEqual(len(part), 4)
60+
self.assertEqual(part.read(3), b'234')
61+
self.assertEqual(part.readall(), b'5')
62+
self.assertEqual(part.read(), b'')
63+
self.assertEqual(part.tell(), 4)
64+
65+
def testSliceFileMiddleStartLength(self):
66+
with tempfile.TemporaryFile() as f:
67+
f.write(b'123456789')
68+
f.flush()
69+
part = FileSlice(f, 1, length=5)
70+
self.assertEqual(len(part), 5)
71+
self.assertEqual(part.read(3), b'234')
72+
self.assertEqual(part.readall(), b'56')
73+
self.assertEqual(part.read(), b'')
74+
self.assertEqual(part.tell(), 5)
75+
76+
def testSliceFileMiddleStartEnd_afterEOF(self):
77+
with tempfile.TemporaryFile() as f:
78+
f.write(b'123456789')
79+
f.flush()
80+
part = FileSlice(f, 8, 15)
81+
self.assertEqual(len(part), 1)
82+
self.assertEqual(part.read(3), b'9')
83+
self.assertEqual(part.readall(), b'')
84+
self.assertEqual(part.read(), b'')
85+
self.assertEqual(part.tell(), 1)
86+
part.seek(-1, io.SEEK_END)
87+
self.assertEqual(part.tell(), 0)
88+
self.assertEqual(part.readall(), b'9')
89+
90+
def testSliceFileMiddleStartLength_afterEOF(self):
91+
with tempfile.TemporaryFile() as f:
92+
f.write(b'123456789')
93+
f.flush()
94+
part = FileSlice(f, 8, length=15)
95+
self.assertEqual(len(part), 1)
96+
self.assertEqual(part.read(3), b'9')
97+
self.assertEqual(part.readall(), b'')
98+
self.assertEqual(part.read(), b'')
99+
self.assertEqual(part.tell(), 1)
100+
part.seek(0)
101+
self.assertEqual(part.tell(), 0)
102+
self.assertEqual(part.readall(), b'9')
103+
104+
105+
def testSeek(self):
106+
with tempfile.TemporaryFile() as f:
107+
f.write(b'123456789')
108+
f.flush()
109+
part = FileSlice(f, 2, 7)
110+
part.seek(3)
111+
part.seek(part.tell(), io.SEEK_SET)
112+
self.assertEqual(part.tell(), 3)
113+
114+
def testSanityChecks(self):
115+
with tempfile.TemporaryFile() as f:
116+
f.write(b'123456789')
117+
f.flush()
118+
with self.assertRaises(ValueError):
119+
part = FileSlice(f, -5, -2)
120+
with self.assertRaises(ValueError):
121+
part = FileSlice(f, 0, -2)
122+
with self.assertRaises(ValueError):
123+
part = FileSlice(f, -10, 2)
124+
with self.assertRaises(ValueError):
125+
part = FileSlice(f, 10, 2)
126+
with self.assertRaises(ValueError):
127+
part = FileSlice(f, 10, length=-2)
128+
129+
part = FileSlice(f, 1, 5)
130+
with self.assertRaises(ValueError):
131+
part.seek(8)
132+
with self.assertRaises(ValueError):
133+
part.seek(8, io.SEEK_SET)
134+
part.seek(3)
135+
with self.assertRaises(ValueError):
136+
part.seek(4, io.SEEK_CUR)
137+
with self.assertRaises(ValueError):
138+
part.seek(-5, io.SEEK_END)
139+
140+
if __name__ == '__main__':
141+
unittest.main()

0 commit comments

Comments
 (0)