Skip to content
This repository was archived by the owner on Oct 2, 2024. It is now read-only.

Commit 510e007

Browse files
wiktornwiktorn
committed
Introduce FileSlice
New class providing "windowed" view over the file objects. It gives the possibility to have a file-alike object that returns only some part of the file. Useful for multipart upload.
1 parent 4609413 commit 510e007

File tree

2 files changed

+302
-0
lines changed

2 files changed

+302
-0
lines changed

‎src/onedrivesdk/helpers/file_slice.py‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
# -*- coding: utf-8 -*-
2+
'''
3+
------------------------------------------------------------------------------
4+
Copyright (c) 2015 Microsoft Corporation
5+
6+
Permission is hereby granted, free of charge, to any person obtaining a copy
7+
of this software and associated documentation files (the "Software"), to deal
8+
in the Software without restriction, including without limitation the rights
9+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
copies of the Software, and to permit persons to whom the Software is
11+
furnished to do so, subject to the following conditions:
12+
13+
The above copyright notice and this permission notice shall be included in
14+
all copies or substantial portions of the Software.
15+
16+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22+
THE SOFTWARE.
23+
------------------------------------------------------------------------------
24+
'''
25+
26+
import io
27+
import os
28+
29+
class FileSlice(io.RawIOBase):
30+
'''
31+
This class represents a window over a file handle. It will allow only access
32+
and read of bytes above start and no further than end/length bytes
33+
'''
34+
def __init__(self, handle, start, end=None, length=None):
35+
'''
36+
Creates new instance of FileSlice on file-like object handle. It behaves
37+
like normal file object, but only allow reading of bytes above start and
38+
no further than end/lenght byte
39+
40+
Args:
41+
handle (file-like object): file handle object to create a view over.
42+
File should be open in binary mode
43+
start (int): start byte number, makring the first byte that can be
44+
read from FileSlice
45+
end (int): Optional. Last byte of the file that can be read.
46+
length (int): Optional. Number of the bytes, starting from the
47+
start, that can be read from FileSlice
48+
49+
One of end or length must be provided
50+
'''
51+
assert end or length, "You need to provide one of end or length parameter"
52+
assert not (end and length), "You need to proivde only one parameter: end or length, not both"
53+
if start < 0:
54+
raise ValueError("Start of the file smaller than 0")
55+
if end and end < start:
56+
raise ValueError("End of the tile smaller than start")
57+
if length and length < 0:
58+
raise ValueError("Length smaller than 0")
59+
60+
self._handle = handle
61+
self._start = start
62+
if end:
63+
self._end = end
64+
else:
65+
self._end = start + length
66+
self._end = min(self._end, os.fstat(handle.fileno()).st_size)
67+
self.seek(0)
68+
69+
@property
70+
def _bytes_left(self):
71+
current_pos = self._handle.tell()
72+
return self._end - current_pos
73+
74+
def close(self):
75+
# do nothing, someone else might want to process this file
76+
return
77+
78+
@property
79+
def closed(self):
80+
return self._handle.closed
81+
82+
def fileno(self):
83+
return self._handle.fileno()
84+
85+
def flush(self):
86+
return self._handle.flush()
87+
88+
def len(self):
89+
# this is provided for requests, so it will properly recognize the size of the file
90+
return self._bytes_left
91+
92+
def __len__(self):
93+
# this is provided for requests, so it will properly recognize the size of the file
94+
return self.len()
95+
96+
def isatty(self):
97+
return self._handle.isatty()
98+
99+
def readable(self):
100+
return self._handle.readable()
101+
102+
def read(self, size=-1):
103+
if size == -1:
104+
read_size = self._bytes_left
105+
else:
106+
read_size = min(size, self._bytes_left)
107+
return self._handle.read(read_size)
108+
109+
def readall(self):
110+
return self._handle.read(self._bytes_left)
111+
112+
def readinto(self, b):
113+
if len(b) > self._bytes_left:
114+
r = self._handle.read(self._bytes_left)
115+
b[:len(r)] = r
116+
return len(r)
117+
return self._handle.readinto(b)
118+
119+
def readline(self, size=-1):
120+
return self._handle.readline(max(size, self._bytes_left))
121+
122+
def readlines(self, hint=-1):
123+
return self._handle.readlines(max(hint, self._bytes_left))
124+
125+
def seek(self, offset, whence=io.SEEK_SET):
126+
if whence == io.SEEK_SET:
127+
desired_pos = self._start + offset
128+
if whence == io.SEEK_CUR:
129+
desired_pos = self._handle.tell() + offset
130+
if whence == io.SEEK_END:
131+
desired_pos = self._end + offset
132+
133+
if desired_pos < self._start:
134+
raise ValueError("Seeking before the file slice")
135+
if desired_pos > self._end:
136+
raise ValueError("Seekeing past the end of file slice")
137+
138+
ret = self._handle.seek(desired_pos, io.SEEK_SET)
139+
if ret:
140+
return ret - self._start
141+
else:
142+
return ret
143+
144+
def seekable(self):
145+
return self._handle.seekable()
146+
147+
def tell(self):
148+
return self._handle.tell() - self._start
149+
150+
def truncate(self, size=None):
151+
raise IOError("Operation not supported")
152+
153+
def writable(self):
154+
return False
155+
156+
def write(self, b):
157+
raise IOError("Operation not supported")
158+
159+
def writelines(self, lines):
160+
raise IOError("Operation not supported")
161+

‎testonedrivesdk/test_file_slice.py‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import unittest
2+
try:
3+
from unittest.mock import patch, mock_open
4+
except ImportError:
5+
from mock import patch, mock_open
6+
7+
import io
8+
import os
9+
import math
10+
import tempfile
11+
12+
from onedrivesdk.helpers.file_slice import FileSlice
13+
14+
class TestFileSlice(unittest.TestCase):
15+
def testSliceFileStartEnd(self):
16+
with tempfile.TemporaryFile() as f:
17+
f.write(b'123456789')
18+
f.flush()
19+
part = FileSlice(f, 0, 5)
20+
self.assertEqual(len(part), 5)
21+
self.assertEqual(part.read(), b'12345')
22+
self.assertEqual(part.read(3), b'')
23+
part.seek(0, io.SEEK_SET)
24+
self.assertEqual(part.read(3), b'123')
25+
self.assertEqual(part.tell(), 3)
26+
part.seek(-3, io.SEEK_CUR)
27+
self.assertEqual(part.tell(), 0)
28+
part.seek(-2, io.SEEK_END)
29+
self.assertEqual(part.tell(), 3)
30+
self.assertEqual(part.readall(), b'45')
31+
with self.assertRaises(IOError):
32+
part.write('abc')
33+
with self.assertRaises(IOError):
34+
part.writelines(['foo', 'bar'])
35+
36+
def testSliceFileStartLength(self):
37+
with tempfile.TemporaryFile() as f:
38+
f.write(b'123456789')
39+
f.flush()
40+
part = FileSlice(f, 0, length=5)
41+
self.assertEqual(len(part), 5)
42+
self.assertEqual(part.read(), b'12345')
43+
self.assertEqual(part.read(3), b'')
44+
part.seek(0)
45+
self.assertEqual(part.read(3), b'123')
46+
self.assertEqual(part.tell(), 3)
47+
part.seek(-3, io.SEEK_CUR)
48+
self.assertEqual(part.readall(), b'12345')
49+
with self.assertRaises(IOError):
50+
part.write('abc')
51+
with self.assertRaises(IOError):
52+
part.writelines(['foo', 'bar'])
53+
54+
def testSliceFileMiddleStartEnd(self):
55+
with tempfile.TemporaryFile() as f:
56+
f.write(b'123456789')
57+
f.flush()
58+
part = FileSlice(f, 1, 5)
59+
self.assertEqual(len(part), 4)
60+
self.assertEqual(part.read(3), b'234')
61+
self.assertEqual(part.readall(), b'5')
62+
self.assertEqual(part.read(), b'')
63+
self.assertEqual(part.tell(), 4)
64+
65+
def testSliceFileMiddleStartLength(self):
66+
with tempfile.TemporaryFile() as f:
67+
f.write(b'123456789')
68+
f.flush()
69+
part = FileSlice(f, 1, length=5)
70+
self.assertEqual(len(part), 5)
71+
self.assertEqual(part.read(3), b'234')
72+
self.assertEqual(part.readall(), b'56')
73+
self.assertEqual(part.read(), b'')
74+
self.assertEqual(part.tell(), 5)
75+
76+
def testSliceFileMiddleStartEnd_afterEOF(self):
77+
with tempfile.TemporaryFile() as f:
78+
f.write(b'123456789')
79+
f.flush()
80+
part = FileSlice(f, 8, 15)
81+
self.assertEqual(len(part), 1)
82+
self.assertEqual(part.read(3), b'9')
83+
self.assertEqual(part.readall(), b'')
84+
self.assertEqual(part.read(), b'')
85+
self.assertEqual(part.tell(), 1)
86+
part.seek(-1, io.SEEK_END)
87+
self.assertEqual(part.tell(), 0)
88+
self.assertEqual(part.readall(), b'9')
89+
90+
def testSliceFileMiddleStartLength_afterEOF(self):
91+
with tempfile.TemporaryFile() as f:
92+
f.write(b'123456789')
93+
f.flush()
94+
part = FileSlice(f, 8, length=15)
95+
self.assertEqual(len(part), 1)
96+
self.assertEqual(part.read(3), b'9')
97+
self.assertEqual(part.readall(), b'')
98+
self.assertEqual(part.read(), b'')
99+
self.assertEqual(part.tell(), 1)
100+
part.seek(0)
101+
self.assertEqual(part.tell(), 0)
102+
self.assertEqual(part.readall(), b'9')
103+
104+
105+
def testSeek(self):
106+
with tempfile.TemporaryFile() as f:
107+
f.write(b'123456789')
108+
f.flush()
109+
part = FileSlice(f, 2, 7)
110+
part.seek(3)
111+
part.seek(part.tell(), io.SEEK_SET)
112+
self.assertEqual(part.tell(), 3)
113+
114+
def testSanityChecks(self):
115+
with tempfile.TemporaryFile() as f:
116+
f.write(b'123456789')
117+
f.flush()
118+
with self.assertRaises(ValueError):
119+
part = FileSlice(f, -5, -2)
120+
with self.assertRaises(ValueError):
121+
part = FileSlice(f, 0, -2)
122+
with self.assertRaises(ValueError):
123+
part = FileSlice(f, -10, 2)
124+
with self.assertRaises(ValueError):
125+
part = FileSlice(f, 10, 2)
126+
with self.assertRaises(ValueError):
127+
part = FileSlice(f, 10, length=-2)
128+
129+
part = FileSlice(f, 1, 5)
130+
with self.assertRaises(ValueError):
131+
part.seek(8)
132+
with self.assertRaises(ValueError):
133+
part.seek(8, io.SEEK_SET)
134+
part.seek(3)
135+
with self.assertRaises(ValueError):
136+
part.seek(4, io.SEEK_CUR)
137+
with self.assertRaises(ValueError):
138+
part.seek(-5, io.SEEK_END)
139+
140+
if __name__ == '__main__':
141+
unittest.main()

0 commit comments

Comments
 (0)