-
-
Notifications
You must be signed in to change notification settings - Fork 33k
GH-139416: Fix portability of sendfile(2) support detection for Lustre filesystems #139417
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 11 commits
ee336eb
646868a
520ee09
a09ff4d
5917a6b
452c877
faa9229
482fb09
883f158
1316d6a
3426f7b
1b88269
1513ea3
f59253b
44c0184
ef900b0
4987596
e624cb5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
@@ -213,6 +213,23 @@ def _fastcopy_sendfile(fsrc, fdst): | |||||||||||||||||||||||||||||||||||||
_USE_CP_SENDFILE = False | ||||||||||||||||||||||||||||||||||||||
raise _GiveupOnFastCopy(err) | ||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||
if err.errno == errno.ENODATA: | ||||||||||||||||||||||||||||||||||||||
# In rare cases, sendfile() on Linux Lustre returns | ||||||||||||||||||||||||||||||||||||||
# ENODATA. | ||||||||||||||||||||||||||||||||||||||
_USE_CP_SENDFILE = False | ||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||
dstpos = os.lseek(outfd, 0, os.SEEK_CUR) | ||||||||||||||||||||||||||||||||||||||
if dstpos > 0: | ||||||||||||||||||||||||||||||||||||||
picnixz marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||||||||||||||||||||||||||||||||||
# Some data has already been written but we use | ||||||||||||||||||||||||||||||||||||||
# sendfile in a mode that does not update the | ||||||||||||||||||||||||||||||||||||||
# input fd position when reading. Hence seek the | ||||||||||||||||||||||||||||||||||||||
# input fd to the correct position before falling | ||||||||||||||||||||||||||||||||||||||
# back on POSIX read/write method. Since sendfile | ||||||||||||||||||||||||||||||||||||||
# requires mmapable infd, it should also be seekable | ||||||||||||||||||||||||||||||||||||||
os.lseek(infd, dstpos, os.SEEK_SET) | ||||||||||||||||||||||||||||||||||||||
Comment on lines
223
to
230
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
AFAIK, There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I will check this but I think this would not work, because offset is updated only at the bottom with the return value of sendfile() (which is the number of bytes actually sent). If are catching the exception then no return value (I think it would be available in C). So this is why the offset is not used in in the next condition but rather the return of lseek, even if it is just to check no data have been written. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the comments. I think I would leave the |
||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||
raise _GiveupOnFastCopy(err) | ||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||
if err.errno == errno.ENOSPC: # filesystem is full | ||||||||||||||||||||||||||||||||||||||
raise err from None | ||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3365,6 +3365,40 @@ def test_file2file_not_supported(self): | |
finally: | ||
shutil._USE_CP_SENDFILE = True | ||
|
||
def test_exception_on_enodata_call(self): | ||
# Test logic when sendfile(2) call returns ENODATA error on | ||
# the not-first call on the file and we need to fall back to | ||
# traditional POSIX while preserving the position of where we | ||
# got to in writing | ||
def syscall(*args, **kwargs): | ||
if not flag: | ||
flag.append(None) | ||
return orig_syscall(*args, **kwargs) | ||
else: | ||
raise OSError(errno.ENODATA, "yo") | ||
bnikolic marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
||
flag = [] | ||
orig_syscall = eval(self.PATCHPOINT) | ||
# Reduce block size so that multiple syscalls are needed | ||
mock = unittest.mock.Mock() | ||
mock.st_size = 65536 + 1 | ||
bnikolic marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
with unittest.mock.patch('os.fstat', return_value=mock) as m: | ||
with unittest.mock.patch(self.PATCHPOINT, create=True, | ||
side_effect=syscall) as m2: | ||
with self.get_files() as (src, dst): | ||
with self.assertRaises(_GiveupOnFastCopy) as cm: | ||
self.zerocopy_fun(src, dst) | ||
|
||
|
||
# Reset flag so that second syscall fails again | ||
flag = [] | ||
bnikolic marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
with unittest.mock.patch(self.PATCHPOINT, create=True, | ||
side_effect=syscall) as m2: | ||
shutil._USE_CP_SENDFILE = True | ||
shutil.copyfile(TESTFN, TESTFN2) | ||
assert m2.called | ||
bnikolic marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
shutil._USE_CP_SENDFILE = True | ||
assert flag | ||
bnikolic marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA) | ||
|
||
@unittest.skipUnless(shutil._USE_CP_COPY_FILE_RANGE, "os.copy_file_range() not supported") | ||
class TestZeroCopyCopyFileRange(_ZeroCopyFileLinuxTest, unittest.TestCase): | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
:func:`shutil.copyfile`: Detect problem seen on Lustre filesystems | ||
giving "Errno 61" due to the sendfile(2) optimised implementation and | ||
fall back to the Posix standard read/write implementation. |
Uh oh!
There was an error while loading. Please reload this page.