Skip to content

Commit 86adef5

Browse files
BCSharpBCSharp
committed
Use PosixFileStream for files on POSIX
1 parent e0bb6ad commit 86adef5

File tree

7 files changed

+525
-96
lines changed

7 files changed

+525
-96
lines changed

‎Src/IronPython.Modules/mmap.cs‎

Lines changed: 101 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
using System.Numerics;
1616
using System.Runtime.CompilerServices;
1717
using System.Runtime.InteropServices;
18+
using System.Runtime.Serialization;
19+
using System.Runtime.Versioning;
1820
using System.Text;
1921
using System.Threading;
2022

@@ -24,6 +26,7 @@
2426
using IronPython.Runtime.Types;
2527

2628
using Microsoft.Scripting.Utils;
29+
using Microsoft.Win32.SafeHandles;
2730

2831
[assembly: PythonModule("mmap", typeof(IronPython.Modules.MmapModule))]
2932
namespace IronPython.Modules {
@@ -92,6 +95,7 @@ public class MmapDefault : IWeakReferenceable {
9295
private readonly long _offset;
9396
private readonly string _mapName;
9497
private readonly MemoryMappedFileAccess _fileAccess;
98+
private readonly SafeFileHandle _handle; // only used on some POSIX platforms, null otherwise
9599

96100
private volatile bool _isClosed;
97101
private int _refCount = 1;
@@ -148,46 +152,65 @@ public MmapDefault(CodeContext/*!*/ context, int fileno, long length, string tag
148152

149153
PythonContext pContext = context.LanguageContext;
150154
if (pContext.FileManager.TryGetStreams(fileno, out StreamBox streams)) {
151-
if ((_sourceStream = streams.ReadStream as FileStream) == null) {
152-
throw WindowsError(PythonExceptions._OSError.ERROR_INVALID_HANDLE);
155+
Stream stream = streams.ReadStream;
156+
if (stream is FileStream fs) {
157+
_sourceStream = fs;
158+
} else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) {
159+
// use file descriptor
160+
#if NET8_0_OR_GREATER
161+
CheckFileAccess(_fileAccess, stream);
162+
_handle = new SafeFileHandle((IntPtr)fileno, ownsHandle: false);
163+
_file = MemoryMappedFile.CreateFromFile(_handle, _mapName, length, _fileAccess, HandleInheritability.None, leaveOpen: true);
164+
#else
165+
_handle = new SafeFileHandle((IntPtr)fileno, ownsHandle: false);
166+
FileAccess fileAccess = stream.CanWrite ? stream.CanRead ? FileAccess.ReadWrite : FileAccess.Write : FileAccess.Read;
167+
// This may or may not work on Mono, but on Mono streams.ReadStream is FileStream (unless dupped in some cases)
168+
_sourceStream = new FileStream(_handle, fileAccess);
169+
#endif
153170
}
171+
// otherwise leaves _file as null and _sourceStream as null
154172
} else {
155173
throw PythonOps.OSError(PythonExceptions._OSError.ERROR_INVALID_BLOCK, "Bad file descriptor");
156174
}
157175

158-
if (_fileAccess == MemoryMappedFileAccess.ReadWrite && !_sourceStream.CanWrite) {
159-
throw WindowsError(PythonExceptions._OSError.ERROR_ACCESS_DENIED);
160-
}
176+
if (_file is null) {
177+
// create _file form _sourceStream
178+
if (_sourceStream is null) {
179+
throw WindowsError(PythonExceptions._OSError.ERROR_INVALID_HANDLE);
180+
}
181+
182+
CheckFileAccess(_fileAccess, _sourceStream);
161183

162-
if (length == 0) {
163-
length = _sourceStream.Length;
164184
if (length == 0) {
165-
throw PythonOps.ValueError("cannot mmap an empty file");
166-
}
167-
if (_offset >= length) {
168-
throw PythonOps.ValueError("mmap offset is greater than file size");
185+
length = _sourceStream.Length;
186+
if (length == 0) {
187+
throw PythonOps.ValueError("cannot mmap an empty file");
188+
}
189+
if (_offset >= length) {
190+
throw PythonOps.ValueError("mmap offset is greater than file size");
191+
}
192+
length -= _offset;
169193
}
170-
length -= _offset;
171-
}
172194

173-
long capacity = checked(_offset + length);
195+
long capacity = checked(_offset + length);
174196

175-
// Enlarge the file as needed.
176-
if (capacity > _sourceStream.Length) {
177-
if (_sourceStream.CanWrite) {
178-
_sourceStream.SetLength(capacity);
179-
} else {
180-
throw WindowsError(PythonExceptions._OSError.ERROR_NOT_ENOUGH_MEMORY);
197+
// Enlarge the file as needed.
198+
if (capacity > _sourceStream.Length) {
199+
if (_sourceStream.CanWrite) {
200+
_sourceStream.SetLength(capacity);
201+
} else {
202+
throw WindowsError(PythonExceptions._OSError.ERROR_NOT_ENOUGH_MEMORY);
203+
}
181204
}
182-
}
183205

184-
_file = CreateFromFile(
185-
_sourceStream,
186-
_mapName,
187-
_sourceStream.Length,
188-
_fileAccess,
189-
HandleInheritability.None,
190-
true);
206+
_file = CreateFromFile(
207+
_sourceStream,
208+
_mapName,
209+
_sourceStream.Length,
210+
_fileAccess,
211+
HandleInheritability.None,
212+
true);
213+
}
191214
}
192215

193216
try {
@@ -198,7 +221,24 @@ public MmapDefault(CodeContext/*!*/ context, int fileno, long length, string tag
198221
throw;
199222
}
200223
_position = 0L;
201-
}
224+
225+
void CheckFileAccess(MemoryMappedFileAccess mmapAccess, Stream stream) {
226+
bool isValid = mmapAccess switch {
227+
MemoryMappedFileAccess.Read => stream.CanRead,
228+
MemoryMappedFileAccess.ReadWrite => stream.CanRead && stream.CanWrite,
229+
MemoryMappedFileAccess.CopyOnWrite => stream.CanRead,
230+
_ => false
231+
};
232+
233+
if (!isValid) {
234+
if (_handle is not null && _sourceStream is not null) {
235+
_sourceStream.Dispose();
236+
}
237+
throw PythonOps.OSError(PythonExceptions._OSError.ERROR_ACCESS_DENIED, "Invalid access mode");
238+
}
239+
}
240+
} // end of constructor
241+
202242

203243
public object __len__() {
204244
using (new MmapLocker(this)) {
@@ -325,6 +365,11 @@ private void CloseWorker() {
325365
_view.Flush();
326366
_view.Dispose();
327367
_file.Dispose();
368+
if (_handle is not null) {
369+
// mmap owns _sourceStream too in this case
370+
_sourceStream?.Dispose();
371+
_handle.Dispose();
372+
}
328373
_sourceStream = null;
329374
_view = null;
330375
_file = null;
@@ -557,6 +602,11 @@ public void resize(long newsize) {
557602
}
558603

559604
if (_sourceStream == null) {
605+
if (_handle is not null && !_handle.IsInvalid
606+
&& (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))) {
607+
// resize on Posix platforms
608+
PythonNT.ftruncateUnix(unchecked((int)_handle.DangerousGetHandle()), newsize);
609+
}
560610
// resizing is not supported without an underlying file
561611
throw WindowsError(PythonExceptions._OSError.ERROR_INVALID_PARAMETER);
562612
}
@@ -716,6 +766,9 @@ public void seek(long pos, int whence = SEEK_SET) {
716766

717767
public object size() {
718768
using (new MmapLocker(this)) {
769+
if (_handle is not null && (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))) {
770+
return GetFileSizeUnix(_handle);
771+
}
719772
if (_sourceStream == null) return ReturnLong(_view.Capacity);
720773
return ReturnLong(new FileInfo(_sourceStream.Name).Length);
721774
}
@@ -830,6 +883,25 @@ internal Bytes GetSearchString() {
830883
}
831884
}
832885

886+
[SupportedOSPlatform("linux"), SupportedOSPlatform("macos")]
887+
private static long GetFileSizeUnix(SafeFileHandle handle) {
888+
long size;
889+
if (handle.IsInvalid) {
890+
throw PythonOps.OSError(PythonExceptions._OSError.ERROR_INVALID_HANDLE, "Invalid file handle");
891+
}
892+
893+
if (Mono.Unix.Native.Syscall.fstat((int)handle.DangerousGetHandle(), out Mono.Unix.Native.Stat status) == 0) {
894+
size = status.st_size;
895+
} else {
896+
Mono.Unix.Native.Errno errno = Mono.Unix.Native.Stdlib.GetLastError();
897+
string msg = Mono.Unix.UnixMarshal.GetErrorDescription(errno);
898+
int error = Mono.Unix.Native.NativeConvert.FromErrno(errno);
899+
throw PythonOps.OSError(error, msg);
900+
}
901+
902+
return size;
903+
}
904+
833905
#endregion
834906

835907
#region Synchronization

‎Src/IronPython.Modules/nt.cs‎

Lines changed: 79 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -431,22 +431,41 @@ public static int dup2(CodeContext/*!*/ context, int fd, int fd2) {
431431
}
432432

433433

434+
[SupportedOSPlatform("linux"), SupportedOSPlatform("osx")]
434435
private static int UnixDup(int fd, int fd2, out Stream? stream) {
435436
int res = fd2 < 0 ? Mono.Unix.Native.Syscall.dup(fd) : Mono.Unix.Native.Syscall.dup2(fd, fd2);
436437
if (res < 0) throw GetLastUnixError();
437438
if (ClrModule.IsMono) {
438-
// This does not work on .NET, probably because .NET FileStream is not aware of Mono.Unix.UnixStream
439-
stream = new Mono.Unix.UnixStream(res, ownsHandle: true);
439+
// Elaborate workaround on Mono to avoid UnixStream as out
440+
stream = new Mono.Unix.UnixStream(res, ownsHandle: false);
441+
FileAccess fileAccess = stream.CanRead ? stream.CanWrite ? FileAccess.ReadWrite : FileAccess.Read : FileAccess.Write;
442+
stream.Dispose();
443+
try {
444+
// FileStream on Mono created with a file descriptor might not work: https://github.com/mono/mono/issues/12783
445+
// Test if it does, without closing the handle if it doesn't
446+
var sfh = new SafeFileHandle((IntPtr)res, ownsHandle: false);
447+
stream = new FileStream(sfh, fileAccess);
448+
// No exception? Great! We can use FileStream.
449+
stream.Dispose();
450+
sfh.Dispose();
451+
stream = null; // Create outside of try block
452+
} catch (IOException) {
453+
// Fall back to UnixStream
454+
stream = new Mono.Unix.UnixStream(res, ownsHandle: true);
455+
}
456+
if (stream is null) {
457+
// FileStream is safe
458+
var sfh = new SafeFileHandle((IntPtr)res, ownsHandle: true);
459+
stream = new FileStream(sfh, fileAccess);
460+
}
440461
} else {
441-
// This does not work 100% correctly on .NET, probably because each FileStream has its own read/write cursor
442-
// (it should be shared between dupped descriptors)
443-
//stream = new FileStream(new SafeFileHandle((IntPtr)res, ownsHandle: true), FileAccess.ReadWrite);
444-
// Accidentaly, this would also not work on Mono: https://github.com/mono/mono/issues/12783
445-
stream = null; // Handle stream sharing in PythonFileManager
462+
// normal case
463+
stream = new PosixFileStream(res);
446464
}
447465
return res;
448466
}
449467

468+
450469
#if FEATURE_PROCESS
451470
/// <summary>
452471
/// single instance of environment dictionary is shared between multiple runtimes because the environment
@@ -470,6 +489,9 @@ public static object fstat(CodeContext/*!*/ context, int fd) {
470489
PythonFileManager fileManager = context.LanguageContext.FileManager;
471490

472491
if (fileManager.TryGetStreams(fd, out StreamBox? streams)) {
492+
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) {
493+
return fstatUnix(fd);
494+
}
473495
if (streams.IsConsoleStream()) return new stat_result(0x2000);
474496
if (streams.IsStandardIOStream()) return new stat_result(0x1000);
475497
if (StatStream(streams.ReadStream) is not null and var res) return res;
@@ -483,15 +505,9 @@ public static object fstat(CodeContext/*!*/ context, int fd) {
483505
#endif
484506
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) {
485507
if (ReferenceEquals(stream, Stream.Null)) return new stat_result(0x2000);
486-
} else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) {
487-
if (IsUnixStream(stream)) return new stat_result(0x1000);
488508
}
489509
return null;
490510
}
491-
492-
static bool IsUnixStream(Stream stream) {
493-
return stream is Mono.Unix.UnixStream;
494-
}
495511
}
496512

497513
public static void fsync(CodeContext context, int fd) {
@@ -869,7 +885,16 @@ public static void mkdir(CodeContext context, object? path, [ParamDictionary, No
869885

870886
private const int DefaultBufferSize = 4096;
871887

872-
[Documentation("open(path, flags, mode=511, *, dir_fd=None)")]
888+
[Documentation("""
889+
open(path, flags, mode=511, *, dir_fd=None)
890+
891+
Open a file for low level IO. Returns a file descriptor (integer).
892+
893+
If dir_fd is not None, it should be a file descriptor open to a directory,
894+
and path should be relative; path will then be relative to that directory.
895+
dir_fd may not be implemented on your platform.
896+
If it is unavailable, using it will raise a NotImplementedError.
897+
""")]
873898
public static object open(CodeContext/*!*/ context, [NotNone] string path, int flags, [ParamDictionary, NotNone] IDictionary<string, object> kwargs, [NotNone] params object[] args) {
874899
var numArgs = args.Length;
875900
CheckOptionalArgsCount(numRegParms: 2, numOptPosParms: 1, numKwParms: 1, numArgs, kwargs.Count);
@@ -889,12 +914,23 @@ public static object open(CodeContext/*!*/ context, [NotNone] string path, int f
889914
}
890915
}
891916

917+
if ((RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) && !ClrModule.IsMono) {
918+
// Use PosixFileStream to operate on fd directly
919+
// On Mono, we must use FileStream due to limitations in MemoryMappedFile
920+
Stream s = PosixFileStream.Open(path, flags, unchecked((uint)mode), out int fd);
921+
//Stream s = PythonIOModule.FileIO.OpenFilePosix(path, flags, mode, out int fd);
922+
if ((flags & O_APPEND) != 0) {
923+
s.Seek(0L, SeekOrigin.End);
924+
}
925+
return context.LanguageContext.FileManager.Add(fd, new(s));
926+
}
927+
892928
try {
893929
FileMode fileMode = FileModeFromFlags(flags);
894930
FileAccess access = FileAccessFromFlags(flags);
895931
FileOptions options = FileOptionsFromFlags(flags);
896932
Stream s; // the stream opened to acces the file
897-
FileStream? fs; // downcast of s if s is FileStream (this is always the case on POSIX)
933+
FileStream? fs; // downcast of s if s is FileStream
898934
Stream? rs = null; // secondary read stream if needed, otherwise same as s
899935
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && IsNulFile(path)) {
900936
fs = null;
@@ -1436,6 +1472,13 @@ private static object statUnix(string path) {
14361472
return LightExceptions.Throw(GetLastUnixError(path));
14371473
}
14381474

1475+
private static object fstatUnix(int fd) {
1476+
if (Mono.Unix.Native.Syscall.fstat(fd, out Mono.Unix.Native.Stat buf) == 0) {
1477+
return new stat_result(buf);
1478+
}
1479+
return LightExceptions.Throw(GetLastUnixError());
1480+
}
1481+
14391482
private const int OPEN_EXISTING = 3;
14401483
private const int FILE_ATTRIBUTE_NORMAL = 0x00000080;
14411484
private const int FILE_READ_ATTRIBUTES = 0x0080;
@@ -1669,8 +1712,27 @@ public static void truncate(CodeContext context, object? path, BigInteger length
16691712
public static void truncate(CodeContext context, int fd, BigInteger length)
16701713
=> ftruncate(context, fd, length);
16711714

1672-
public static void ftruncate(CodeContext context, int fd, BigInteger length)
1673-
=> context.LanguageContext.FileManager.GetStreams(fd).Truncate((long)length);
1715+
public static void ftruncate(CodeContext context, int fd, BigInteger length) {
1716+
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) {
1717+
ftruncateUnix(fd, (long)length);
1718+
} else {
1719+
context.LanguageContext.FileManager.GetStreams(fd).Truncate((long)length);
1720+
}
1721+
}
1722+
1723+
1724+
[SupportedOSPlatform("linux"), SupportedOSPlatform("osx")]
1725+
internal static void ftruncateUnix(int fd, long length) {
1726+
int result;
1727+
Mono.Unix.Native.Errno errno;
1728+
do {
1729+
result = Mono.Unix.Native.Syscall.ftruncate(fd, length);
1730+
} while (Mono.Unix.UnixMarshal.ShouldRetrySyscall(result, out errno));
1731+
1732+
if (errno != 0)
1733+
throw GetOsError(Mono.Unix.Native.NativeConvert.FromErrno(errno));
1734+
}
1735+
16741736

16751737
#if FEATURE_FILESYSTEM
16761738
public static object times() {

0 commit comments

Comments
 (0)