From fa6799b023db12cea81b3a5677ee844f81ed99b8 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Mon, 3 Feb 2025 15:31:21 -0800 Subject: [PATCH 1/3] Implement syscall `ioctl` --- src/core/IronPython.Modules/fcntl.cs | 189 ++++++++++++++++++++++++--- 1 file changed, 169 insertions(+), 20 deletions(-) diff --git a/src/core/IronPython.Modules/fcntl.cs b/src/core/IronPython.Modules/fcntl.cs index b53bf4dff..d63e35b8b 100644 --- a/src/core/IronPython.Modules/fcntl.cs +++ b/src/core/IronPython.Modules/fcntl.cs @@ -5,6 +5,7 @@ #nullable enable using System; +using System.Buffers; using System.Diagnostics; using System.Numerics; using System.Reflection; @@ -119,32 +120,180 @@ public static object fcntl(CodeContext context, object? fd, int cmd, object? arg #endregion - #region ioctl + #region ioctl + + // The actual signature of ioctl is + // + // int ioctl(int, unsigned long, ...) + // + // but .NET, as of Jan 2025, still does not support varargs in P/Invoke [1] + // so as a workaround, nonvararg prototypes are defined for each architecture. + // [1]: https://github.com/dotnet/runtime/issues/48796 + +#if NET10_0_OR_GREATER +#error Check if this version of .NET supports P/Invoke of variadic functions; if not, change the condition to recheck at next major .NET version +#endif + + [DllImport("libc", SetLastError = true, EntryPoint = "ioctl")] + private static extern unsafe int _ioctl(int fd, ulong request, void* arg); + [DllImport("libc", SetLastError = true, EntryPoint = "ioctl")] + private static extern int _ioctl(int fd, ulong request, long arg); + + [DllImport("libc", SetLastError = true, EntryPoint = "ioctl")] + private static extern unsafe int _ioctl_arm64(int fd, ulong request, + // pad register arguments (first 8) to force vararg on stack + // ARM: https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#appendix-variable-argument-lists + // Apple: https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms + nint r2, nint r3, nint r4, nint r5, nint r6, nint r7, + void* arg); + [DllImport("libc", SetLastError = true, EntryPoint = "ioctl")] + private static extern int _ioctl_arm64(int fd, ulong request, + nint r2, nint r3, nint r4, nint r5, nint r6, nint r7, + long arg); + + + // request will be int, uint or BigInteger, and in Python is limited to values that can fit in 32 bits (unchecked) + // long should capture all allowed values + // return value is int, bytes, or LightException + [LightThrowing] + public static object ioctl(int fd, long request, [NotNone] IBufferProtocol arg, bool mutate_flag = true) { + CheckFileDescriptor(fd); + + ulong cmd = unchecked((ulong)request); - // supporting fcntl.ioctl(fileno, termios.TIOCGWINSZ, buf) - // where buf = array.array('h', [0, 0, 0, 0]) - public static object ioctl(CodeContext context, int fd, int cmd, [NotNone] IBufferProtocol arg, int mutate_flag = 1) { - if (cmd == PythonTermios.TIOCGWINSZ) { - using IPythonBuffer buf = arg.GetBuffer(); + const int defaultBufSize = 1024; + int bufSize; + IPythonBuffer? buf = null; + + if (mutate_flag) { + buf = arg.GetBufferNoThrow(BufferFlags.Writable); + } + if (buf is not null) { + bufSize = buf.AsReadOnlySpan().Length; + } else { + buf = arg.GetBuffer(BufferFlags.Simple); + bufSize = buf.AsReadOnlySpan().Length; + if (bufSize > defaultBufSize) { + buf.Dispose(); + throw PythonOps.ValueError("ioctl bytes arg too long"); + } + mutate_flag = false; // return a buffer, not integer + } + bool in_place = bufSize > defaultBufSize; // only large buffers are mutated in place + Debug.Assert(!in_place || mutate_flag); // in_place implies mutate_flag + +#if !NETCOREAPP + throw new PlatformNotSupportedException("ioctl is not supported on Mono"); +#else + try { + unsafe { + MemoryHandle hmem = default; + void* ptr = null; + if (in_place) { + hmem = buf.Pin(); + } else { + ptr = NativeMemory.AllocZeroed(defaultBufSize + 1); // +1 for extra nul byte + } + try { + if (in_place) { + ptr = hmem.Pointer; + } else { + Debug.Assert(bufSize <= defaultBufSize); + var dest = new Span((byte*)ptr, bufSize); + buf.AsReadOnlySpan().CopyTo(dest); + } + Debug.Assert(ptr != null); + + int result; + Errno errno; + do { + if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { + // workaround for Arm64 vararg calling convention (but not for ARM64EC on Windows) + result = _ioctl_arm64(fd, cmd, 0, 0, 0, 0, 0, 0, ptr); + } else { + result = _ioctl(fd, cmd, ptr); + } + } while (UnixMarshal.ShouldRetrySyscall(result, out errno)); + + if (result == -1) { + return LightExceptions.Throw(PythonNT.GetOsError(NativeConvert.FromErrno(errno))); + } + if (mutate_flag) { + if (!in_place) { + var src = new Span((byte*)ptr, bufSize); + src.CopyTo(buf.AsSpan()); + } + return ScriptingRuntimeHelpers.Int32ToObject(result); + } else { + Debug.Assert(!in_place); + byte[] response = new byte[bufSize]; + var src = new Span((byte*)ptr, bufSize); + src.CopyTo(response); + return Bytes.Make(response); + } + } finally { + if (in_place) { + hmem.Dispose(); + } else { + NativeMemory.Free(ptr); + } + } + } + } finally { + buf.Dispose(); + } +#endif + } - Span winsize = stackalloc short[4]; - winsize[0] = (short)Console.WindowHeight; - winsize[1] = (short)Console.WindowWidth; - winsize[2] = (short)Console.BufferHeight; // buffer height and width are not accurate on macOS - winsize[3] = (short)Console.BufferWidth; - Span payload = MemoryMarshal.Cast(winsize); - if (buf.IsReadOnly || mutate_flag == 0) { - byte[] res = buf.ToArray(); - payload.Slice(0, Math.Min(payload.Length, res.Length)).CopyTo(res); - return Bytes.Make(res); + [LightThrowing] + public static object ioctl(int fd, long request, [Optional] object? arg, bool mutate_flag = true) { + CheckFileDescriptor(fd); + + ulong cmd = unchecked((ulong)request); + + long data = arg switch { + Missing => 0, + int i => i, + uint ui => ui, + long l => l, + ulong ul => (long)ul, + BigInteger bi => (long)bi, + Extensible ebi => (long)ebi.Value, + _ => throw PythonOps.TypeErrorForBadInstance("integer argument expected, got {0}", arg) + }; + +#if !NETCOREAPP + throw new PlatformNotSupportedException("ioctl is not supported on Mono"); +#else + int result; + Errno errno; + do { + if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { + // workaround for Arm64 vararg calling convention (but not for ARM64EC on Windows) + result = _ioctl_arm64(fd, cmd, 0, 0, 0, 0, 0, 0, data); } else { - var res = buf.AsSpan(); - payload.Slice(0, Math.Min(payload.Length, res.Length)).CopyTo(res); - return 0; + result = _ioctl(fd, cmd, data); } + } while (UnixMarshal.ShouldRetrySyscall(result, out errno)); + + if (result == -1) { + return LightExceptions.Throw(PythonNT.GetOsError(NativeConvert.FromErrno(errno))); + } + return ScriptingRuntimeHelpers.Int32ToObject(result); +#endif + } + + + [LightThrowing] + public static object ioctl(CodeContext context, object? fd, long request, [Optional] object? arg, bool mutate_flag = true) { + int fileno = GetFileDescriptor(context, fd); + + if (arg is IBufferProtocol bp) { + return ioctl(fileno, request, bp, mutate_flag); } - throw new NotImplementedException($"ioctl: unsupported command {cmd}"); + + return ioctl(fileno, request, arg, mutate_flag); } #endregion From ddd210e830652a5857418f6a540e160057c814f8 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Mon, 3 Feb 2025 16:09:25 -0800 Subject: [PATCH 2/3] Simplify by avoiding unmanaged memory allocation --- src/core/IronPython.Modules/fcntl.cs | 73 +++++++++++----------------- 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/src/core/IronPython.Modules/fcntl.cs b/src/core/IronPython.Modules/fcntl.cs index d63e35b8b..6fda37fcc 100644 --- a/src/core/IronPython.Modules/fcntl.cs +++ b/src/core/IronPython.Modules/fcntl.cs @@ -153,7 +153,7 @@ private static extern int _ioctl_arm64(int fd, ulong request, // request will be int, uint or BigInteger, and in Python is limited to values that can fit in 32 bits (unchecked) - // long should capture all allowed values + // long should capture all allowed request values // return value is int, bytes, or LightException [LightThrowing] public static object ioctl(int fd, long request, [NotNone] IBufferProtocol arg, bool mutate_flag = true) { @@ -169,7 +169,7 @@ public static object ioctl(int fd, long request, [NotNone] IBufferProtocol arg, buf = arg.GetBufferNoThrow(BufferFlags.Writable); } if (buf is not null) { - bufSize = buf.AsReadOnlySpan().Length; + bufSize = buf.AsSpan().Length; // check early if buf is indeed writable } else { buf = arg.GetBuffer(BufferFlags.Simple); bufSize = buf.AsReadOnlySpan().Length; @@ -180,32 +180,25 @@ public static object ioctl(int fd, long request, [NotNone] IBufferProtocol arg, mutate_flag = false; // return a buffer, not integer } bool in_place = bufSize > defaultBufSize; // only large buffers are mutated in place - Debug.Assert(!in_place || mutate_flag); // in_place implies mutate_flag #if !NETCOREAPP throw new PlatformNotSupportedException("ioctl is not supported on Mono"); #else try { + Debug.Assert(!in_place || mutate_flag); // in_place implies mutate_flag + + Span workSpan; + if (in_place) { + workSpan = buf.AsSpan(); + } else { + workSpan = new byte[defaultBufSize + 1]; // +1 for extra NUL byte + Debug.Assert(bufSize <= defaultBufSize); + buf.AsReadOnlySpan().CopyTo(workSpan); + } + int result; + Errno errno; unsafe { - MemoryHandle hmem = default; - void* ptr = null; - if (in_place) { - hmem = buf.Pin(); - } else { - ptr = NativeMemory.AllocZeroed(defaultBufSize + 1); // +1 for extra nul byte - } - try { - if (in_place) { - ptr = hmem.Pointer; - } else { - Debug.Assert(bufSize <= defaultBufSize); - var dest = new Span((byte*)ptr, bufSize); - buf.AsReadOnlySpan().CopyTo(dest); - } - Debug.Assert(ptr != null); - - int result; - Errno errno; + fixed (byte* ptr = workSpan) { do { if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { // workaround for Arm64 vararg calling convention (but not for ARM64EC on Windows) @@ -214,30 +207,22 @@ public static object ioctl(int fd, long request, [NotNone] IBufferProtocol arg, result = _ioctl(fd, cmd, ptr); } } while (UnixMarshal.ShouldRetrySyscall(result, out errno)); + } + } - if (result == -1) { - return LightExceptions.Throw(PythonNT.GetOsError(NativeConvert.FromErrno(errno))); - } - if (mutate_flag) { - if (!in_place) { - var src = new Span((byte*)ptr, bufSize); - src.CopyTo(buf.AsSpan()); - } - return ScriptingRuntimeHelpers.Int32ToObject(result); - } else { - Debug.Assert(!in_place); - byte[] response = new byte[bufSize]; - var src = new Span((byte*)ptr, bufSize); - src.CopyTo(response); - return Bytes.Make(response); - } - } finally { - if (in_place) { - hmem.Dispose(); - } else { - NativeMemory.Free(ptr); - } + if (result == -1) { + return LightExceptions.Throw(PythonNT.GetOsError(NativeConvert.FromErrno(errno))); + } + if (mutate_flag) { + if (!in_place) { + workSpan.Slice(0, bufSize).CopyTo(buf.AsSpan()); } + return ScriptingRuntimeHelpers.Int32ToObject(result); + } else { + Debug.Assert(!in_place); + byte[] response = new byte[bufSize]; + workSpan.Slice(0, bufSize).CopyTo(response); + return Bytes.Make(response); } } finally { buf.Dispose(); From 42244b405d45e25235607dc76cab27d8de502370 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Tue, 4 Feb 2025 14:58:03 -0800 Subject: [PATCH 3/3] Accept `str` as argument to `fcntl` and `ioctl` --- src/core/IronPython.Modules/fcntl.cs | 79 +++++++++++++++------------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/src/core/IronPython.Modules/fcntl.cs b/src/core/IronPython.Modules/fcntl.cs index 6fda37fcc..bee20458d 100644 --- a/src/core/IronPython.Modules/fcntl.cs +++ b/src/core/IronPython.Modules/fcntl.cs @@ -11,6 +11,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Runtime.Versioning; +using System.Text; using Mono.Unix; using Mono.Unix.Native; @@ -78,15 +79,13 @@ public static object fcntl(int fd, int cmd, [NotNone] Bytes arg) { public static object fcntl(int fd, int cmd, [Optional] object? arg) { CheckFileDescriptor(fd); - long data = arg switch { - Missing => 0, - int i => i, - uint ui => ui, - long l => l, - ulong ul => (long)ul, - BigInteger bi => (long)bi, - Extensible ebi => (long)ebi.Value, - _ => throw PythonOps.TypeErrorForBadInstance("integer argument expected, got {0}", arg) + if (!TryGetInt64(arg, out long data)) { + return arg switch { + Bytes bytes => fcntl(fd, cmd, bytes), + string s => fcntl(fd, cmd, Bytes.Make(Encoding.UTF8.GetBytes(s))), + Extensible es => fcntl(fd, cmd, Bytes.Make(Encoding.UTF8.GetBytes(es.Value))), + _ => throw PythonOps.TypeErrorForBadInstance("integer or bytes argument expected, got {0}", arg) + }; }; if (!NativeConvert.TryToFcntlCommand(cmd, out FcntlCommand fcntlCommand)) { @@ -107,15 +106,8 @@ public static object fcntl(int fd, int cmd, [Optional] object? arg) { [LightThrowing] - public static object fcntl(CodeContext context, object? fd, int cmd, object? arg = null) { - int fileno = GetFileDescriptor(context, fd); - - if (arg is Bytes bytes) { - return fcntl(fileno, cmd, bytes); - } - - return fcntl(fileno, cmd, arg); - } + public static object fcntl(CodeContext context, object? fd, int cmd, [Optional] object? arg) + => fcntl(GetFileDescriptor(context, fd), cmd, arg); #endregion @@ -235,19 +227,17 @@ public static object ioctl(int fd, long request, [NotNone] IBufferProtocol arg, public static object ioctl(int fd, long request, [Optional] object? arg, bool mutate_flag = true) { CheckFileDescriptor(fd); - ulong cmd = unchecked((ulong)request); - - long data = arg switch { - Missing => 0, - int i => i, - uint ui => ui, - long l => l, - ulong ul => (long)ul, - BigInteger bi => (long)bi, - Extensible ebi => (long)ebi.Value, - _ => throw PythonOps.TypeErrorForBadInstance("integer argument expected, got {0}", arg) + if (!TryGetInt64(arg, out long data)) { + return arg switch { + IBufferProtocol bp => ioctl(fd, request, bp), + string s => ioctl(fd, request, Bytes.Make(Encoding.UTF8.GetBytes(s))), + Extensible es => ioctl(fd, request, Bytes.Make(Encoding.UTF8.GetBytes(es.Value))), + _ => throw PythonOps.TypeErrorForBadInstance("integer or a bytes-like argument expected, got {0}", arg) + }; }; + ulong cmd = unchecked((ulong)request); + #if !NETCOREAPP throw new PlatformNotSupportedException("ioctl is not supported on Mono"); #else @@ -271,15 +261,8 @@ public static object ioctl(int fd, long request, [Optional] object? arg, bool mu [LightThrowing] - public static object ioctl(CodeContext context, object? fd, long request, [Optional] object? arg, bool mutate_flag = true) { - int fileno = GetFileDescriptor(context, fd); - - if (arg is IBufferProtocol bp) { - return ioctl(fileno, request, bp, mutate_flag); - } - - return ioctl(fileno, request, arg, mutate_flag); - } + public static object ioctl(CodeContext context, object? fd, long request, [Optional] object? arg, bool mutate_flag = true) + => ioctl(GetFileDescriptor(context, fd), request, arg, mutate_flag); #endregion @@ -376,6 +359,26 @@ private static void CheckFileDescriptor(int fd) { } } + + private static bool TryGetInt64(object? obj, out long value) { + int success = 1; + value = obj switch { + Missing => 0, + int i => i, + uint ui => ui, + long l => l, + ulong ul => (long)ul, + BigInteger bi => (long)bi, + Extensible ebi => (long)ebi.Value, + byte b => b, + sbyte sb => sb, + short s => s, + ushort us => us, + _ => success = 0 + }; + return success != 0; + } + #endregion