Skip to content

Commit 847db52

Browse files
BCSharpBCSharp
committed
Merge branch 'upstream/main' into 'open_file_posix'
2 parents 1cf9ac6 + aaf701d commit 847db52

24 files changed

+298
-262
lines changed

‎Src/IronPython.Modules/IterTools.cs‎

Lines changed: 33 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,16 @@
88
using System.Numerics;
99
using System.Runtime.InteropServices;
1010

11-
using Microsoft.Scripting;
12-
using Microsoft.Scripting.Runtime;
13-
using Microsoft.Scripting.Utils;
14-
1511
using IronPython.Runtime;
1612
using IronPython.Runtime.Binding;
1713
using IronPython.Runtime.Exceptions;
1814
using IronPython.Runtime.Operations;
1915
using IronPython.Runtime.Types;
2016

17+
using Microsoft.Scripting;
18+
using Microsoft.Scripting.Runtime;
19+
using Microsoft.Scripting.Utils;
20+
2121
[assembly: PythonModule("itertools", typeof(IronPython.Modules.PythonIterTools))]
2222
namespace IronPython.Modules {
2323
public static class PythonIterTools {
@@ -233,7 +233,7 @@ private static void EnsureIterator(CodeContext/*!*/ context, object iter) {
233233
if (iter == null ||
234234
!PythonOps.HasAttr(context, iter, "__iter__") &&
235235
!PythonOps.HasAttr(context, iter, "__getitem__")) {
236-
throw PythonOps.TypeError("'{0}' object is not iterable", PythonOps.GetPythonTypeName(iter));
236+
throw PythonOps.TypeError("'{0}' object is not iterable", PythonOps.GetPythonTypeName(iter));
237237
}
238238
}
239239

@@ -276,13 +276,13 @@ public count(BigInteger start) {
276276
InnerEnumerator = BigIntYielder(this, start, 1);
277277
}
278278

279-
public count(int start=0, int step=1) {
279+
public count(int start = 0, int step = 1) {
280280
_curInt = start;
281281
_step = step;
282282
InnerEnumerator = IntYielder(this, start, step);
283283
}
284284

285-
public count([DefaultParameterValue(0)]int start, BigInteger step) {
285+
public count([DefaultParameterValue(0)] int start, BigInteger step) {
286286
_curInt = start;
287287
_step = step;
288288
InnerEnumerator = IntYielder(this, start, step);
@@ -300,7 +300,7 @@ public count(BigInteger start, BigInteger step) {
300300
InnerEnumerator = BigIntYielder(this, start, step);
301301
}
302302

303-
public count(CodeContext/*!*/ context, [DefaultParameterValue(0)]object start, [DefaultParameterValue(1)]object step) {
303+
public count(CodeContext/*!*/ context, [DefaultParameterValue(0)] object start, [DefaultParameterValue(1)] object step) {
304304
EnsureNumeric(context, start);
305305
EnsureNumeric(context, step);
306306
_cur = start;
@@ -315,7 +315,7 @@ private static void EnsureNumeric(CodeContext/*!*/ context, object num) {
315315
if (num == null ||
316316
!PythonOps.HasAttr(context, num, "__int__") &&
317317
!PythonOps.HasAttr(context, num, "__float__")) {
318-
throw PythonOps.TypeError("a number is required");
318+
throw PythonOps.TypeError("a number is required");
319319
}
320320
}
321321

@@ -548,9 +548,9 @@ private IEnumerator<object> Yielder(IEnumerator iter) {
548548
private IEnumerator<object> Grouper(IEnumerator iter, object curKey) {
549549
while (PythonContext.Equal(GetKey(iter.Current), curKey)) {
550550
yield return iter.Current;
551-
if (!MoveNextHelper(iter)) {
552-
_fFinished = true;
553-
yield break;
551+
if (!MoveNextHelper(iter)) {
552+
_fFinished = true;
553+
yield break;
554554
}
555555
}
556556
}
@@ -675,7 +675,7 @@ public zip_longest(params object[] iterables) {
675675
}
676676
}
677677

678-
public zip_longest([ParamDictionary]IDictionary<object, object> paramDict, params object[] iterables) {
678+
public zip_longest([ParamDictionary] IDictionary<object, object> paramDict, params object[] iterables) {
679679
object fill;
680680

681681
if (paramDict.TryGetValue("fillvalue", out fill)) {
@@ -760,11 +760,14 @@ private static Exception UnexpectedKeywordArgument(IDictionary<object, object> p
760760

761761
[PythonType]
762762
public class product : IterBase {
763+
private PythonTuple[] tuples;
764+
763765
public product(CodeContext context, params object[] iterables) {
764-
InnerEnumerator = Yielder(ArrayUtils.ConvertAll(iterables, x => new PythonList(context, PythonOps.GetEnumerator(x))));
766+
tuples = ArrayUtils.ConvertAll(iterables, x => new PythonTuple(PythonOps.GetEnumerator(x)));
767+
InnerEnumerator = Yielder(tuples);
765768
}
766769

767-
public product(CodeContext context, [ParamDictionary]IDictionary<object, object> paramDict, params object[] iterables) {
770+
public product(CodeContext context, [ParamDictionary] IDictionary<object, object> paramDict, params object[] iterables) {
768771
object repeat;
769772
int iRepeat = 1;
770773
if (paramDict.TryGetValue("repeat", out repeat)) {
@@ -782,29 +785,28 @@ public product(CodeContext context, [ParamDictionary]IDictionary<object, object>
782785
throw UnexpectedKeywordArgument(paramDict);
783786
}
784787

785-
PythonList[] finalIterables = new PythonList[iterables.Length * iRepeat];
788+
tuples = new PythonTuple[iterables.Length * iRepeat];
786789
for (int i = 0; i < iRepeat; i++) {
787790
for (int j = 0; j < iterables.Length; j++) {
788-
finalIterables[i * iterables.Length + j] = new PythonList(context, iterables[j]);
791+
tuples[i * iterables.Length + j] = new PythonTuple(iterables[j]);
789792
}
790793
}
791-
InnerEnumerator = Yielder(finalIterables);
794+
InnerEnumerator = Yielder(tuples);
792795
}
793796

794797
public PythonTuple __reduce__() {
795-
// TODO
796798
return PythonTuple.MakeTuple(
797799
DynamicHelpers.GetPythonType(this),
798-
PythonTuple.MakeTuple(), // arguments
799-
null // state
800+
PythonTuple.MakeTuple(tuples), // arguments
801+
null // TODO: state
800802
);
801803
}
802804

803805
public void __setstate__(object state) {
804806
// TODO
805807
}
806808

807-
private IEnumerator<object> Yielder(PythonList[] iterables) {
809+
private IEnumerator<object> Yielder(PythonTuple[] iterables) {
808810
if (iterables.Length > 0) {
809811
IEnumerator[] enums = new IEnumerator[iterables.Length];
810812
enums[0] = iterables[0].GetEnumerator();
@@ -833,6 +835,7 @@ private IEnumerator<object> Yielder(PythonList[] iterables) {
833835
} else {
834836
yield return PythonTuple.EMPTY;
835837
}
838+
tuples = new PythonTuple[1] { PythonTuple.EMPTY };
836839
}
837840
}
838841

@@ -987,7 +990,7 @@ public permutations(CodeContext context, object iterable) {
987990

988991
public permutations(CodeContext context, object iterable, object r) {
989992
_data = new PythonList(context, iterable);
990-
993+
991994
InnerEnumerator = Yielder(GetR(r, _data));
992995
}
993996

@@ -1078,7 +1081,7 @@ public repeat(object @object) {
10781081
public repeat(object @object, int times) {
10791082
_obj = @object;
10801083
InnerEnumerator = Yielder();
1081-
_remaining = times;
1084+
_remaining = times < 0 ? 0 : times;
10821085
}
10831086

10841087
private IEnumerator<object> Yielder() {
@@ -1094,10 +1097,9 @@ public int __length_hint__() {
10941097
}
10951098

10961099
public PythonTuple __reduce__() {
1097-
// TODO
10981100
return PythonTuple.MakeTuple(
10991101
DynamicHelpers.GetPythonType(this),
1100-
PythonTuple.MakeTuple() // arguments
1102+
_fInfinite ? PythonTuple.MakeTuple(_obj) : PythonTuple.MakeTuple(_obj, _remaining) // arguments
11011103
);
11021104
}
11031105

@@ -1150,9 +1152,9 @@ IEnumerator IEnumerable.GetEnumerator() {
11501152

11511153
#endregion
11521154
}
1153-
1155+
11541156
[PythonType]
1155-
public class starmap : IterBase {
1157+
public class starmap : IterBase {
11561158
public starmap(CodeContext context, object function, object iterable) {
11571159
InnerEnumerator = Yielder(context, function, PythonOps.GetEnumerator(iterable));
11581160
}
@@ -1211,7 +1213,7 @@ public void __setstate__(object state) {
12111213

12121214
private IEnumerator<object> Yielder(object predicate, IEnumerator iter) {
12131215
while (MoveNextHelper(iter)) {
1214-
if(!Converter.ConvertToBoolean(
1216+
if (!Converter.ConvertToBoolean(
12151217
_context.LanguageContext.CallSplat(predicate, iter.Current)
12161218
)) {
12171219
break;
@@ -1222,8 +1224,8 @@ private IEnumerator<object> Yielder(object predicate, IEnumerator iter) {
12221224
}
12231225
}
12241226

1225-
[PythonHidden]
1226-
public class TeeIterator : IEnumerator, IWeakReferenceable {
1227+
[PythonType("_tee")]
1228+
public sealed class TeeIterator : IEnumerator, IWeakReferenceable {
12271229
internal IEnumerator _iter;
12281230
internal PythonList _data;
12291231
private int _curIndex = -1;

‎Src/IronPython.Modules/_datetime.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1099,7 +1099,7 @@ internal override int CompareTo(date other) {
10991099
if (_tz != null) {
11001100
sb.AppendFormat(", tzinfo={0}", PythonOps.Repr(context, _tz));
11011101
}
1102-
sb.AppendFormat(")");
1102+
sb.Append(')');
11031103
return sb.ToString();
11041104
}
11051105
#endregion
@@ -1395,7 +1395,7 @@ private int CompareTo(time other) {
13951395
sb.AppendFormat(", tzinfo={0}", ltzname.ToLowerInvariant());
13961396
}
13971397

1398-
sb.AppendFormat(")");
1398+
sb.Append(')');
13991399

14001400
return sb.ToString();
14011401
}

‎Src/IronPython.Modules/mmap.cs‎

Lines changed: 67 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
[assembly: PythonModule("mmap", typeof(IronPython.Modules.MmapModule))]
3232
namespace IronPython.Modules {
3333
public static class MmapModule {
34+
public const int ACCESS_DEFAULT = 0; // Since Python 3.7
3435
public const int ACCESS_READ = 1;
3536
public const int ACCESS_WRITE = 2;
3637
public const int ACCESS_COPY = 3;
@@ -45,8 +46,6 @@ public static class MmapModule {
4546
[PythonHidden(PlatformsAttribute.PlatformFamily.Windows)]
4647
public const int MAP_PRIVATE = 2;
4748

48-
[PythonHidden(PlatformsAttribute.PlatformFamily.Windows)]
49-
public const int PROT_NONE = 0;
5049
[PythonHidden(PlatformsAttribute.PlatformFamily.Windows)]
5150
public const int PROT_READ = 1;
5251
[PythonHidden(PlatformsAttribute.PlatformFamily.Windows)]
@@ -72,20 +71,65 @@ private static Exception WindowsError(int code) {
7271
public static PythonType mmap {
7372
get {
7473
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) {
75-
return DynamicHelpers.GetPythonTypeFromType(typeof(MmapDefault));
74+
return DynamicHelpers.GetPythonTypeFromType(typeof(MmapWindows));
7675
}
7776

7877
return DynamicHelpers.GetPythonTypeFromType(typeof(MmapUnix));
7978
}
8079
}
8180

81+
8282
[PythonType("mmap"), PythonHidden]
8383
public class MmapUnix : MmapDefault {
84-
public MmapUnix(CodeContext/*!*/ context, int fileno, long length, int flags = MAP_SHARED, int prot = PROT_WRITE | PROT_READ, int access = ACCESS_WRITE, long offset = 0)
85-
: base(context, fileno, length, null, access, offset) { }
84+
public MmapUnix(CodeContext/*!*/ context, int fileno, long length, int flags = MAP_SHARED, int prot = PROT_WRITE | PROT_READ, int access = ACCESS_DEFAULT, long offset = 0)
85+
: base(context, fileno, length, null, ToMmapFileAccess(flags, prot, access), offset) { }
86+
87+
private static MemoryMappedFileAccess ToMmapFileAccess(int flags, int prot, int access) {
88+
if (access == ACCESS_DEFAULT) {
89+
if ((flags & (MAP_PRIVATE | MAP_SHARED)) == 0) {
90+
throw PythonOps.OSError(PythonErrorNumber.EINVAL, "Invalid argument");
91+
}
92+
if ((prot & PROT_WRITE) != 0) {
93+
prot |= PROT_READ;
94+
}
95+
return (prot & (PROT_READ | PROT_WRITE | PROT_EXEC)) switch {
96+
PROT_READ => MemoryMappedFileAccess.Read,
97+
PROT_READ | PROT_WRITE => (flags & MAP_PRIVATE) == 0 ? MemoryMappedFileAccess.ReadWrite : MemoryMappedFileAccess.CopyOnWrite,
98+
PROT_READ | PROT_EXEC => MemoryMappedFileAccess.ReadExecute,
99+
PROT_READ | PROT_WRITE | PROT_EXEC when (flags & MAP_PRIVATE) == 0 => MemoryMappedFileAccess.ReadWriteExecute,
100+
_ => throw PythonOps.NotImplementedError("this combination of prot is not supported"),
101+
};
102+
} else if (flags != MAP_SHARED || prot != (PROT_WRITE | PROT_READ)) {
103+
throw PythonOps.ValueError("mmap can't specify both access and flags, prot.");
104+
} else {
105+
return access switch {
106+
ACCESS_READ => MemoryMappedFileAccess.Read,
107+
ACCESS_WRITE => MemoryMappedFileAccess.ReadWrite,
108+
ACCESS_COPY => MemoryMappedFileAccess.CopyOnWrite,
109+
_ => throw PythonOps.ValueError("mmap invalid access parameter"),
110+
};
111+
}
112+
}
86113
}
87114

115+
88116
[PythonType("mmap"), PythonHidden]
117+
public class MmapWindows : MmapDefault {
118+
public MmapWindows(CodeContext context, int fileno, long length, string tagname = null, int access = ACCESS_DEFAULT, long offset = 0)
119+
: base(context, fileno, length, tagname, ToMmapFileAccess(access), offset) { }
120+
121+
private static MemoryMappedFileAccess ToMmapFileAccess(int access) {
122+
return access switch {
123+
ACCESS_READ => MemoryMappedFileAccess.Read,
124+
// On Windows, default access is write-through
125+
ACCESS_DEFAULT or ACCESS_WRITE => MemoryMappedFileAccess.ReadWrite,
126+
ACCESS_COPY => MemoryMappedFileAccess.CopyOnWrite,
127+
_ => throw PythonOps.ValueError("mmap invalid access parameter"),
128+
};
129+
}
130+
}
131+
132+
[PythonHidden]
89133
public class MmapDefault : IWeakReferenceable {
90134
private MemoryMappedFile _file;
91135
private MemoryMappedViewAccessor _view;
@@ -100,20 +144,8 @@ public class MmapDefault : IWeakReferenceable {
100144
private volatile bool _isClosed;
101145
private int _refCount = 1;
102146

103-
public MmapDefault(CodeContext/*!*/ context, int fileno, long length, string tagname = null, int access = ACCESS_WRITE, long offset = 0) {
104-
switch (access) {
105-
case ACCESS_READ:
106-
_fileAccess = MemoryMappedFileAccess.Read;
107-
break;
108-
case ACCESS_WRITE:
109-
_fileAccess = MemoryMappedFileAccess.ReadWrite;
110-
break;
111-
case ACCESS_COPY:
112-
_fileAccess = MemoryMappedFileAccess.CopyOnWrite;
113-
break;
114-
default:
115-
throw PythonOps.ValueError("mmap invalid access parameter");
116-
}
147+
public MmapDefault(CodeContext/*!*/ context, int fileno, long length, string tagname, MemoryMappedFileAccess fileAccess, long offset) {
148+
_fileAccess = fileAccess;
117149

118150
if (length < 0) {
119151
throw PythonOps.OverflowError("memory mapped size must be positive");
@@ -163,9 +195,9 @@ public MmapDefault(CodeContext/*!*/ context, int fileno, long length, string tag
163195
_file = MemoryMappedFile.CreateFromFile(_handle, _mapName, length, _fileAccess, HandleInheritability.None, leaveOpen: true);
164196
#else
165197
_handle = new SafeFileHandle((IntPtr)fileno, ownsHandle: false);
166-
FileAccess fileAccess = stream.CanWrite ? stream.CanRead ? FileAccess.ReadWrite : FileAccess.Write : FileAccess.Read;
198+
FileAccess fa = stream.CanWrite ? stream.CanRead ? FileAccess.ReadWrite : FileAccess.Write : FileAccess.Read;
167199
// This may or may not work on Mono, but on Mono streams.ReadStream is FileStream (unless dupped in some cases)
168-
_sourceStream = new FileStream(_handle, fileAccess);
200+
_sourceStream = new FileStream(_handle, fa);
169201
#endif
170202
}
171203
// otherwise leaves _file as null and _sourceStream as null
@@ -227,6 +259,8 @@ void CheckFileAccess(MemoryMappedFileAccess mmapAccess, Stream stream) {
227259
MemoryMappedFileAccess.Read => stream.CanRead,
228260
MemoryMappedFileAccess.ReadWrite => stream.CanRead && stream.CanWrite,
229261
MemoryMappedFileAccess.CopyOnWrite => stream.CanRead,
262+
MemoryMappedFileAccess.ReadExecute => stream.CanRead,
263+
MemoryMappedFileAccess.ReadWriteExecute => stream.CanRead && stream.CanWrite,
230264
_ => false
231265
};
232266

@@ -236,6 +270,16 @@ void CheckFileAccess(MemoryMappedFileAccess mmapAccess, Stream stream) {
236270
}
237271
throw PythonOps.OSError(PythonExceptions._OSError.ERROR_ACCESS_DENIED, "Invalid access mode");
238272
}
273+
274+
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) {
275+
// Unix map does not support increasing size on open
276+
if (_offset + length > stream.Length) {
277+
if (_handle is not null && _sourceStream is not null) {
278+
_sourceStream.Dispose();
279+
}
280+
throw PythonOps.ValueError("mmap length is greater than file size");
281+
}
282+
}
239283
}
240284
} // end of constructor
241285

@@ -597,7 +641,7 @@ public string readline() {
597641

598642
public void resize(long newsize) {
599643
using (new MmapLocker(this)) {
600-
if (_fileAccess != MemoryMappedFileAccess.ReadWrite) {
644+
if (_fileAccess is not MemoryMappedFileAccess.ReadWrite and not MemoryMappedFileAccess.ReadWriteExecute) {
601645
throw PythonOps.TypeError("mmap can't resize a readonly or copy-on-write memory map.");
602646
}
603647

@@ -828,7 +872,7 @@ private long Position {
828872
}
829873

830874
private void EnsureWritable() {
831-
if (_fileAccess == MemoryMappedFileAccess.Read) {
875+
if (_fileAccess is MemoryMappedFileAccess.Read or MemoryMappedFileAccess.ReadExecute) {
832876
throw PythonOps.TypeError("mmap can't modify a read-only memory map.");
833877
}
834878
}

0 commit comments

Comments
 (0)