Skip to content

Use module context with iterables #1949

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 25, 2025
Merged

Use module context with iterables #1949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 18 additions & 18 deletions src/core/IronPython.Modules/_collections.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,24 +56,24 @@ public void __init__([ParamDictionary] IDictionary<object, object> dict) {
clear();
}

public void __init__(object iterable) {
public void __init__(CodeContext context, object iterable) {
_maxLen = -1;
clear();
extend(iterable);
extend(context, iterable);
}

public void __init__(object iterable, object maxlen) {
public void __init__(CodeContext context, object iterable, object maxlen) {
_maxLen = VerifyMaxLenValue(maxlen);

clear();
extend(iterable);
extend(context, iterable);
}

public void __init__(object iterable, [ParamDictionary] IDictionary<object, object> dict) {
public void __init__(CodeContext context, object iterable, [ParamDictionary] IDictionary<object, object> dict) {
if (VerifyMaxLen(dict) < 0) {
__init__(iterable);
__init__(context, iterable);
} else {
__init__(iterable, VerifyMaxLen(dict));
__init__(context, iterable, VerifyMaxLen(dict));
}
}

Expand Down Expand Up @@ -187,7 +187,7 @@ public void clear() {
public object copy(CodeContext context)
=> __copy__(context);

public void extend(object iterable) {
public void extend(CodeContext context, object iterable) {
// d.extend(d)
if (ReferenceEquals(iterable, this)) {
WalkDeque(idx => {
Expand All @@ -197,13 +197,13 @@ public void extend(object iterable) {
return;
}

IEnumerator e = PythonOps.GetEnumerator(iterable);
IEnumerator e = PythonOps.GetEnumerator(context, iterable);
while (e.MoveNext()) {
append(e.Current);
}
}

public void extendleft(object iterable) {
public void extendleft(CodeContext context, object iterable) {
// d.extendleft(d)
if (ReferenceEquals(iterable, this)) {
WalkDeque(idx => {
Expand All @@ -213,7 +213,7 @@ public void extendleft(object iterable) {
return;
}

IEnumerator e = PythonOps.GetEnumerator(iterable);
IEnumerator e = PythonOps.GetEnumerator(context, iterable);
while (e.MoveNext()) {
appendleft(e.Current);
}
Expand Down Expand Up @@ -511,7 +511,7 @@ public bool __contains__(CodeContext/*!*/ context, object key) {
public object __copy__(CodeContext/*!*/ context) {
if (GetType() == typeof(deque)) {
deque res = new deque(_maxLen);
res.extend(((IEnumerable)this).GetEnumerator());
res.extend(context, ((IEnumerable)this).GetEnumerator());
return res;
} else {
return PythonCalls.Call(context, DynamicHelpers.GetPythonType(this), ((IEnumerable)this).GetEnumerator());
Expand Down Expand Up @@ -564,8 +564,8 @@ public int __len__() {
}

[SpecialName]
public deque InPlaceAdd(object other) {
extend(other);
public deque InPlaceAdd(CodeContext context, object other) {
extend(context, other);
return this;
}

Expand All @@ -582,18 +582,18 @@ public static deque Add(CodeContext context, [NotNone] deque x, object y) {
public static deque Add(CodeContext context, [NotNone] deque x, [NotNone] deque y) {
var d = (deque)__new__(context, DynamicHelpers.GetPythonType(x), null, null);
if (x._maxLen > 0) {
d.__init__(x, x._maxLen);
d.__init__(context, x, x._maxLen);
} else {
d.__init__(x);
d.__init__(context, x);
}
d.extend(y);
d.extend(context, y);
return d;
}

private static deque MultiplyWorker(deque self, int count) {
var d = new deque(self._maxLen);
if (count <= 0 || self._itemCnt == 0) return d;
d.extend(self);
d.extend(DefaultContext.Default, self); // TODO: context
if (count == 1) return d;

if (d._maxLen < 0 || d._itemCnt * count <= d._maxLen) {
Expand Down
4 changes: 2 additions & 2 deletions src/core/IronPython.Modules/_functools.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static class FunctionTools {
public const string __doc__ = "provides functionality for manipulating callable objects";

public static object? reduce(CodeContext/*!*/ context, SiteLocalStorage<CallSite<Func<CallSite, CodeContext, object?, object?, object?, object?>>> siteData, object? func, object? seq) {
IEnumerator i = PythonOps.GetEnumerator(seq);
IEnumerator i = PythonOps.GetEnumerator(context, seq);
if (!i.MoveNext()) {
throw PythonOps.TypeError("reduce() of empty sequence with no initial value");
}
Expand All @@ -43,7 +43,7 @@ public static class FunctionTools {
}

public static object? reduce(CodeContext/*!*/ context, SiteLocalStorage<CallSite<Func<CallSite, CodeContext, object?, object?, object?, object?>>> siteData, object? func, object? seq, object? initializer) {
IEnumerator i = PythonOps.GetEnumerator(seq);
IEnumerator i = PythonOps.GetEnumerator(context, seq);
EnsureReduceData(context, siteData);

CallSite<Func<CallSite, CodeContext, object?, object?, object?, object?>> site = siteData.Data;
Expand Down
4 changes: 2 additions & 2 deletions src/core/IronPython.Modules/_heapq.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public static PythonList nlargest(CodeContext/*!*/ context, int n, object? itera
}

PythonList ret = new PythonList(Math.Min(n, 4000)); // don't allocate anything too huge
IEnumerator en = PythonOps.GetEnumerator(iterable);
IEnumerator en = PythonOps.GetEnumerator(context, iterable);

// populate list with first n items
for (int i = 0; i < n; i++) {
Expand Down Expand Up @@ -120,7 +120,7 @@ public static PythonList nsmallest(CodeContext/*!*/ context, int n, object? iter
}

PythonList ret = new PythonList(Math.Min(n, 4000)); // don't allocate anything too huge
IEnumerator en = PythonOps.GetEnumerator(iterable);
IEnumerator en = PythonOps.GetEnumerator(context, iterable);

// populate list with first n items
for (int i = 0; i < n; i++) {
Expand Down
4 changes: 2 additions & 2 deletions src/core/IronPython.Modules/_operator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ public static bool contains(CodeContext/*!*/ context, object? a, object? b) {
}

public static int countOf(CodeContext/*!*/ context, object? a, object? b) {
System.Collections.IEnumerator e = PythonOps.GetEnumerator(a);
System.Collections.IEnumerator e = PythonOps.GetEnumerator(context, a);
int count = 0;
while (e.MoveNext()) {
if (PythonOps.IsOrEqualsRetBool(context, e.Current, b)) {
Expand All @@ -294,7 +294,7 @@ public static object getitem(CodeContext/*!*/ context, object? a, object? b) {
}

public static int indexOf(CodeContext/*!*/ context, object? a, object? b) {
System.Collections.IEnumerator e = PythonOps.GetEnumerator(a);
System.Collections.IEnumerator e = PythonOps.GetEnumerator(context, a);
int index = 0;
while (e.MoveNext()) {
if (PythonOps.IsOrEqualsRetBool(context, e.Current, b)) {
Expand Down
4 changes: 2 additions & 2 deletions src/core/IronPython.Modules/array.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,8 +326,8 @@ public void extend(object? iterable) {
}
}

public void fromlist([NotNone] PythonList iterable) {
IEnumerator ie = PythonOps.GetEnumerator(iterable);
public void fromlist(CodeContext context, [NotNone] PythonList iterable) {
IEnumerator ie = PythonOps.GetEnumerator(context, iterable);

List<object> items = new List<object>();
while (ie.MoveNext()) {
Expand Down
2 changes: 1 addition & 1 deletion src/core/IronPython.Modules/select.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ private static void ProcessSocketSequence(CodeContext context, object sequence,
socketToOriginal = new Dictionary<Socket, object>();
socketList = new PythonList();

IEnumerator cursor = PythonOps.GetEnumerator(sequence);
IEnumerator cursor = PythonOps.GetEnumerator(context, sequence);
while (cursor.MoveNext()) {
object original = cursor.Current;
Socket socket = ObjectToSocket(context, original);
Expand Down
12 changes: 6 additions & 6 deletions src/core/IronPython/Modules/Builtin.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -768,7 +768,7 @@ public static object locals(CodeContext/*!*/ context) {
private static readonly object UndefinedKeywordArgument = new object();

public static object? max(CodeContext/*!*/ context, object? x) {
IEnumerator i = PythonOps.GetEnumerator(x);
IEnumerator i = PythonOps.GetEnumerator(context, x);
if (!i.MoveNext())
throw PythonOps.ValueError("max() arg is an empty sequence");
object? ret = i.Current;
Expand Down Expand Up @@ -804,7 +804,7 @@ public static object locals(CodeContext/*!*/ context) {
}

public static object? max(CodeContext/*!*/ context, object? x, [ParamDictionary] IDictionary<string, object?> dict) {
IEnumerator i = PythonOps.GetEnumerator(x);
IEnumerator i = PythonOps.GetEnumerator(context, x);

var kwargTuple = GetMaxKwArg(dict, isDefaultAllowed: true);
object? method = kwargTuple.Item1;
Expand Down Expand Up @@ -881,7 +881,7 @@ public static object locals(CodeContext/*!*/ context) {
}

public static object? min(CodeContext/*!*/ context, object? x) {
IEnumerator i = PythonOps.GetEnumerator(x);
IEnumerator i = PythonOps.GetEnumerator(context, x);
if (!i.MoveNext()) {
throw PythonOps.ValueError("empty sequence");
}
Expand Down Expand Up @@ -915,7 +915,7 @@ public static object locals(CodeContext/*!*/ context) {
}

public static object? min(CodeContext/*!*/ context, object? x, [ParamDictionary] IDictionary<string, object?> dict) {
IEnumerator i = PythonOps.GetEnumerator(x);
IEnumerator i = PythonOps.GetEnumerator(context, x);
var kwargTuple = GetMinKwArg(dict, isDefaultAllowed: true);
object? method = kwargTuple.Item1;
object? def = kwargTuple.Item2;
Expand Down Expand Up @@ -1371,7 +1371,7 @@ public static PythonList sorted(CodeContext/*!*/ context,
object? iterable,
[ParamDictionary] IDictionary<string, object> kwArgs) {

IEnumerator iter = PythonOps.GetEnumerator(iterable);
IEnumerator iter = PythonOps.GetEnumerator(context, iterable);
PythonList l = new PythonList(10);
while (iter.MoveNext()) {
l.AddNoLock(iter.Current);
Expand All @@ -1395,7 +1395,7 @@ public static PythonList sorted(CodeContext/*!*/ context,
}

public static object? sum(CodeContext/*!*/ context, object? sequence, object? start) {
IEnumerator i = PythonOps.GetEnumerator(sequence);
IEnumerator i = PythonOps.GetEnumerator(context, sequence);

ValidateSumStart(start);

Expand Down
2 changes: 1 addition & 1 deletion src/core/IronPython/Runtime/Binding/PythonBinaryOperationBinder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -567,7 +567,7 @@ private object ListAdd(CallSite site, object self, object other) {
private object ListAddAssign(CallSite site, object self, object other) {
if (self != null && self.GetType() == typeof(PythonList) &&
other != null && other.GetType() == typeof(PythonList)) {
return ((PythonList)self).InPlaceAdd(other);
return ((PythonList)self).InPlaceAdd(DefaultContext.Default, other);
}

return ((CallSite<Func<CallSite, object, object, object>>)site).Update(site, self, other);
Expand Down
4 changes: 2 additions & 2 deletions src/core/IronPython/Runtime/ByteArray.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,8 +585,8 @@ public bool isupper() {
/// in the sequence seq. The separator between elements is the
/// string providing this method
/// </summary>
public ByteArray join(object? iterable) {
IEnumerator seq = PythonOps.GetEnumerator(iterable);
public ByteArray join(CodeContext context, object? iterable) {
IEnumerator seq = PythonOps.GetEnumerator(context, iterable);
if (!seq.MoveNext()) {
return new ByteArray();
}
Expand Down
4 changes: 2 additions & 2 deletions src/core/IronPython/Runtime/Bytes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,8 +456,8 @@ public int index(BigInteger @byte, object? start, object? end)
/// in the sequence seq. The separator between elements is the
/// string providing this method
/// </summary>
public Bytes join(object? iterable) {
IEnumerator seq = PythonOps.GetEnumerator(iterable);
public Bytes join(CodeContext context, object? iterable) {
IEnumerator seq = PythonOps.GetEnumerator(context, iterable);
if (!seq.MoveNext()) {
return Empty;
}
Expand Down
10 changes: 5 additions & 5 deletions src/core/IronPython/Runtime/DictionaryOps.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,17 +138,17 @@ private static void SlowUpdate(CodeContext/*!*/ context, PythonDictionary/*!*/ s
}
} else if (PythonOps.TryGetBoundAttr(other, "keys", out object keysFunc)) {
// user defined dictionary
IEnumerator i = PythonOps.GetEnumerator(PythonCalls.Call(context, keysFunc));
IEnumerator i = PythonOps.GetEnumerator(context, PythonCalls.Call(context, keysFunc));
while (i.MoveNext()) {
self._storage.Add(ref self._storage, i.Current, PythonOps.GetIndex(context, other, i.Current));
}
} else {
// list of lists (key/value pairs), list of tuples,
// tuple of tuples, etc...
IEnumerator i = PythonOps.GetEnumerator(other);
IEnumerator i = PythonOps.GetEnumerator(context, other);
int index = 0;
while (i.MoveNext()) {
if (!AddKeyValue(self, i.Current)) {
if (!AddKeyValue(context, self, i.Current)) {
throw PythonOps.ValueError("dictionary update sequence element #{0} has bad length; 2 is required", index);
}
index++;
Expand Down Expand Up @@ -196,8 +196,8 @@ internal static bool TryGetValueVirtual(CodeContext context, PythonDictionary se
return false;
}

internal static bool AddKeyValue(PythonDictionary self, object o) {
IEnumerator i = PythonOps.GetEnumerator(o); //c.GetEnumerator();
internal static bool AddKeyValue(CodeContext context, PythonDictionary self, object o) {
IEnumerator i = PythonOps.GetEnumerator(context, o); //c.GetEnumerator();
if (i.MoveNext()) {
object key = i.Current;
if (i.MoveNext()) {
Expand Down
4 changes: 2 additions & 2 deletions src/core/IronPython/Runtime/Exceptions/PythonExceptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -775,8 +775,8 @@ internal static BaseException CreateBaseExceptionForRaise(CodeContext/*!*/ conte

if (PythonOps.IsInstance(value, type)) {
pyEx = value;
} else if (value is PythonTuple) {
pyEx = PythonOps.CallWithArgsTuple(type, [], value);
} else if (value is PythonTuple pt) {
pyEx = PythonOps.CallWithArgsTuple(type, [], pt);
} else if (value != null) {
pyEx = PythonCalls.Call(context, type, value);
} else {
Expand Down
2 changes: 1 addition & 1 deletion src/core/IronPython/Runtime/Importer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,7 +588,7 @@ private static bool TryGetNestedModule(CodeContext/*!*/ context, PythonModule/*!
Assert.NotNull(context, scope, name);
if (scope.__dict__.TryGetValue(name, out nested)) {
if (nested is PythonModule pm) {
var fullPath = ".".join(SubArray(parts, current));
var fullPath = string.Join(".", SubArray(parts, current));
// double check, some packages mess with package namespace
// see cp35116
if (pm.GetName() == fullPath) {
Expand Down
2 changes: 1 addition & 1 deletion src/core/IronPython/Runtime/Operations/ArrayOps.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public static object __new__(CodeContext context, PythonType pythonType, object
Array res = @base == 0 ?
Array.CreateInstance(type, len) : Array.CreateInstance(type, [len], [@base]);

IEnumerator ie = PythonOps.GetEnumerator(items);
IEnumerator ie = PythonOps.GetEnumerator(context, items);
int i = @base;
while (ie.MoveNext()) {
res.SetValue(Converter.Convert(ie.Current, type), i++);
Expand Down
2 changes: 1 addition & 1 deletion src/core/IronPython/Runtime/Operations/ByteOps.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ internal static IList<byte> GetBytes(object? value, bool useHint, CodeContext? c
int len = 0;
if (useHint) PythonOps.TryInvokeLengthHint(context ?? DefaultContext.Default, value, out len);
List<byte> ret = new List<byte>(len);
IEnumerator ie = PythonOps.GetEnumerator(value);
IEnumerator ie = PythonOps.GetEnumerator(context ?? DefaultContext.Default, value);
while (ie.MoveNext()) {
ret.Add(GetByte(ie.Current));
}
Expand Down
2 changes: 1 addition & 1 deletion src/core/IronPython/Runtime/Operations/ObjectOps.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ private static PythonTuple ReduceProtocol2(CodeContext/*!*/ context, object self

object? listIterator = null;
if (self is PythonList) {
listIterator = PythonOps.GetEnumerator(self);
listIterator = PythonOps.GetEnumerator(context, self);
}

object? dictIterator = null;
Expand Down
6 changes: 3 additions & 3 deletions src/core/IronPython/Runtime/Operations/PythonOps.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ internal static bool IsSubClass(CodeContext/*!*/ context, PythonType c, [NotNull

IEnumerator ie = PythonOps.GetEnumerator(bases);
while (ie.MoveNext()) {
if (!(ie.Current is PythonType baseType)) continue;
if (ie.Current is not PythonType baseType) continue;
if (c.IsSubclassOf(baseType)) return true;
}
return false;
Expand Down Expand Up @@ -1025,7 +1025,7 @@ internal static bool TryInvokeLengthHint(CodeContext context, object? sequence,
}
}

public static object? CallWithArgsTuple(object func, object?[] args, object argsTuple) {
public static object? CallWithArgsTuple(object func, object?[] args, IEnumerable argsTuple) {
Copy link
Contributor

@slozier slozier Apr 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess the goal with the signature change is to get rid of the context-less PythonOps.GetEnumerator call? Maybe adding the CodeContext argument would be "better"? Although looking at usage this function is basically useless and is a good candidate for removing/obsoleting...

Copy link
Member Author

@BCSharp BCSharp Apr 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The goal was to get rid of the call to PythonOps.GetEnumerator here altogether. Arguably, a call with the context is "better" (in most cases but not all) than the context-less call, but in both cases it goes though PythonContext.TryConvertToIEnumerable, which unnecessarily "pollutes" the call site cache. It just caught my attention while reviewing the usage of PythonOps.GetEnumerator in the codebase.

The signature has changed to a more restrictive one, indeed, but looking at the usage and the name of of the method, one can assume that the type of parameter argsTuple is supposed to be a tuple… Indeed, originally I have changed the type to PythonTuple here, but since it is a public method, I thought that having IEnumerable will be less of an inconvenience for the user code that happens to use this method. For a while I also considered of typing it as ITuple, but (1) it would require adding this interface to PythonTuple (perhaps not a bad idea anyway for other reasons) and (2) no code in the wild makes calls to this method with Tuple because it is not enumerable, so why bother.

I have no opinion about obsolescence of this method; if you think it is worthwhile removing then go ahead!

if (argsTuple is PythonTuple tp) {
object?[] nargs = new object[args.Length + tp.__len__()];
for (int i = 0; i < args.Length; i++) nargs[i] = args[i];
Expand All @@ -1035,7 +1035,7 @@ internal static bool TryInvokeLengthHint(CodeContext context, object? sequence,

PythonList allArgs = new PythonList(args.Length + 10);
allArgs.AddRange(args);
IEnumerator e = PythonOps.GetEnumerator(argsTuple);
IEnumerator e = argsTuple.GetEnumerator();
while (e.MoveNext()) allArgs.AddNoLock(e.Current);

return PythonCalls.Call(func, allArgs.GetObjectArray());
Expand Down
Loading