Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
bd5b81c
https://github.com/python/cpython/issues/129948: support `multiproces…
mmingyu Feb 10, 2025
d762661
fix lint
mmingyu Feb 10, 2025
3514b42
📜🤖 Added by blurb_it.
blurb-it[bot] Feb 10, 2025
e529f5f
📜🤖 Added by blurb_it.
blurb-it[bot] Feb 11, 2025
f0f7d25
remove first doc
mmingyu Feb 11, 2025
41cb4a6
remove doc
mmingyu Feb 11, 2025
4796816
📜🤖 Added by blurb_it.
blurb-it[bot] Feb 11, 2025
40c60cc
fix lint
mmingyu Feb 11, 2025
5e0977f
fix doc lint
mmingyu Feb 11, 2025
5c02940
doc: update comment
mmingyu Feb 11, 2025
3132d2a
Remove hard coding in MakeProxyType
mmingyu Feb 11, 2025
deb8b73
test: Seperate test_set, Add test_set_contain_all_method
mmingyu Feb 11, 2025
b0b3d3f
Update blurb to simply
mmingyu Feb 11, 2025
a381b1d
doc: multiprocessing and 3.14 what's news
mmingyu Feb 11, 2025
e332082
fix doc ref
mmingyu Feb 11, 2025
928ad6d
Add set document in multiprocessing.rst
mmingyu Feb 11, 2025
b89beb9
fix lint
mmingyu Feb 11, 2025
e88f5c7
Update Doc/whatsnew/3.14.rst
mmingyu Feb 12, 2025
3d5d828
Update _test_multiprocessing
mmingyu Feb 12, 2025
9c3478d
Update blurb
mmingyu Feb 12, 2025
7658e24
Add version change at multiprocessing.rst
mmingyu Feb 12, 2025
d2052a7
Fix typo
mmingyu Feb 12, 2025
74db275
update version change to next
mmingyu Feb 12, 2025
920e06a
Addressed all comment in reviewing
mmingyu Feb 12, 2025
12a65e3
refactor test
mmingyu Feb 12, 2025
d707691
Enhance set operator tests with additional assertions and edge case h…
mmingyu Feb 12, 2025
0eb6fdb
Add test case of __iter__, __xor__, __and__, __rand__
mmingyu Feb 12, 2025
dd295c3
Reordering set operator tests to improve clarity
mmingyu Feb 12, 2025
cf8ef27
Change register order after dict
mmingyu Feb 22, 2025
3706c78
Fix the bug where SetProxy's __ior__, __ixor__, __iand__, and __isub_…
mmingyu Feb 23, 2025
a259b39
Fix missing __ge__, __gt__, __le__, and __lt__ methods in SetProxy
mmingyu Feb 23, 2025
121b742
Addressed code review comments
mmingyu Feb 23, 2025
e8b449f
Hardcode SetProxy methods
mmingyu Feb 23, 2025
c3c3075
Grouping tests for issuperset and issubset
mmingyu Feb 23, 2025
3af48c3
Merge branch 'main' into feature/set-proxy
picnixz Feb 23, 2025
13c18c7
Update Lib/test/_test_multiprocessing.py
gpshead Feb 23, 2025
af37eae
Update Lib/multiprocessing/managers.py
gpshead Feb 23, 2025
a34e461
Update Lib/multiprocessing/managers.py
gpshead Feb 23, 2025
df6330a
Update Lib/test/_test_multiprocessing.py
gpshead Feb 23, 2025
9d13129
Update Doc/library/multiprocessing.rst
gpshead Feb 23, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions Doc/library/multiprocessing.rst
Original file line number Diff line number Diff line change
Expand Up @@ -380,35 +380,40 @@ However, if you really do need to use some shared data then
proxies.

A manager returned by :func:`Manager` will support types
:class:`list`, :class:`dict`, :class:`~managers.Namespace`, :class:`Lock`,
:class:`list`, :class:`dict`, :class:`set`, :class:`~managers.Namespace`, :class:`Lock`,
:class:`RLock`, :class:`Semaphore`, :class:`BoundedSemaphore`,
:class:`Condition`, :class:`Event`, :class:`Barrier`,
:class:`Queue`, :class:`Value` and :class:`Array`. For example, ::

from multiprocessing import Process, Manager

def f(d, l):
def f(d, l, s):
d[1] = '1'
d['2'] = 2
d[0.25] = None
l.reverse()
s.add('a')
s.add('b')

if __name__ == '__main__':
with Manager() as manager:
d = manager.dict()
l = manager.list(range(10))
s = manager.set()

p = Process(target=f, args=(d, l))
p = Process(target=f, args=(d, l, s))
p.start()
p.join()

print(d)
print(l)
print(s)

will print ::

{0.25: None, 1: '1', '2': 2}
[9, 8, 7, 6, 5, 4, 3, 2, 1, 0]
{'a', 'b'}

Server process managers are more flexible than using shared memory objects
because they can be made to support arbitrary object types. Also, a single
Expand Down Expand Up @@ -1942,6 +1947,15 @@ their parent process exits. The manager classes are defined in the

Create a shared :class:`list` object and return a proxy for it.

.. method:: set()
set(sequence)
set(mapping)

Create a shared :class:`set` object and return a proxy for it.

.. versionadded:: next
:class:`set` support was added.

.. versionchanged:: 3.6
Shared objects are capable of being nested. For example, a shared
container object such as a shared list can contain other shared objects
Expand Down
5 changes: 5 additions & 0 deletions Doc/whatsnew/3.14.rst
Original file line number Diff line number Diff line change
Expand Up @@ -700,6 +700,11 @@ multiprocessing

(Contributed by Roy Hyunjin Han for :gh:`103134`.)

* Add support for shared :class:`set` objects via
:meth:`SyncManager.set() <multiprocessing.managers.SyncManager.set>`.
The :func:`set` in :func:`multiprocessing.Manager` method is now available.
(Contributed by Mingyu Park in :gh:`129949`.)


operator
--------
Expand Down
31 changes: 31 additions & 0 deletions Lib/multiprocessing/managers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1195,6 +1195,36 @@ def __ior__(self, value):

collections.abc.MutableMapping.register(_BaseDictProxy)

_BaseSetProxy = MakeProxyType("_BaseSetProxy", (
'__and__', '__class_getitem__', '__contains__', '__iand__', '__ior__',
'__isub__', '__iter__', '__ixor__', '__len__', '__or__', '__rand__',
'__ror__', '__rsub__', '__rxor__', '__sub__', '__xor__',
'__ge__', '__gt__', '__le__', '__lt__',
'add', 'clear', 'copy', 'difference', 'difference_update', 'discard',
'intersection', 'intersection_update', 'isdisjoint', 'issubset',
'issuperset', 'pop', 'remove', 'symmetric_difference',
'symmetric_difference_update', 'union', 'update',
))

class SetProxy(_BaseSetProxy):
def __ior__(self, value):
self._callmethod('__ior__', (value,))
return self
def __iand__(self, value):
self._callmethod('__iand__', (value,))
return self
def __ixor__(self, value):
self._callmethod('__ixor__', (value,))
return self
def __isub__(self, value):
self._callmethod('__isub__', (value,))
return self

__class_getitem__ = classmethod(types.GenericAlias)

collections.abc.MutableMapping.register(_BaseSetProxy)


ArrayProxy = MakeProxyType('ArrayProxy', (
'__len__', '__getitem__', '__setitem__'
))
Expand Down Expand Up @@ -1245,6 +1275,7 @@ class SyncManager(BaseManager):
SyncManager.register('Pool', pool.Pool, PoolProxy)
SyncManager.register('list', list, ListProxy)
SyncManager.register('dict', dict, DictProxy)
SyncManager.register('set', set, SetProxy)
SyncManager.register('Value', Value, ValueProxy)
SyncManager.register('Array', Array, ArrayProxy)
SyncManager.register('Namespace', Namespace, NamespaceProxy)
Expand Down
144 changes: 144 additions & 0 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -6441,6 +6441,150 @@ def test_namespace(self):
o.y = 1
self.run_worker(self._test_namespace, o)

@classmethod
def _test_set_operator_symbols(cls, obj):
case = unittest.TestCase()
obj.update(['a', 'b', 'c'])
case.assertEqual(len(obj), 3)
case.assertIn('a', obj)
case.assertNotIn('d', obj)
result = obj | {'d', 'e'}
case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'})
result = {'d', 'e'} | obj
case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'})
obj |= {'d', 'e'}
case.assertSetEqual(obj, {'a', 'b', 'c', 'd', 'e'})
case.assertIsInstance(obj, multiprocessing.managers.SetProxy)

obj.clear()
obj.update(['a', 'b', 'c'])
result = {'a', 'b', 'd'} - obj
case.assertSetEqual(result, {'d'})
result = obj - {'a', 'b'}
case.assertSetEqual(result, {'c'})
obj -= {'a', 'b'}
case.assertSetEqual(obj, {'c'})
case.assertIsInstance(obj, multiprocessing.managers.SetProxy)

obj.clear()
obj.update(['a', 'b', 'c'])
result = {'b', 'c', 'd'} ^ obj
case.assertSetEqual(result, {'a', 'd'})
result = obj ^ {'b', 'c', 'd'}
case.assertSetEqual(result, {'a', 'd'})
obj ^= {'b', 'c', 'd'}
case.assertSetEqual(obj, {'a', 'd'})
case.assertIsInstance(obj, multiprocessing.managers.SetProxy)

obj.clear()
obj.update(['a', 'b', 'c'])
result = obj & {'b', 'c', 'd'}
case.assertSetEqual(result, {'b', 'c'})
result = {'b', 'c', 'd'} & obj
case.assertSetEqual(result, {'b', 'c'})
obj &= {'b', 'c', 'd'}
case.assertSetEqual(obj, {'b', 'c'})
case.assertIsInstance(obj, multiprocessing.managers.SetProxy)

obj.clear()
obj.update(['a', 'b', 'c'])
case.assertSetEqual(set(obj), {'a', 'b', 'c'})

@classmethod
def _test_set_operator_methods(cls, obj):
case = unittest.TestCase()
obj.add('d')
case.assertIn('d', obj)

obj.clear()
obj.update(['a', 'b', 'c'])
copy_obj = obj.copy()
case.assertSetEqual(copy_obj, obj)
obj.remove('a')
case.assertNotIn('a', obj)
case.assertRaises(KeyError, obj.remove, 'a')

obj.clear()
obj.update(['a'])
obj.discard('a')
case.assertNotIn('a', obj)
obj.discard('a')
case.assertNotIn('a', obj)
obj.update(['a'])
popped = obj.pop()
case.assertNotIn(popped, obj)

obj.clear()
obj.update(['a', 'b', 'c'])
result = obj.intersection({'b', 'c', 'd'})
case.assertSetEqual(result, {'b', 'c'})
obj.intersection_update({'b', 'c', 'd'})
case.assertSetEqual(obj, {'b', 'c'})

obj.clear()
obj.update(['a', 'b', 'c'])
result = obj.difference({'a', 'b'})
case.assertSetEqual(result, {'c'})
obj.difference_update({'a', 'b'})
case.assertSetEqual(obj, {'c'})

obj.clear()
obj.update(['a', 'b', 'c'])
result = obj.symmetric_difference({'b', 'c', 'd'})
case.assertSetEqual(result, {'a', 'd'})
obj.symmetric_difference_update({'b', 'c', 'd'})
case.assertSetEqual(obj, {'a', 'd'})

@classmethod
def _test_set_comparisons(cls, obj):
case = unittest.TestCase()
obj.update(['a', 'b', 'c'])
result = obj.union({'d', 'e'})
case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'})
case.assertTrue(obj.isdisjoint({'d', 'e'}))
case.assertFalse(obj.isdisjoint({'a', 'd'}))

case.assertTrue(obj.issubset({'a', 'b', 'c', 'd'}))
case.assertFalse(obj.issubset({'a', 'b'}))
case.assertLess(obj, {'a', 'b', 'c', 'd'})
case.assertLessEqual(obj, {'a', 'b', 'c'})

case.assertTrue(obj.issuperset({'a', 'b'}))
case.assertFalse(obj.issuperset({'a', 'b', 'd'}))
case.assertGreater(obj, {'a'})
case.assertGreaterEqual(obj, {'a', 'b'})

def test_set(self):
o = self.manager.set()
self.run_worker(self._test_set_operator_symbols, o)
o = self.manager.set()
self.run_worker(self._test_set_operator_methods, o)
o = self.manager.set()
self.run_worker(self._test_set_comparisons, o)

def test_set_init(self):
o = self.manager.set({'a', 'b', 'c'})
self.assertSetEqual(o, {'a', 'b', 'c'})
o = self.manager.set(["a", "b", "c"])
self.assertSetEqual(o, {"a", "b", "c"})
o = self.manager.set({"a": 1, "b": 2, "c": 3})
self.assertSetEqual(o, {"a", "b", "c"})
self.assertRaises(RemoteError, self.manager.set, 1234)

def test_set_contain_all_method(self):
o = self.manager.set()
set_methods = {
'__and__', '__class_getitem__', '__contains__', '__iand__', '__ior__',
'__isub__', '__iter__', '__ixor__', '__len__', '__or__', '__rand__',
'__ror__', '__rsub__', '__rxor__', '__sub__', '__xor__',
'__ge__', '__gt__', '__le__', '__lt__',
'add', 'clear', 'copy', 'difference', 'difference_update', 'discard',
'intersection', 'intersection_update', 'isdisjoint', 'issubset',
'issuperset', 'pop', 'remove', 'symmetric_difference',
'symmetric_difference_update', 'union', 'update',
}
self.assertLessEqual(set_methods, set(dir(o)))


class TestNamedResource(unittest.TestCase):
@only_run_in_spawn_testsuite("spawn specific test.")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Add support for shared :class:`set` to :class:`multiprocessing.managers.SyncManager`
via :meth:`SyncManager.set() <multiprocessing.managers.SyncManager.set>`.
Loading