Skip to content

Commit b9234de

Browse files
author
Release Manager
committed
Trac #34815: pep cleanup in parallel/
also some pylint and codespell fixes there URL: https://trac.sagemath.org/34815 Reported by: chapoton Ticket author(s): Frédéric Chapoton Reviewer(s): Matthias Koeppe
2 parents eff11ab + f930224 commit b9234de

File tree

6 files changed

+78
-76
lines changed

6 files changed

+78
-76
lines changed

src/sage/parallel/decorate.py

Lines changed: 34 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ def normalize_input(a):
2323
2424
INPUT:
2525
26-
- ``a`` -- object
26+
- ``a`` -- object
2727
2828
OUTPUT:
2929
30-
- ``args`` -- tuple
31-
- ``kwds`` -- dictionary
30+
- ``args`` -- tuple
31+
- ``kwds`` -- dictionary
3232
3333
EXAMPLES::
3434
@@ -41,7 +41,7 @@ def normalize_input(a):
4141
sage: sage.parallel.decorate.normalize_input( 5 )
4242
((5,), {})
4343
"""
44-
if isinstance(a, tuple) and len(a) == 2 and isinstance(a[0],tuple) and isinstance(a[1],dict):
44+
if isinstance(a, tuple) and len(a) == 2 and isinstance(a[0], tuple) and isinstance(a[1], dict):
4545
return a
4646
elif isinstance(a, tuple):
4747
return (a, {})
@@ -97,11 +97,11 @@ def __call__(self, f):
9797
9898
INPUT:
9999
100-
- ``f`` -- Python callable object or function
100+
- ``f`` -- Python callable object or function
101101
102102
OUTPUT:
103103
104-
- Decorated version of ``f``
104+
- Decorated version of ``f``
105105
106106
EXAMPLES::
107107
@@ -131,8 +131,8 @@ def __init__(self, parallel, func):
131131
"""
132132
.. note::
133133
134-
This is typically accessed indirectly through
135-
:meth:`Parallel.__call__`.
134+
This is typically accessed indirectly through
135+
:meth:`Parallel.__call__`.
136136
137137
INPUT:
138138
@@ -159,13 +159,11 @@ def __call__(self, *args, **kwds):
159159
4
160160
sage: sorted(pf([2,3]))
161161
[(((2,), {}), 4), (((3,), {}), 9)]
162-
"""
163-
if len(args) > 0 and isinstance(args[0], (list,
164-
types.GeneratorType)):
162+
"""
163+
if len(args) > 0 and isinstance(args[0], (list, types.GeneratorType)):
165164
return self.parallel.p_iter(self.func, (normalize_input(a)
166-
for a in args[0]))
167-
else:
168-
return self.func(*args, **kwds)
165+
for a in args[0]))
166+
return self.func(*args, **kwds)
169167

170168
def __get__(self, instance, owner):
171169
"""
@@ -174,7 +172,7 @@ def __get__(self, instance, owner):
174172
175173
.. note::
176174
177-
This is the key to fixing :trac:`11461`.
175+
This is the key to fixing :trac:`11461`.
178176
179177
EXAMPLES:
180178
@@ -217,14 +215,14 @@ def __get__(self, instance, owner):
217215
[(((2,), {}), 4), (((3,), {}), 9)]
218216
"""
219217
try:
220-
#If this ParallelFunction object is accessed as an
221-
#attribute of a class or instance, the underlying function
222-
#should be "accessed" in the same way.
218+
# If this ParallelFunction object is accessed as an
219+
# attribute of a class or instance, the underlying function
220+
# should be "accessed" in the same way.
223221
new_func = self.func.__get__(instance, owner)
224222
except AttributeError:
225-
#This will happen if a non-function attribute is
226-
#decorated. For example, an expression that's an
227-
#attribute of a class.
223+
# This will happen if a non-function attribute is
224+
# decorated. For example, an expression that's an
225+
# attribute of a class.
228226
new_func = self.func
229227
return ParallelFunction(self.parallel, new_func)
230228

@@ -413,8 +411,6 @@ def parallel(p_iter='fork', ncpus=None, **kwds):
413411
return Parallel(p_iter, ncpus, **kwds)
414412

415413

416-
417-
418414
###################################################################
419415
# The @fork decorator -- evaluate a function with no side effects
420416
# in memory, so the only side effects (if any) are on disk.
@@ -437,10 +433,10 @@ def __init__(self, timeout=0, verbose=False):
437433
"""
438434
INPUT:
439435
440-
- ``timeout`` -- (default: 0) kill the subprocess after it has run this
441-
many seconds (wall time), or if ``timeout`` is zero, do not kill it.
442-
- ``verbose`` -- (default: ``False``) whether to print anything about
443-
what the decorator does (e.g., killing the subprocess)
436+
- ``timeout`` -- (default: 0) kill the subprocess after it has run this
437+
many seconds (wall time), or if ``timeout`` is zero, do not kill it.
438+
- ``verbose`` -- (default: ``False``) whether to print anything about
439+
what the decorator does (e.g., killing the subprocess)
444440
445441
EXAMPLES::
446442
@@ -456,11 +452,11 @@ def __call__(self, f):
456452
"""
457453
INPUT:
458454
459-
- ``f`` -- a function
455+
- ``f`` -- a function
460456
461457
OUTPUT:
462458
463-
- A decorated function.
459+
- A decorated function.
464460
465461
EXAMPLES::
466462
@@ -481,18 +477,18 @@ def h(*args, **kwds):
481477

482478
def fork(f=None, timeout=0, verbose=False):
483479
"""
484-
Decorate a function so that when called it runs in a forked
485-
subprocess. This means that it won't have any in-memory
486-
side effects on the parent Sage process. The pexpect interfaces
487-
are all reset.
480+
Decorate a function so that when called it runs in a forked subprocess.
481+
482+
This means that it will not have any in-memory side effects on the
483+
parent Sage process. The pexpect interfaces are all reset.
488484
489485
INPUT:
490486
491-
- ``f`` -- a function
492-
- ``timeout`` -- (default: 0) if positive, kill the subprocess after
493-
this many seconds (wall time)
494-
- ``verbose`` -- (default: ``False``) whether to print anything
495-
about what the decorator does (e.g., killing the subprocess)
487+
- ``f`` -- a function
488+
- ``timeout`` -- (default: 0) if positive, kill the subprocess after
489+
this many seconds (wall time)
490+
- ``verbose`` -- (default: ``False``) whether to print anything
491+
about what the decorator does (e.g., killing the subprocess)
496492
497493
.. warning::
498494

src/sage/parallel/map_reduce.py

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -410,7 +410,7 @@
410410
- ``worker._request`` -- a :class:`~multiprocessing.queues.SimpleQueue` storing
411411
steal request submitted to ``worker``.
412412
- ``worker._read_task``, ``worker._write_task`` -- a
413-
:class:`~multiprocessing.queues.Pipe` used to transfert node during steal.
413+
:class:`~multiprocessing.queues.Pipe` used to transfer node during steal.
414414
- ``worker._thief`` -- a :class:`~threading.Thread` which is in charge of
415415
stealing from ``worker._todo``.
416416
@@ -1196,9 +1196,9 @@ def get_results(self, timeout=None):
11961196
active_proc = self._nprocess
11971197
while active_proc > 0:
11981198
try:
1199-
logger.debug('Waiting on results; active_proc: %s, '
1200-
'timeout: %s, aborted: %s' %
1201-
(active_proc, timeout, self._aborted.value))
1199+
logger.debug('Waiting on results; active_proc: {}, '
1200+
'timeout: {}, aborted: {}'.format(
1201+
active_proc, timeout, self._aborted.value))
12021202
newres = self._results.get(timeout=timeout)
12031203
except queue.Empty:
12041204
logger.debug('Timed out waiting for results; aborting')
@@ -1249,13 +1249,13 @@ def finish(self):
12491249
if not self._aborted.value:
12501250
logger.debug("Joining worker processes...")
12511251
for worker in self._workers:
1252-
logger.debug("Joining %s" % worker.name)
1252+
logger.debug(f"Joining {worker.name}")
12531253
worker.join()
12541254
logger.debug("Joining done")
12551255
else:
12561256
logger.debug("Killing worker processes...")
12571257
for worker in self._workers:
1258-
logger.debug("Terminating %s" % worker.name)
1258+
logger.debug(f"Terminating {worker.name}")
12591259
worker.terminate()
12601260
logger.debug("Killing done")
12611261

@@ -1527,9 +1527,9 @@ def print_communication_statistics(self, blocksize=16):
15271527
# local function (see e.g:
15281528
# https://stackoverflow.com/questions/2609518/python-nested-function-scopes).
15291529

1530-
def pstat(name, start, end, ist):
1530+
def pstat(name, start, end, istat):
15311531
res[0] += ("\n" + name + " ".join(
1532-
"%4i" % (self._stats[i][ist]) for i in range(start, end)))
1532+
"%4i" % (self._stats[i][istat]) for i in range(start, end)))
15331533
for start in range(0, self._nprocess, blocksize):
15341534
end = min(start + blocksize, self._nprocess)
15351535
res[0] = ("#proc: " +
@@ -1613,17 +1613,17 @@ def _thief(self):
16131613
for ireq in iter(self._request.get, AbortError):
16141614
reqs += 1
16151615
target = self._mapred._workers[ireq]
1616-
logger.debug("Got a Steal request from %s" % target.name)
1616+
logger.debug(f"Got a Steal request from {target.name}")
16171617
self._mapred._signal_task_start()
16181618
try:
16191619
work = self._todo.popleft()
16201620
except IndexError:
16211621
target._write_task.send(None)
1622-
logger.debug("Failed Steal %s" % target.name)
1622+
logger.debug(f"Failed Steal {target.name}")
16231623
self._mapred._signal_task_done()
16241624
else:
16251625
target._write_task.send(work)
1626-
logger.debug("Succesful Steal %s" % target.name)
1626+
logger.debug(f"Successful Steal {target.name}")
16271627
thefts += 1
16281628
except AbortError:
16291629
logger.debug("Thief aborted")
@@ -1668,10 +1668,10 @@ def steal(self):
16681668
while node is None:
16691669
victim = self._mapred.random_worker()
16701670
if victim is not self:
1671-
logger.debug("Trying to steal from %s" % victim.name)
1671+
logger.debug(f"Trying to steal from {victim.name}")
16721672
victim._request.put(self._iproc)
16731673
self._stats[0] += 1
1674-
logger.debug("waiting for steal answer from %s" % victim.name)
1674+
logger.debug(f"waiting for steal answer from {victim.name}")
16751675
node = self._read_task.recv()
16761676
# logger.debug("Request answer: %s" % (node,))
16771677
if node is AbortError:
@@ -1713,7 +1713,7 @@ def run(self):
17131713
PROFILER.runcall(self.run_myself)
17141714

17151715
output = profile + str(self._iproc)
1716-
logger.warn("Profiling in %s ..." % output)
1716+
logger.warn(f"Profiling in {output} ...")
17171717
PROFILER.dump_stats(output)
17181718
else:
17191719
self.run_myself()

src/sage/parallel/multiprocessing_sage.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,15 @@
88
# Distributed under the terms of (any version of) the GNU
99
# General Public License (GPL). The full text of the GPL is available at:
1010
#
11-
# http://www.gnu.org/licenses/
11+
# https://www.gnu.org/licenses/
1212
################################################################################
1313

1414
from multiprocessing import Pool
1515
from functools import partial
1616
from sage.misc.fpickle import pickle_function, call_pickled_function
1717
from . import ncpus
1818

19+
1920
def pyprocessing(processes=0):
2021
"""
2122
Return a parallel iterator using a given number of processes

src/sage/parallel/ncpus.py

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import os
3434
import subprocess
3535

36+
3637
def ncpus():
3738
"""
3839
Detects the number of effective CPUs in the system.
@@ -52,22 +53,25 @@ def ncpus():
5253
else:
5354
return int(n)
5455

55-
#for Linux, Unix and MacOS
56+
# for Linux, Unix and MacOS
5657
if hasattr(os, "sysconf"):
5758
if "SC_NPROCESSORS_ONLN" in os.sysconf_names:
58-
#Linux and Unix
59+
# Linux and Unix
5960
ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
6061
if isinstance(ncpus, int) and ncpus > 0:
6162
return ncpus
6263
else:
63-
#MacOS X
64-
#deprecated: return int(os.popen2("sysctl -n hw.ncpu")[1].read())
65-
process = subprocess.Popen("sysctl -n hw.ncpu", shell=True, stdin=subprocess.PIPE, stdout = subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
64+
# MacOS X
65+
# deprecated: return int(os.popen2("sysctl -n hw.ncpu")[1].read())
66+
process = subprocess.Popen("sysctl -n hw.ncpu", shell=True,
67+
stdin=subprocess.PIPE,
68+
stdout=subprocess.PIPE,
69+
stderr=subprocess.PIPE, close_fds=True)
6670
return int(process.stdout.read())
67-
#for Windows
71+
# for Windows
6872
if "NUMBER_OF_PROCESSORS" in os.environ:
6973
ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
7074
if ncpus > 0:
7175
return ncpus
72-
#return the default value
76+
# return the default value
7377
return 1

src/sage/parallel/parallelism.py

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from sage.parallel.ncpus import ncpus
2929
from sage.rings.integer import Integer
3030

31+
3132
class Parallelism(Singleton, SageObject):
3233
r"""
3334
Singleton class for managing the number of processes used in parallel
@@ -106,9 +107,12 @@ def __init__(self):
106107
sage: TestSuite(par).run()
107108
108109
"""
109-
self._default = ncpus() # default number of proc. used in parallelizations
110-
self._nproc = {'tensor' : 1, 'linbox' : 1} # dict. of number of processes to be used
111-
# (keys: computational field)
110+
self._default = ncpus()
111+
# default number of proc. used in parallelizations
112+
113+
self._nproc = {'tensor': 1, 'linbox': 1}
114+
# dict. of number of processes to be used
115+
# (keys: computational field)
112116

113117
def _repr_(self):
114118
r"""
@@ -242,7 +246,7 @@ def set(self, field=None, nproc=None):
242246
if nproc is None:
243247
self._nproc[field] = self._default
244248
else:
245-
if not isinstance(nproc, (int,Integer)):
249+
if not isinstance(nproc, (int, Integer)):
246250
raise TypeError("nproc must be integer")
247251
self._nproc[field] = nproc
248252

@@ -281,7 +285,6 @@ def get(self, field):
281285
"implemented in Parallelism()")
282286
return self._nproc[field]
283287

284-
285288
def get_all(self):
286289
r"""
287290
Return the number of processes which will be used in parallel
@@ -307,7 +310,6 @@ def get_all(self):
307310
"""
308311
return self._nproc
309312

310-
311313
def set_default(self, nproc=None):
312314
r"""
313315
Set the default number of processes to be launched in parallel
@@ -343,7 +345,7 @@ def set_default(self, nproc=None):
343345
if nproc is None:
344346
self._default = ncpus()
345347
else:
346-
if not isinstance(nproc,(int,Integer)):
348+
if not isinstance(nproc, (int, Integer)):
347349
raise TypeError("nproc must be integer")
348350
self._default = nproc
349351

0 commit comments

Comments
 (0)