Skip to content

Commit cb4af6e

Browse files
author
Kurt Smith
committed
Merge branch 'master' into feature/bump-0.4
Conflicts: setup.py
2 parents 149e24b + 9e2fd27 commit cb4af6e

File tree

7 files changed

+220
-43
lines changed

7 files changed

+220
-43
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,11 @@ install:
4949
# ----------------------------------------------------------------------------
5050

5151
test_client:
52-
${PYTHON} -m unittest discover
52+
${PYTHON} -m unittest discover -c
5353
.PHONY: test_client
5454

5555
test_client_with_coverage:
56-
${COVERAGE} run -pm unittest discover -v
56+
${COVERAGE} run -pm unittest discover -cv
5757
.PHONY: test_client_with_coverage
5858

5959
${PARALLEL_OUT_DIR} :

‎README.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
1+
.. Travis badge
12
.. image:: https://travis-ci.org/enthought/distarray.png?branch=master
23
:target: https://travis-ci.org/enthought/distarray
34

5+
.. Coveralls badge
46
.. image:: https://coveralls.io/repos/enthought/distarray/badge.png?branch=master
57
:target: https://coveralls.io/r/enthought/distarray?branch=master
8+
9+
.. All content before the next comment will be stripped off for release.
10+
.. *** begin README content ***
611
712
DistArray
813
=========

‎distarray/dist/distarray.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,7 @@ def view(self, dtype=None):
448448
449449
Note
450450
----
451-
No redistribution is done. The sizes of the all localarrays must be
451+
No redistribution is done. The sizes of all `LocalArray`\s must be
452452
compatible with the new view.
453453
"""
454454
# special case for same dtype

‎distarray/dist/maps.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,7 @@ class Distribution(object):
501501

502502
@classmethod
503503
def from_maps(cls, context, maps, targets=None):
504-
"""Create a Distribution from a sequence of `Map`s.
504+
"""Create a Distribution from a sequence of `Map`\s.
505505
506506
Parameters
507507
----------
@@ -612,7 +612,17 @@ def __new__(cls, context, shape, dist=None, grid_shape=None, targets=None):
612612

613613
# list of `ClientMap` objects, one per dimension.
614614
maps = [map_from_sizes(*args) for args in zip(shape, dist, grid_shape)]
615-
return cls.from_maps(context=context, maps=maps, targets=targets)
615+
616+
self = cls.from_maps(context=context, maps=maps, targets=targets)
617+
618+
# TODO: FIXME: this is a workaround. The reason we slice here is to
619+
# return a distribution with no empty local shapes. The `from_maps()`
620+
# classmethod should be fixed to ensure no empty local arrays are
621+
# created in the first place. That will remove the need to slice the
622+
# distribution to remove empty localshapes.
623+
if all(d in ('n', 'b') for d in self.dist):
624+
self = self.slice((slice(None),)*self.ndim)
625+
return self
616626

617627
@classmethod
618628
def from_global_dim_data(cls, context, global_dim_data, targets=None):

‎distarray/dist/tests/test_maps.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,5 +300,23 @@ def test_all_n_dist(self):
300300
self.context.ones(distribution)
301301

302302

303+
class TestNoEmptyLocals(ContextTestCase):
304+
305+
def test_no_empty_local_arrays_4_targets(self):
306+
for n in range(1, 20):
307+
dist = Distribution(self.context, shape=(n,),
308+
dist=('b',),
309+
targets=self.context.targets[:4])
310+
for ls in dist.localshapes():
311+
self.assertNotIn(0, ls)
312+
313+
def test_no_empty_local_arrays_3_targets(self):
314+
for n in range(1, 20):
315+
dist = Distribution(self.context, shape=(n,),
316+
dist=('b',),
317+
targets=self.context.targets[:3])
318+
for ls in dist.localshapes():
319+
self.assertNotIn(0, ls)
320+
303321
if __name__ == '__main__':
304322
unittest.main(verbosity=2)

‎docs/source/releases/release-0.4.rst‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
==============================================================================
2+
DistArray 0.4 development release
3+
==============================================================================
4+
5+
**Documentation:** http://distarray.readthedocs.org
6+
7+
**License:** Three-clause BSD
8+
9+
**Python versions:** 2.7 and 3.3
10+
11+
**OS support:** \*nix and Mac OS X
12+
13+
What is DistArray?
14+
------------------
15+
16+
DistArray aims to bring the strengths of NumPy to data-parallel
17+
high-performance computing. It provides distributed multi-dimensional
18+
NumPy-like arrays and distributed ufuncs, distributed IO capabilities, and can
19+
integrate with external distributed libraries like Trilinos. DistArray works
20+
with NumPy and builds on top of it in a flexible and natural way.
21+
22+
0.4 Release
23+
-----------
24+
25+
This is the third development release.
26+
27+
Noteworthy improvements in 0.4 include:
28+
29+
* basic slicing support;
30+
* significant performance enhancements;
31+
* reduction methods now support boolean arrays;
32+
* an IPython notebook that demos basic functionality; and
33+
* many bug fixes, API improvements, and refactorings.
34+
35+
DistArray is nearly ready for real-world use. The project is evolving rapidly
36+
and input from the larger scientific-Python community is very valuable and
37+
helps drive development.
38+
39+
Existing features
40+
-----------------
41+
42+
DistArray:
43+
44+
* has a client-engine (or master-worker) process design -- data resides on the
45+
worker processes, and commands are initiated from master;
46+
* allows full control over what is executed on the worker processes and
47+
integrates transparently with the master process;
48+
* allows direct communication between workers, bypassing the master process
49+
for scalability;
50+
* integrates with IPython.parallel for interactive creation and exploration of
51+
distributed data;
52+
* supports distributed ufuncs (currently without broadcasting);
53+
* builds on and leverages MPI via MPI4Py in a transparent and user-friendly
54+
way;
55+
* supports NumPy-like multidimensional arrays;
56+
* has basic support for unstructured arrays;
57+
* supports user-controllable array distributions across workers (block,
58+
cyclic, block-cyclic, and unstructured) on a per-axis basis;
59+
* has a straightforward API to control how an array is distributed;
60+
* has basic plotting support for visualization of array distributions;
61+
* separates the array’s distribution from the array’s data -- useful for
62+
slicing, reductions, redistribution, broadcasting, and other operations;
63+
* implements distributed random arrays;
64+
* supports ``.npy``-like flat-file IO and hdf5 parallel IO (via ``h5py``);
65+
leverages MPI-based IO parallelism in an easy-to-use and transparent way;
66+
and
67+
* supports the distributed array protocol [protocol]_, which allows
68+
independently developed parallel libraries to share distributed arrays
69+
without copying, analogous to the PEP-3118 new buffer protocol.
70+
71+
Planned features and roadmap
72+
----------------------------
73+
74+
Near-term features and improvements include:
75+
76+
* MPI-only communication for performance and deployment on clusters and
77+
supercomputers;
78+
* array re-distribution capabilities;
79+
* interoperation with Trilinos [Trilinos]_;
80+
* expanded tutorials, examples, and other introductory material; and
81+
* distributed broadcasting support.
82+
83+
The longer-term roadmap includes:
84+
85+
* Lazy evaluation and deferred computation for latency hiding;
86+
* Integration with other packages [petsc]_ that subscribe to the distributed
87+
array protocol [protocol]_;
88+
* Distributed fancy indexing;
89+
* Out-of-core computations;
90+
* Support for distributed sorting and other non-trivial distributed
91+
algorithms; and
92+
* End-user control over communication and temporary array creation, and other
93+
performance aspects of distributed computations.
94+
95+
History and funding
96+
-------------------
97+
98+
Brian Granger started DistArray as a NASA-funded SBIR project in 2008.
99+
Enthought picked it up as part of a DOE Phase II SBIR [SBIR]_ to provide a
100+
generally useful distributed array package. It builds on NumPy, MPI, MPI4Py,
101+
IPython, IPython.parallel, and interfaces with the Trilinos suite of
102+
distributed HPC solvers (via PyTrilinos [Trilinos]_).
103+
104+
This material is based upon work supported by the Department of Energy under
105+
Award Number DE-SC0007699.
106+
107+
This report was prepared as an account of work sponsored by an agency of the
108+
United States Government. Neither the United States Government nor any agency
109+
thereof, nor any of their employees, makes any warranty, express or implied,
110+
or assumes any legal liability or responsibility for the accuracy,
111+
completeness, or usefulness of any information, apparatus, product, or process
112+
disclosed, or represents that its use would not infringe privately owned
113+
rights. Reference herein to any specific commercial product, process, or
114+
service by trade name, trademark, manufacturer, or otherwise does not
115+
necessarily constitute or imply its endorsement, recommendation, or favoring
116+
by the United States Government or any agency thereof. The views and opinions
117+
of authors expressed herein do not necessarily state or reflect those of the
118+
United States Government or any agency thereof.
119+
120+
121+
.. [protocol] http://distributed-array-protocol.readthedocs.org/en/rel-0.10.0/
122+
.. [Trilinos] http://trilinos.org/
123+
.. [petsc] http://www.mcs.anl.gov/petsc/
124+
.. [SBIR] http://www.sbir.gov/sbirsearch/detail/410257
125+
126+
.. vim:spell

‎setup.py‎

Lines changed: 56 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -6,41 +6,59 @@
66

77
from setuptools import setup, find_packages
88

9-
install_requires = [
10-
'ipython',
11-
'numpy',
12-
'mpi4py'
13-
]
14-
15-
metadata = {
16-
'name': 'distarray',
17-
'version': '0.4.0',
18-
'description': 'Distributed Memory Arrays for Python',
19-
'keywords': 'parallel mpi distributed array',
20-
'license': 'New BSD',
21-
'author': 'IPython Development Team and Enthought, Inc.',
22-
'author_email': '[email protected]',
23-
'url': 'https://github.com/enthought/distarray',
24-
'packages': find_packages(),
25-
'install_requires': install_requires,
26-
'long_description': open('README.rst').read(),
27-
'platforms': ["Linux", "Mac OS-X"],
28-
'entry_points': {'console_scripts': ['dacluster = '
29-
'distarray.apps.dacluster:main']},
30-
'classifiers': [c.strip() for c in """\
31-
Development Status :: 2 - Pre-Alpha
32-
Intended Audience :: Developers
33-
Intended Audience :: Science/Research
34-
License :: OSI Approved :: BSD License
35-
Operating System :: MacOS
36-
Operating System :: OS Independent
37-
Operating System :: POSIX
38-
Operating System :: Unix
39-
Programming Language :: Python
40-
Topic :: Scientific/Engineering
41-
Topic :: Software Development
42-
Topic :: Software Development :: Libraries
43-
""".splitlines() if len(c.strip()) > 0],
44-
}
45-
46-
setup(**metadata)
9+
10+
def parse_readme(filename='README.rst', sentinel="README"):
11+
"""
12+
Return file `filename` as a string.
13+
14+
Skips lines until it finds a comment that contains `sentinel` text. This
15+
effectively strips off any badges or other undesirable content.
16+
"""
17+
with open(filename, 'r') as fp:
18+
lines = fp.readlines()
19+
for idx, line in enumerate(lines):
20+
if line.startswith('..') and sentinel in line:
21+
break
22+
return "".join(lines[idx+1:])
23+
24+
25+
if __name__ == "__main__":
26+
27+
install_requires = [
28+
'ipython',
29+
'numpy',
30+
'mpi4py'
31+
]
32+
33+
metadata = {
34+
'name': 'distarray',
35+
'version': '0.4.0',
36+
'description': 'Distributed Memory Arrays for Python',
37+
'keywords': 'parallel mpi distributed array',
38+
'license': 'New BSD',
39+
'author': 'IPython Development Team and Enthought, Inc.',
40+
'author_email': '[email protected]',
41+
'url': 'https://github.com/enthought/distarray',
42+
'packages': find_packages(),
43+
'install_requires': install_requires,
44+
'long_description': parse_readme(),
45+
'platforms': ["Linux", "Mac OS-X"],
46+
'entry_points': {'console_scripts': ['dacluster = '
47+
'distarray.apps.dacluster:main']},
48+
'classifiers': [c.strip() for c in """\
49+
Development Status :: 2 - Pre-Alpha
50+
Intended Audience :: Developers
51+
Intended Audience :: Science/Research
52+
License :: OSI Approved :: BSD License
53+
Operating System :: MacOS
54+
Operating System :: OS Independent
55+
Operating System :: POSIX
56+
Operating System :: Unix
57+
Programming Language :: Python
58+
Topic :: Scientific/Engineering
59+
Topic :: Software Development
60+
Topic :: Software Development :: Libraries
61+
""".splitlines() if len(c.strip()) > 0],
62+
}
63+
64+
setup(**metadata)

0 commit comments

Comments
 (0)