Skip to content

Commit 37da0fd

Browse files
minrkminrk
authored
Merge pull request #494 from minrk/serialize-cluster
Serialize cluster objects
2 parents da3b007 + cd36329 commit 37da0fd

File tree

15 files changed

+1278
-742
lines changed

15 files changed

+1278
-742
lines changed

‎examples/Cluster API.ipynb

Lines changed: 215 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,25 +32,25 @@
3232
},
3333
{
3434
"cell_type": "code",
35-
"execution_count": 1,
35+
"execution_count": 5,
3636
"id": "ef0dff26-f6d2-4d79-901b-049debe0c0d1",
3737
"metadata": {},
3838
"outputs": [
3939
{
4040
"data": {
4141
"text/plain": [
42-
"<Cluster(cluster_id='touchy-1623757384-cpbt', profile='default')>"
42+
"<Cluster(cluster_id='touchy-1624880089-y1md', profile='default')>"
4343
]
4444
},
45-
"execution_count": 1,
45+
"execution_count": 5,
4646
"metadata": {},
4747
"output_type": "execute_result"
4848
}
4949
],
5050
"source": [
51-
"from ipyparallel import Cluster\n",
51+
"import ipyparallel as ipp\n",
5252
"\n",
53-
"cluster = Cluster()\n",
53+
"cluster = ipp.Cluster()\n",
5454
"cluster"
5555
]
5656
},
@@ -706,6 +706,216 @@
706706
"source": [
707707
"await cluster.stop_cluster()"
708708
]
709+
},
710+
{
711+
"cell_type": "markdown",
712+
"id": "4607e5e6-e585-413f-85aa-281e606f7f82",
713+
"metadata": {},
714+
"source": [
715+
"## Connecting to existing clusters\n",
716+
"\n",
717+
"a Cluster object writes its state to disk,\n",
718+
"in a file accessible as `cluster.cluster_file`.\n",
719+
"By default, this willb e `$PROFILE_DIR/security/ipcluster-$cluster-id.json`.\n",
720+
"\n",
721+
"Cluster objects can load state from a dictionary with `Cluster.from_dict(d)`\n",
722+
"or from a JSON file containing that information with `Cluster.from_file()`.\n",
723+
"\n",
724+
"The default arguments for `from_file` are to use the current IPython profile (default: 'default')\n",
725+
"and empty cluster id,\n",
726+
"so if you start a cluster with `ipcluster start`, you can connect to it immediately with\n",
727+
"\n",
728+
"```python\n",
729+
"cluster = ipp.Cluster.from_file()\n",
730+
"```"
731+
]
732+
},
733+
{
734+
"cell_type": "code",
735+
"execution_count": 2,
736+
"id": "0da44ee0-2a3f-4b98-b31a-9c71621b9777",
737+
"metadata": {},
738+
"outputs": [],
739+
"source": [
740+
"import ipyparallel as ipp\n",
741+
"cluster = ipp.Cluster.from_file()"
742+
]
743+
},
744+
{
745+
"cell_type": "code",
746+
"execution_count": 3,
747+
"id": "0df1b2b2-0d35-45a7-8ad8-c7f55eb52a5f",
748+
"metadata": {},
749+
"outputs": [
750+
{
751+
"data": {
752+
"text/plain": [
753+
"<Cluster(cluster_id='', profile='default', controller=<running>, engine_sets=['1624884556-z9qr'])>"
754+
]
755+
},
756+
"execution_count": 3,
757+
"metadata": {},
758+
"output_type": "execute_result"
759+
}
760+
],
761+
"source": [
762+
"cluster"
763+
]
764+
},
765+
{
766+
"cell_type": "markdown",
767+
"id": "947edd74-5b1c-42ca-a83a-1755a96c6469",
768+
"metadata": {},
769+
"source": [
770+
"`ipp.ClusterManager` provides an API for collecting/discovering/loading all the clusters on your system.\n",
771+
"\n",
772+
"By default, it finds loads clusters in all your IPython profiles,\n",
773+
"but can be confined to one profile or use explicit profile directories."
774+
]
775+
},
776+
{
777+
"cell_type": "code",
778+
"execution_count": 6,
779+
"id": "c8b30824-b833-449c-b1eb-71b56ff47ba6",
780+
"metadata": {},
781+
"outputs": [],
782+
"source": [
783+
"clusters = ipp.ClusterManager().load_clusters()"
784+
]
785+
},
786+
{
787+
"cell_type": "code",
788+
"execution_count": 8,
789+
"id": "6c5e2fe0-c45c-4528-8732-d6bd56580885",
790+
"metadata": {},
791+
"outputs": [
792+
{
793+
"data": {
794+
"text/plain": [
795+
"{'mpi:abc-123': <Cluster(cluster_id='abc-123', profile='mpi', controller=<running>, engine_sets=['1624884663-euj7'])>,\n",
796+
" 'default:': <Cluster(cluster_id='', profile='default', controller=<running>, engine_sets=['1624884556-z9qr'])>}"
797+
]
798+
},
799+
"execution_count": 8,
800+
"metadata": {},
801+
"output_type": "execute_result"
802+
}
803+
],
804+
"source": [
805+
"clusters"
806+
]
807+
},
808+
{
809+
"cell_type": "markdown",
810+
"id": "b75d78c5-5f49-436f-b313-c56085f70e43",
811+
"metadata": {},
812+
"source": [
813+
"This is the class that powers the new `ipcluster list`"
814+
]
815+
},
816+
{
817+
"cell_type": "code",
818+
"execution_count": 10,
819+
"id": "cbf3d3f3-70e1-4c92-b6d5-bf17ea913253",
820+
"metadata": {},
821+
"outputs": [
822+
{
823+
"name": "stdout",
824+
"output_type": "stream",
825+
"text": [
826+
"PROFILE CLUSTER ID RUNNING ENGINES LAUNCHER\n",
827+
"default '' True 4 Local\n",
828+
"mpi abc-123 True 4 MPI\n"
829+
]
830+
}
831+
],
832+
"source": [
833+
"!ipcluster list"
834+
]
835+
},
836+
{
837+
"cell_type": "code",
838+
"execution_count": 11,
839+
"id": "08a6c241-8d05-45fa-bfe1-ba89f4257ade",
840+
"metadata": {},
841+
"outputs": [
842+
{
843+
"name": "stdout",
844+
"output_type": "stream",
845+
"text": [
846+
"2021-06-28 14:53:00.591 [IPClusterStop] Stopping engine(s): 1624884663-euj7\n",
847+
"2021-06-28 14:53:00.592 [IPClusterStop] Stopping controller\n"
848+
]
849+
}
850+
],
851+
"source": [
852+
"!ipcluster stop --profile mpi --cluster-id abc-123"
853+
]
854+
},
855+
{
856+
"cell_type": "code",
857+
"execution_count": 12,
858+
"id": "5ec2c3c2-153c-4fbf-b52d-567530533a3c",
859+
"metadata": {},
860+
"outputs": [
861+
{
862+
"name": "stdout",
863+
"output_type": "stream",
864+
"text": [
865+
"PROFILE CLUSTER ID RUNNING ENGINES LAUNCHER\n",
866+
"default '' True 4 Local\n"
867+
]
868+
}
869+
],
870+
"source": [
871+
"!ipcluster list"
872+
]
873+
},
874+
{
875+
"cell_type": "markdown",
876+
"id": "3b3e53cf-4b30-4496-8b84-993daf4fc527",
877+
"metadata": {},
878+
"source": [
879+
"The same operation can be done from the Python API:"
880+
]
881+
},
882+
{
883+
"cell_type": "code",
884+
"execution_count": 13,
885+
"id": "7be40506-7108-4fbc-8b8f-16e4d478ed84",
886+
"metadata": {},
887+
"outputs": [
888+
{
889+
"name": "stdout",
890+
"output_type": "stream",
891+
"text": [
892+
"Stopping engine(s): 1624884556-z9qr\n",
893+
"Stopping controller\n"
894+
]
895+
}
896+
],
897+
"source": [
898+
"cluster = ipp.Cluster.from_file(profile=\"default\", cluster_id=\"\")\n",
899+
"await cluster.stop_cluster()"
900+
]
901+
},
902+
{
903+
"cell_type": "code",
904+
"execution_count": 14,
905+
"id": "572b2e45-4b20-4f8d-81b0-9334a42e69e7",
906+
"metadata": {},
907+
"outputs": [
908+
{
909+
"name": "stdout",
910+
"output_type": "stream",
911+
"text": [
912+
"PROFILE CLUSTER ID RUNNING ENGINES LAUNCHER\n"
913+
]
914+
}
915+
],
916+
"source": [
917+
"!ipcluster list"
918+
]
709919
}
710920
],
711921
"metadata": {

‎ipyparallel/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from .client.remotefunction import *
1616
from .client.view import *
1717
from .cluster import Cluster
18+
from .cluster import ClusterManager
1819
from .controller.dependency import *
1920
from .error import *
2021
from .serialize import *

‎ipyparallel/apps/baseapp.py

Lines changed: 0 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -219,65 +219,3 @@ def reinit_logging(self):
219219
# ipcluster app will sometimes print duplicate messages during shutdown
220220
# if this is 1 (default):
221221
self.log.propagate = False
222-
223-
def write_pid_file(self, overwrite=False):
224-
"""Create a .pid file in the pid_dir with my pid.
225-
226-
This must be called after pre_construct, which sets `self.pid_dir`.
227-
This raises :exc:`PIDFileError` if the pid file exists already.
228-
"""
229-
pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
230-
if os.path.isfile(pid_file):
231-
pid = self.get_pid_from_file()
232-
if not overwrite:
233-
raise PIDFileError(
234-
'The pid file [%s] already exists. \nThis could mean that this '
235-
'server is already running with [pid=%s].' % (pid_file, pid)
236-
)
237-
with open(pid_file, 'w') as f:
238-
self.log.info("Creating pid file: %s" % pid_file)
239-
f.write(repr(os.getpid()) + '\n')
240-
241-
def remove_pid_file(self):
242-
"""Remove the pid file.
243-
244-
This should be called at shutdown by registering a callback with
245-
:func:`reactor.addSystemEventTrigger`. This needs to return
246-
``None``.
247-
"""
248-
pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
249-
if os.path.isfile(pid_file):
250-
try:
251-
self.log.info("Removing pid file: %s" % pid_file)
252-
os.remove(pid_file)
253-
except:
254-
self.log.warn("Error removing the pid file: %s" % pid_file)
255-
256-
def get_pid_from_file(self):
257-
"""Get the pid from the pid file.
258-
259-
If the pid file doesn't exist a :exc:`PIDFileError` is raised.
260-
"""
261-
pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
262-
if os.path.isfile(pid_file):
263-
with open(pid_file, 'r') as f:
264-
s = f.read().strip()
265-
try:
266-
pid = int(s)
267-
except:
268-
raise PIDFileError(
269-
"invalid pid file: %s (contents: %r)" % (pid_file, s)
270-
)
271-
return pid
272-
else:
273-
raise PIDFileError('pid file not found: %s' % pid_file)
274-
275-
def check_pid(self, pid):
276-
try:
277-
return check_pid(pid)
278-
except Exception:
279-
self.log.warn(
280-
"Could not determine whether pid %i is running. "
281-
" Making the likely assumption that it is." % pid
282-
)
283-
return True

‎ipyparallel/apps/daemonize.py

Lines changed: 0 additions & 25 deletions
This file was deleted.

‎ipyparallel/client/asyncresult.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
import zmq
1717
from decorator import decorator
1818
from IPython import get_ipython
19-
from IPython.core.display import display
20-
from IPython.core.display import display_pretty
19+
from IPython.display import display
20+
from IPython.display import display_pretty
2121
from ipython_genutils.py3compat import string_types
2222

2323
from .futures import MessageFuture

0 commit comments

Comments
 (0)