updating to networkx ver.2 (dev): removing all *_iter and adding list(*) in a few places (nodes_iter, in_edges_iter, etc., dont exist in ver 2, but nodes, in_edges, etc., are iterators); had to add .copy to .subgraph for ver 2 (otherwise its a view), but had to keep original code for ver 1

Author: djarecka

nipype/pipeline/engine/tests/test_engine.py

@@ -316,7 +316,7 @@ def test_disconnect():
     flow1 = pe.Workflow(name='test')
     flow1.connect(a, 'a', b, 'a')
     flow1.disconnect(a, 'a', b, 'a')
-    assert flow1._graph.edges() == []
+    assert list(flow1._graph.edges()) == []
 
 
 def test_doubleconnect():
@@ -637,7 +637,7 @@ def func1(in1):
     n1.inputs.in1 = [1]
     eg = w1.run()
 
-    node = eg.nodes()[0]
+    node = list(eg.nodes())[0]
     outjson = glob(os.path.join(node.output_dir(), '_0x*.json'))
     assert len(outjson) == 1
 

nipype/pipeline/engine/utils.py

@@ -24,6 +24,7 @@
 from functools import reduce
 import numpy as np
 from ...utils.misc import package_check
+from distutils.version import LooseVersion
 
 package_check('networkx', '1.3')
 
@@ -267,7 +268,7 @@ def _write_detailed_dot(graph, dotfilename):
     for n in nx.topological_sort(graph):
         nodename = str(n)
         inports = []
-        for u, v, d in graph.in_edges_iter(nbunch=n, data=True):
+        for u, v, d in graph.in_edges(nbunch=n, data=True):
             for cd in d['connect']:
                 if isinstance(cd[0], (str, bytes)):
                     outport = cd[0]
@@ -287,7 +288,7 @@ def _write_detailed_dot(graph, dotfilename):
             inputstr += '|<in%s> %s' % (replacefunk(ip), ip)
         inputstr += '}'
         outports = []
-        for u, v, d in graph.out_edges_iter(nbunch=n, data=True):
+        for u, v, d in graph.out_edges(nbunch=n, data=True):
             for cd in d['connect']:
                 if isinstance(cd[0], (str, bytes)):
                     outport = cd[0]
@@ -446,7 +447,7 @@ def get_levels(G):
     levels = {}
     for n in nx.topological_sort(G):
         levels[n] = 0
-        for pred in G.predecessors_iter(n):
+        for pred in G.predecessors(n):
             levels[n] = max(levels[n], levels[pred] + 1)
     return levels
 
@@ -491,9 +492,9 @@ def _merge_graphs(supergraph, nodes, subgraph, nodeid, iterables,
         raise Exception(("Execution graph does not have a unique set of node "
                          "names. Please rerun the workflow"))
     edgeinfo = {}
-    for n in subgraph.nodes():
+    for n in list(subgraph.nodes()):
         nidx = ids.index(n._hierarchy + n._id)
-        for edge in supergraph.in_edges_iter(supernodes[nidx]):
+        for edge in supergraph.in_edges(list(supernodes)[nidx]):
                 # make sure edge is not part of subgraph
             if edge[0] not in subgraph.nodes():
                 if n._hierarchy + n._id not in list(edgeinfo.keys()):
@@ -514,7 +515,7 @@ def _merge_graphs(supergraph, nodes, subgraph, nodeid, iterables,
         Gc = deepcopy(subgraph)
         ids = [n._hierarchy + n._id for n in Gc.nodes()]
         nodeidx = ids.index(nodeid)
-        rootnode = Gc.nodes()[nodeidx]
+        rootnode = list(Gc.nodes())[nodeidx]
         paramstr = ''
         for key, val in sorted(params.items()):
             paramstr = '{}_{}_{}'.format(
@@ -613,10 +614,10 @@ def _node_ports(graph, node):
     """
     portinputs = {}
     portoutputs = {}
-    for u, _, d in graph.in_edges_iter(node, data=True):
+    for u, _, d in graph.in_edges(node, data=True):
         for src, dest in d['connect']:
             portinputs[dest] = (u, src)
-    for _, v, d in graph.out_edges_iter(node, data=True):
+    for _, v, d in graph.out_edges(node, data=True):
         for src, dest in d['connect']:
             if isinstance(src, tuple):
                 srcport = src[0]
@@ -682,7 +683,7 @@ def generate_expanded_graph(graph_in):
     logger.debug("PE: expanding iterables")
     graph_in = _remove_nonjoin_identity_nodes(graph_in, keep_iterables=True)
     # standardize the iterables as {(field, function)} dictionaries
-    for node in graph_in.nodes_iter():
+    for node in graph_in.nodes():
         if node.iterables:
             _standardize_iterables(node)
     allprefixes = list('abcdefghijklmnopqrstuvwxyz')
@@ -697,7 +698,7 @@ def generate_expanded_graph(graph_in):
         logger.debug("Expanding the iterable node %s..." % inode)
 
         # the join successor nodes of the current iterable node
-        jnodes = [node for node in graph_in.nodes_iter()
+        jnodes = [node for node in graph_in.nodes()
                   if hasattr(node, 'joinsource') and
                   inode.name == node.joinsource and
                   nx.has_path(graph_in, inode, node)]
@@ -709,7 +710,7 @@ def generate_expanded_graph(graph_in):
         for jnode in jnodes:
             in_edges = jedge_dict[jnode] = {}
             edges2remove = []
-            for src, dest, data in graph_in.in_edges_iter(jnode, True):
+            for src, dest, data in graph_in.in_edges(jnode, True):
                 in_edges[src.itername] = data
                 edges2remove.append((src, dest))
 
@@ -726,7 +727,7 @@ def generate_expanded_graph(graph_in):
                 src_fields = [src_fields]
             # find the unique iterable source node in the graph
             try:
-                iter_src = next((node for node in graph_in.nodes_iter()
+                iter_src = next((node for node in graph_in.nodes()
                                  if node.name == src_name and
                                  nx.has_path(graph_in, node, inode)))
             except StopIteration:
@@ -781,7 +782,11 @@ def make_field_func(*pair):
         inode._id += ('.' + iterable_prefix + 'I')
 
         # merge the iterated subgraphs
-        subgraph = graph_in.subgraph(subnodes)
+        # dj: the behaviour of .copy changes in version 2
+        if LooseVersion(nx.__version__) < LooseVersion('2'):
+            subgraph = graph_in.subgraph(subnodes)
+        else:
+            subgraph = graph_in.subgraph(subnodes).copy()
         graph_in = _merge_graphs(graph_in, subnodes,
                                  subgraph, inode._hierarchy + inode._id,
                                  iterables, iterable_prefix, inode.synchronize)
@@ -793,7 +798,7 @@ def make_field_func(*pair):
             old_edge_dict = jedge_dict[jnode]
             # the edge source node replicates
             expansions = defaultdict(list)
-            for node in graph_in.nodes_iter():
+            for node in graph_in.nodes():
                 for src_id, edge_data in list(old_edge_dict.items()):
                     if node.itername.startswith(src_id):
                         expansions[src_id].append(node)
@@ -1283,7 +1288,7 @@ def write_workflow_prov(graph, filename=None, format='all'):
 
     # add dependencies (edges)
     # Process->Process
-    for idx, edgeinfo in enumerate(graph.in_edges_iter()):
+    for idx, edgeinfo in enumerate(graph.in_edges()):
         ps.g.wasStartedBy(processes[nodes.index(edgeinfo[1])],
                           starter=processes[nodes.index(edgeinfo[0])])
 
@@ -1295,7 +1300,7 @@ def write_workflow_prov(graph, filename=None, format='all'):
 def topological_sort(graph, depth_first=False):
     """Returns a depth first sorted order if depth_first is True
     """
-    nodesort = nx.topological_sort(graph)
+    nodesort = list(nx.topological_sort(graph))
     if not depth_first:
         return nodesort, None
     logger.debug("Performing depth first search")

nipype/pipeline/engine/workflows.py

@@ -185,7 +185,7 @@ def connect(self, *args, **kwargs):
             # check to see which ports of destnode are already
             # connected.
             if not disconnect and (destnode in self._graph.nodes()):
-                for edge in self._graph.in_edges_iter(destnode):
+                for edge in self._graph.in_edges(destnode):
                     data = self._graph.get_edge_data(*edge)
                     for sourceinfo, destname in data['connect']:
                         if destname not in connected_ports[destnode]:
@@ -506,8 +506,8 @@ def export(self, filename=None, prefix="output", format="python",
                     else:
                         lines.append(line)
                 # write connections
-                for u, _, d in flatgraph.in_edges_iter(nbunch=node,
-                                                       data=True):
+                for u, _, d in flatgraph.in_edges(nbunch=node,
+                                                  data=True):
                     for cd in d['connect']:
                         if isinstance(cd[0], tuple):
                             args = list(cd[0])
@@ -633,7 +633,7 @@ def _write_report_info(self, workingdir, name, graph):
                                             total=N,
                                             name='Group_%05d' % gid))
         json_dict['maxN'] = maxN
-        for u, v in graph.in_edges_iter():
+        for u, v in graph.in_edges():
             json_dict['links'].append(dict(source=nodes.index(u),
                                            target=nodes.index(v),
                                            value=1))
@@ -654,7 +654,7 @@ def getname(u, i):
         json_dict = []
         for i, node in enumerate(nodes):
             imports = []
-            for u, v in graph.in_edges_iter(nbunch=node):
+            for u, v in graph.in_edges(nbunch=node):
                 imports.append(getname(u, nodes.index(u)))
             json_dict.append(dict(name=getname(node, i),
                                   size=1,
@@ -669,7 +669,7 @@ def _set_needed_outputs(self, graph):
             return
         for node in graph.nodes():
             node.needed_outputs = []
-            for edge in graph.out_edges_iter(node):
+            for edge in graph.out_edges(node):
                 data = graph.get_edge_data(*edge)
                 sourceinfo = [v1[0] if isinstance(v1, tuple) else v1
                               for v1, v2 in data['connect']]
@@ -683,7 +683,7 @@ def _configure_exec_nodes(self, graph):
         """
         for node in graph.nodes():
             node.input_source = {}
-            for edge in graph.in_edges_iter(node):
+            for edge in graph.in_edges(node):
                 data = graph.get_edge_data(*edge)
                 for sourceinfo, field in data['connect']:
                     node.input_source[field] = \
@@ -758,8 +758,8 @@ def _get_inputs(self):
                 setattr(inputdict, node.name, node.inputs)
             else:
                 taken_inputs = []
-                for _, _, d in self._graph.in_edges_iter(nbunch=node,
-                                                         data=True):
+                for _, _, d in self._graph.in_edges(nbunch=node,
+                                                    data=True):
                     for cd in d['connect']:
                         taken_inputs.append(cd[1])
                 unconnectedinputs = TraitedSpec()
@@ -864,7 +864,8 @@ def _generate_flatgraph(self):
                 # use in_edges instead of in_edges_iter to allow
                 # disconnections to take place properly. otherwise, the
                 # edge dict is modified.
-                for u, _, d in self._graph.in_edges(nbunch=node, data=True):
+                # dj: added list() for networkx ver.2
+                for u, _, d in list(self._graph.in_edges(nbunch=node, data=True)):
                     logger.debug('in: connections-> %s', to_str(d['connect']))
                     for cd in deepcopy(d['connect']):
                         logger.debug("in: %s", to_str(cd))
@@ -877,7 +878,8 @@ def _generate_flatgraph(self):
                         self.disconnect(u, cd[0], node, cd[1])
                         self.connect(srcnode, srcout, dstnode, dstin)
                 # do not use out_edges_iter for reasons stated in in_edges
-                for _, v, d in self._graph.out_edges(nbunch=node, data=True):
+                # dj: for ver 2 use list(out_edges)
+                for _, v, d in list(self._graph.out_edges(nbunch=node, data=True)):
                     logger.debug('out: connections-> %s', to_str(d['connect']))
                     for cd in deepcopy(d['connect']):
                         logger.debug("out: %s", to_str(cd))
@@ -965,7 +967,7 @@ def _get_dot(self, prefix=None, hierarchy=None, colored=False,
                                              simple_form=simple_form, level=level + 3))
                 dotlist.append('}')
             else:
-                for subnode in self._graph.successors_iter(node):
+                for subnode in self._graph.successors(node):
                     if node._hierarchy != subnode._hierarchy:
                         continue
                     if not isinstance(subnode, Workflow):
@@ -980,7 +982,7 @@ def _get_dot(self, prefix=None, hierarchy=None, colored=False,
                                                           subnodename))
                         logger.debug('connection: %s', dotlist[-1])
         # add between workflow connections
-        for u, v, d in self._graph.edges_iter(data=True):
+        for u, v, d in self._graph.edges(data=True):
             uname = '.'.join(hierarchy + [u.fullname])
             vname = '.'.join(hierarchy + [v.fullname])
             for src, dest in d['connect']: