Merge branch 'develop' of https://github.com/nltk/nltk into develop

Author: stevenbird

nltk/collections.py

@@ -6,7 +6,6 @@
 # For license information, see LICENSE.TXT
 
 import bisect
-
 from functools import total_ordering
 from itertools import chain, islice
 

nltk/corpus/reader/wordnet.py

@@ -589,12 +589,13 @@ def closure(self, rel, depth=-1):
 
         >>> dog = wn.synset('dog.n.01')
         >>> hyp = lambda s:sorted(s.hypernyms())
-        >>> print(sorted(dog.closure(hyp)))
-        [Synset('animal.n.01'), Synset('canine.n.02'), Synset('carnivore.n.01'),\
-Synset('chordate.n.01'), Synset('domestic_animal.n.01'), Synset('entity.n.01'),\
-Synset('living_thing.n.01'), Synset('mammal.n.01'), Synset('object.n.01'),\
-Synset('organism.n.01'), Synset('physical_entity.n.01'), Synset('placental.n.01'),\
-Synset('vertebrate.n.01'), Synset('whole.n.02')]
+        >>> print(list(dog.closure(hyp)))
+        [Synset('canine.n.02'), Synset('domestic_animal.n.01'), Synset('carnivore.n.01'),\
+ Synset('animal.n.01'), Synset('placental.n.01'), Synset('organism.n.01'),\
+ Synset('mammal.n.01'), Synset('living_thing.n.01'), Synset('vertebrate.n.01'),\
+ Synset('whole.n.02'), Synset('chordate.n.01'), Synset('object.n.01'),\
+ Synset('physical_entity.n.01'), Synset('entity.n.01')]
+
         UserWarning: Discarded redundant search for Synset('animal.n.01') at depth 7
         """
 

nltk/data.py

@@ -52,7 +52,6 @@
     from zlib import Z_FINISH as FLUSH
 
 from nltk import grammar, sem
-from nltk.compat import add_py3_data, py3_data
 from nltk.internals import deprecated
 
 textwrap_indent = functools.partial(textwrap.indent, prefix="  ")
@@ -300,7 +299,6 @@ class FileSystemPathPointer(PathPointer, str):
     directly via a given absolute path.
     """
 
-    @py3_data
     def __init__(self, _path):
         """
         Create a new path pointer for the given absolute path.
@@ -349,7 +347,6 @@ class BufferedGzipFile(GzipFile):
     Python versions.
     """
 
-    @py3_data
     def __init__(
         self, filename=None, mode=None, compresslevel=9, fileobj=None, **kwargs
     ):
@@ -382,7 +379,6 @@ class ZipFilePathPointer(PathPointer):
     which can be accessed by reading that zipfile.
     """
 
-    @py3_data
     def __init__(self, zipfile, entry=""):
         """
         Create a new path pointer pointing at the specified entry
@@ -791,7 +787,6 @@ def load(
     :param encoding: the encoding of the input; only used for text formats.
     """
     resource_url = normalize_resource_url(resource_url)
-    resource_url = add_py3_data(resource_url)
 
     # Determine the format of the resource.
     if format == "auto":
@@ -818,7 +813,6 @@ def load(
                 print(f"<<Using cached copy of {resource_url}>>")
             return resource_val
 
-    resource_url = normalize_resource_url(resource_url)
     protocol, path_ = split_resource_url(resource_url)
 
     if path_[-7:] == ".pickle":
@@ -979,7 +973,7 @@ def _open(resource_url):
 
 
 class LazyLoader:
-    @py3_data
+
     def __init__(self, _path):
         self._path = _path
 
@@ -1020,7 +1014,6 @@ class OpenOnDemandZipFile(zipfile.ZipFile):
     read-only (i.e. ``write()`` and ``writestr()`` are disabled.
     """
 
-    @py3_data
     def __init__(self, filename):
         if not isinstance(filename, str):
             raise TypeError("ReopenableZipFile filename must be a string")
@@ -1077,7 +1070,6 @@ class SeekableUnicodeStreamReader:
 
     DEBUG = True  # : If true, then perform extra sanity checks.
 
-    @py3_data
     def __init__(self, stream, encoding, errors="strict"):
         # Rewind the stream to its beginning.
         stream.seek(0)

nltk/sem/boxer.py

@@ -530,11 +530,15 @@ def _handle_time_expression(self, arg):
         else:
             return None
         self.assertToken(self.token(), ")")
+
+        def func_gen(x):
+            return lambda sent_index, word_indices: x
+
         return [
             lambda sent_index, word_indices: BoxerPred(
                 self.discourse_id, sent_index, word_indices, arg, tok, "n", 0
             )
-        ] + [lambda sent_index, word_indices: cond for cond in conds]
+        ] + [func_gen(cond) for cond in conds]
 
     def _handle_date(self, arg):
         # []: (+), []:'XXXX', [1004]:'04', []:'XX'

nltk/test/unit/test_wordnet.py

@@ -45,7 +45,7 @@ def test_hyperhyponyms(self):
             S("brunch.n.01"),
             S("buffet.n.02"),
         ]
-        self.assertEqual(sorted(S("meal.n.1").hyponyms()[:5]), first_five_meal_hypo)
+        self.assertEqual(sorted(S("meal.n.1").hyponyms())[:5], first_five_meal_hypo)
         self.assertEqual(S("Austen.n.1").instance_hypernyms(), [S("writer.n.01")])
         first_five_composer_hypo = [
             S("ambrose.n.01"),
@@ -55,14 +55,14 @@ def test_hyperhyponyms(self):
             S("beethoven.n.01"),
         ]
         self.assertEqual(
-            S("composer.n.1").instance_hyponyms()[:5], first_five_composer_hypo
+            sorted(S("composer.n.1").instance_hyponyms())[:5], first_five_composer_hypo
         )
 
         # Test root hyper-/hyponyms
         self.assertEqual(S("person.n.01").root_hypernyms(), [S("entity.n.01")])
         self.assertEqual(S("sail.v.01").root_hypernyms(), [S("travel.v.01")])
         self.assertEqual(
-            S("fall.v.12").root_hypernyms(), [S("act.v.01"), S("fall.v.17")]
+            sorted(S("fall.v.12").root_hypernyms()), [S("act.v.01"), S("fall.v.17")]
         )
 
     def test_derivationally_related_forms(self):
@@ -84,24 +84,25 @@ def test_derivationally_related_forms(self):
     def test_meronyms_holonyms(self):
         # Test meronyms, holonyms.
         self.assertEqual(
-            S("dog.n.01").member_holonyms(), [S("canis.n.01"), S("pack.n.06")]
+            sorted(S("dog.n.01").member_holonyms()), [S("canis.n.01"), S("pack.n.06")]
         )
         self.assertEqual(S("dog.n.01").part_meronyms(), [S("flag.n.07")])
 
         self.assertEqual(S("faculty.n.2").member_meronyms(), [S("professor.n.01")])
         self.assertEqual(S("copilot.n.1").member_holonyms(), [S("crew.n.01")])
 
         self.assertEqual(
-            S("table.n.2").part_meronyms(),
+            sorted(S("table.n.2").part_meronyms()),
             [S("leg.n.03"), S("tabletop.n.01"), S("tableware.n.01")],
         )
         self.assertEqual(S("course.n.7").part_holonyms(), [S("meal.n.01")])
 
         self.assertEqual(
-            S("water.n.1").substance_meronyms(), [S("hydrogen.n.01"), S("oxygen.n.01")]
+            sorted(S("water.n.1").substance_meronyms()),
+            [S("hydrogen.n.01"), S("oxygen.n.01")],
         )
         self.assertEqual(
-            S("gin.n.1").substance_holonyms(),
+            sorted(S("gin.n.1").substance_holonyms()),
             [
                 S("gin_and_it.n.01"),
                 S("gin_and_tonic.n.01"),
@@ -123,7 +124,7 @@ def test_misc_relations(self):
         # Test misc relations.
         self.assertEqual(S("snore.v.1").entailments(), [S("sleep.v.01")])
         self.assertEqual(
-            S("heavy.a.1").similar_tos(),
+            sorted(S("heavy.a.1").similar_tos()),
             [
                 S("dense.s.03"),
                 S("doughy.s.01"),
@@ -162,10 +163,14 @@ def test_domains(self):
     def test_in_topic_domains(self):
         # Test in domains.
         self.assertEqual(
-            S("computer_science.n.01").in_topic_domains()[0], S("access.n.05")
+            sorted(S("computer_science.n.01").in_topic_domains())[0], S("access.n.05")
+        )
+        self.assertEqual(
+            sorted(S("germany.n.01").in_region_domains())[23], S("trillion.n.02")
+        )
+        self.assertEqual(
+            sorted(S("slang.n.02").in_usage_domains())[1], S("airhead.n.01")
         )
-        self.assertEqual(S("germany.n.01").in_region_domains()[23], S("trillion.n.02"))
-        self.assertEqual(S("slang.n.02").in_usage_domains()[1], S("airhead.n.01"))
 
     def test_wordnet_similarities(self):
         # Path based similarities.

nltk/util.py

@@ -257,7 +257,13 @@ def breadth_first(tree, children=iter, maxdepth=-1):
 
 
 def edge_closure(tree, children=iter, maxdepth=-1, verbose=False):
-    """Yield the edges of a graph in breadth-first order,
+    """
+    :param tree: the tree root
+    :param children: a function taking as argument a tree node
+    :param maxdepth: to limit the search depth
+    :param verbose: to print warnings when cycles are discarded
+
+    Yield the edges of a graph in breadth-first order,
     discarding eventual cycles.
     The first argument should be the start node;
     children should be a function taking as argument a graph node
@@ -295,13 +301,13 @@ def edge_closure(tree, children=iter, maxdepth=-1, verbose=False):
 def edges2dot(edges, shapes=None, attr=None):
     """
     :param edges: the set (or list) of edges of a directed graph.
-
-    :return dot_string: a representation of 'edges' as a string in the DOT
-        graph language, which can be converted to an image by the 'dot' program
-        from the Graphviz package, or nltk.parse.dependencygraph.dot2img(dot_string).
-
     :param shapes: dictionary of strings that trigger a specified shape.
     :param attr: dictionary with global graph attributes
+    :return: a representation of 'edges' as a string in the DOT graph language.
+
+    Returns dot_string: a representation of 'edges' as a string in the DOT
+    graph language, which can be converted to an image by the 'dot' program
+    from the Graphviz package, or nltk.parse.dependencygraph.dot2img(dot_string).
 
     >>> import nltk
     >>> from nltk.util import edges2dot
@@ -337,8 +343,12 @@ def edges2dot(edges, shapes=None, attr=None):
 
 def unweighted_minimum_spanning_digraph(tree, children=iter, shapes=None, attr=None):
     """
+    :param tree: the tree root
+    :param children: a function taking as argument a tree node
+    :param shapes: dictionary of strings that trigger a specified shape.
+    :param attr: dictionary with global graph attributes
 
-    Build a Minimum Spanning Tree (MST) of an unweighted graph,
+        Build a Minimum Spanning Tree (MST) of an unweighted graph,
     by traversing the nodes of a tree in breadth-first order,
     discarding eventual cycles.
 
@@ -378,7 +388,15 @@ def unweighted_minimum_spanning_digraph(tree, children=iter, shapes=None, attr=N
 
 
 def acyclic_breadth_first(tree, children=iter, maxdepth=-1, verbose=False):
-    """Traverse the nodes of a tree in breadth-first order,
+    """
+    :param tree: the tree root
+    :param children: a function taking as argument a tree node
+    :param maxdepth: to limit the search depth
+    :param verbose: to print warnings when cycles are discarded
+    :return: the tree in breadth-first order
+
+        Adapted from breadth_first() above, to discard cycles.
+    Traverse the nodes of a tree in breadth-first order,
     discarding eventual cycles.
 
     The first argument should be the tree root;
@@ -389,32 +407,41 @@ def acyclic_breadth_first(tree, children=iter, maxdepth=-1, verbose=False):
     queue = deque([(tree, 0)])
     while queue:
         node, depth = queue.popleft()
+        if node in traversed:
+            continue
         yield node
         traversed.add(node)
         if depth != maxdepth:
             try:
                 for child in children(node):
                     if child not in traversed:
                         queue.append((child, depth + 1))
-                    else:
-                        if verbose:
-                            warnings.warn(
-                                "Discarded redundant search for {} at depth {}".format(
-                                    child, depth + 1
-                                ),
-                                stacklevel=2,
-                            )
+                    elif verbose:
+                        warnings.warn(
+                            "Discarded redundant search for {} at depth {}".format(
+                                child, depth + 1
+                            ),
+                            stacklevel=2,
+                        )
             except TypeError:
                 pass
 
 
 def acyclic_depth_first(
     tree, children=iter, depth=-1, cut_mark=None, traversed=None, verbose=False
 ):
-    """Traverse the nodes of a tree in depth-first order,
+    """
+    :param tree: the tree root
+    :param children: a function taking as argument a tree node
+    :param depth: the maximum depth of the search
+    :param cut_mark: the mark to add when cycles are truncated
+    :param traversed: the set of traversed nodes
+    :param verbose: to print warnings when cycles are discarded
+    :return: the tree in depth-first order
+
+    Traverse the nodes of a tree in depth-first order,
     discarding eventual cycles within any branch,
     adding cut_mark (when specified) if cycles were truncated.
-
     The first argument should be the tree root;
     children should be a function taking as argument a tree node
     and returning an iterator of the node's children.
@@ -476,7 +503,17 @@ def acyclic_depth_first(
 def acyclic_branches_depth_first(
     tree, children=iter, depth=-1, cut_mark=None, traversed=None, verbose=False
 ):
-    """Traverse the nodes of a tree in depth-first order,
+    """
+    :param tree: the tree root
+    :param children: a function taking as argument a tree node
+    :param depth: the maximum depth of the search
+    :param cut_mark: the mark to add when cycles are truncated
+    :param traversed: the set of traversed nodes
+    :param verbose: to print warnings when cycles are discarded
+    :return: the tree in depth-first order
+
+        Adapted from acyclic_depth_first() above, to
+    traverse the nodes of a tree in depth-first order,
     discarding eventual cycles within the same branch,
     but keep duplicate paths in different branches.
     Add cut_mark (when defined) if cycles were truncated.
@@ -548,15 +585,22 @@ def acyclic_branches_depth_first(
 
 
 def acyclic_dic2tree(node, dic):
-    """Convert acyclic dictionary 'dic', where the keys are nodes, and the
+    """
+    :param node: the root node
+    :param dic: the dictionary of children
+
+    Convert acyclic dictionary 'dic', where the keys are nodes, and the
     values are lists of children, to output tree suitable for pprint(),
     starting at root 'node', with subtrees as nested lists."""
     return [node] + [acyclic_dic2tree(child, dic) for child in dic[node]]
 
 
 def unweighted_minimum_spanning_dict(tree, children=iter):
     """
-    Output a dictionary representing a Minimum Spanning Tree (MST)
+    :param tree: the tree root
+    :param children: a function taking as argument a tree node
+
+            Output a dictionary representing a Minimum Spanning Tree (MST)
     of an unweighted graph, by traversing the nodes of a tree in
     breadth-first order, discarding eventual cycles.
 
@@ -598,7 +642,10 @@ def unweighted_minimum_spanning_dict(tree, children=iter):
 
 def unweighted_minimum_spanning_tree(tree, children=iter):
     """
-    Output a Minimum Spanning Tree (MST) of an unweighted graph,
+    :param tree: the tree root
+    :param children: a function taking as argument a tree node
+
+       Output a Minimum Spanning Tree (MST) of an unweighted graph,
     by traversing the nodes of a tree in breadth-first order,
     discarding eventual cycles.