Fix wordnet's all_synsets() function (nltk#3078)

* Fix all_synsets() function

* Add simple regression tests for nltk#3077

* Add suggestions by @tomaarsen

Co-authored-by: Tom Aarsen <[email protected]>

Author: ekaf

nltk/corpus/reader/wordnet.py

@@ -108,6 +108,11 @@
     "Somebody %s VERB-ing",
     "It %s that CLAUSE",
     "Something %s INFINITIVE",
+    # OEWN additions:
+    "Somebody %s at something",
+    "Somebody %s for something",
+    "Somebody %s on somebody",
+    "Somebody %s out of somebody",
 )
 
 SENSENUM_RE = re.compile(r"\.[\d]+\.")
@@ -454,7 +459,7 @@ def _doc(self, doc_type, default, lang="eng"):
             corpus._load_lang_data(lang)
             of = corpus.ss2of(self)
             i = corpus.lg_attrs.index(doc_type)
-            if of in corpus._lang_data[lang][i].keys():
+            if of in corpus._lang_data[lang][i]:
                 return corpus._lang_data[lang][i][of]
             else:
                 return None
@@ -481,11 +486,11 @@ def lemma_names(self, lang="eng"):
         if lang == "eng":
             return self._lemma_names
         else:
-            self._wordnet_corpus_reader._load_lang_data(lang)
-
-            i = self._wordnet_corpus_reader.ss2of(self, lang)
-            if i in self._wordnet_corpus_reader._lang_data[lang][0]:
-                return self._wordnet_corpus_reader._lang_data[lang][0][i]
+            reader = self._wordnet_corpus_reader
+            reader._load_lang_data(lang)
+            i = reader.ss2of(self)
+            if i in reader._lang_data[lang][0]:
+                return reader._lang_data[lang][0][i]
             else:
                 return []
 
@@ -1203,59 +1208,77 @@ def __init__(self, root, omw_reader):
         # load the exception file data into memory
         self._load_exception_map()
 
+        self.nomap = []
+        self.splits = {}
+
         # map from WordNet 3.0 for OMW data
         self.map30 = self.map_wn30()
 
         # Language data attributes
         self.lg_attrs = ["lemma", "none", "def", "exe"]
 
-    def corpus2sk(self, corpus=None):
+    def index_sense(self, version=None):
         """Read sense key to synset id mapping from index.sense file in corpus directory"""
         fn = "index.sense"
-        if corpus:
+        if version:
             from nltk.corpus import CorpusReader, LazyCorpusLoader
 
-            ixreader = LazyCorpusLoader(corpus, CorpusReader, r".*/" + fn)
+            ixreader = LazyCorpusLoader(version, CorpusReader, r".*/" + fn)
         else:
             ixreader = self
         with ixreader.open(fn) as fp:
-            sk_map = {}
+            sensekey_map = {}
             for line in fp:
-                items = line.strip().split(" ")
-                sk = items[0]
-                pos = self._pos_names[int(sk.split("%")[1].split(":")[0])]
-                sk_map[sk] = f"{items[1]}-{pos}"
-        return sk_map
+                fields = line.strip().split()
+                sensekey = fields[0]
+                pos = self._pos_names[int(sensekey.split("%")[1].split(":")[0])]
+                sensekey_map[sensekey] = f"{fields[1]}-{pos}"
+        return sensekey_map
+
+    def map_to_many(self):
+        sensekey_map1 = self.index_sense("wordnet")
+        sensekey_map2 = self.index_sense()
+        synset_to_many = {}
+        for synsetid in set(sensekey_map1.values()):
+            synset_to_many[synsetid] = []
+        for sensekey in set(sensekey_map1.keys()).intersection(
+            set(sensekey_map2.keys())
+        ):
+            source = sensekey_map1[sensekey]
+            target = sensekey_map2[sensekey]
+            synset_to_many[source].append(target)
+        return synset_to_many
+
+    def map_to_one(self):
+        synset_to_many = self.map_to_many()
+        synset_to_one = {}
+        for source in synset_to_many:
+            candidates_bag = synset_to_many[source]
+            if candidates_bag:
+                candidates_set = set(candidates_bag)
+                if len(candidates_set) == 1:
+                    target = candidates_bag[0]
+                else:
+                    counts = []
+                    for candidate in candidates_set:
+                        counts.append((candidates_bag.count(candidate), candidate))
+                    self.splits[source] = counts
+                    target = max(counts)[1]
+                synset_to_one[source] = target
+                if source[-1] == "s":
+                    # Add a mapping from "a" to target for applications like omw,
+                    # where only Lithuanian and Slovak use the "s" ss_type.
+                    synset_to_one[f"{source[:-1]}a"] = target
+            else:
+                self.nomap.append(source)
+        return synset_to_one
 
     def map_wn30(self):
         """Mapping from Wordnet 3.0 to currently loaded Wordnet version"""
         if self.get_version() == "3.0":
             return None
-        # warnings.warn(f"Mapping WN v. 3.0 to Wordnet v. {self.version}")
-        sk1 = self.corpus2sk("wordnet")
-        sk2 = self.corpus2sk()
-
-        skmap = {}
-        for sk in set(sk1.keys()).intersection(set(sk2.keys())):
-            of1 = sk1[sk]
-            of2 = sk2[sk]
-            if of1 not in skmap.keys():
-                skmap[of1] = [of2]
-            else:
-                skmap[of1].append(of2)
-
-        map30 = {}
-        for of in skmap.keys():
-            candidates = skmap[of]
-            # map to candidate that covers most lemmas:
-            of2 = max((candidates.count(x), x) for x in set(candidates))[1]
-            # warnings.warn(f"Map {of} {of2}")
-            map30[of] = of2
-            if of[-1] == "s":
-                # Add a mapping from "a" to "a" for applications like omw,
-                # which don't use the "s" ss_type:
-                map30[f"{of[:-1]}a"] = f"{of2[:-1]}a"
-        return map30
+        else:
+            return self.map_to_one()
 
     # Open Multilingual WordNet functions, contributed by
     # Nasruddin A’aidil Shari, Sim Wei Ying Geraldine, and Soe Lynn
@@ -1264,19 +1287,16 @@ def of2ss(self, of):
         """take an id and return the synsets"""
         return self.synset_from_pos_and_offset(of[-1], int(of[:8]))
 
-    def ss2of(self, ss, lang=None):
+    def ss2of(self, ss):
         """return the ID of the synset"""
-        pos = ss.pos()
-        # Only these 3 WordNets retain the satellite pos tag
-        if lang not in ["nld", "lit", "slk"] and pos == "s":
-            pos = "a"
-        return f"{ss.offset():08d}-{pos}"
+        if ss:
+            return f"{ss.offset():08d}-{ss.pos()}"
 
     def _load_lang_data(self, lang):
         """load the wordnet data of the requested language from the file to
         the cache, _lang_data"""
 
-        if lang in self._lang_data.keys():
+        if lang in self._lang_data:
             return
 
         if self._omw_reader and not self.omw_langs:
@@ -1308,7 +1328,7 @@ def add_provs(self, reader):
             file_name, file_extension = os.path.splitext(langfile)
             if file_extension == ".tab":
                 lang = file_name.split("-")[-1]
-                if lang in self.provenances.keys() or prov in ["cldr", "wikt"]:
+                if lang in self.provenances or prov in ["cldr", "wikt"]:
                     # We already have another resource for this lang,
                     # so we need to further specify the lang id:
                     lang = f"{lang}_{prov}"
@@ -1540,7 +1560,7 @@ def synset_from_pos_and_offset(self, pos, offset):
             assert synset._offset == offset
             self._synset_offset_cache[pos][offset] = synset
         else:
-            synset = Synset(self)
+            synset = None
             warnings.warn(f"No WordNet synset found for pos={pos} at offset={offset}.")
         data_file.seek(0)
         return synset
@@ -1807,16 +1827,15 @@ def all_omw_synsets(self, pos=None, lang=None):
         if lang not in self.langs():
             return None
         self._load_lang_data(lang)
-        for of in self._lang_data[lang][0].keys():
-            try:
+        for of in self._lang_data[lang][0]:
+            if not pos or of[-1] == pos:
                 ss = self.of2ss(of)
-                yield ss
-            except:
-                # A few OMW offsets don't exist in Wordnet 3.0.
-                # Additionally, when mapped to later Wordnets,
-                # increasing numbers of synsets are lost in the mapping.
-                #    warnings.warn(f"Language {lang}: no synset found for {of}")
-                pass
+                if ss:
+                    yield ss
+
+    #            else:
+    # A few OMW offsets don't exist in Wordnet 3.0.
+    #                warnings.warn(f"Language {lang}: no synset found for {of}")
 
     def all_synsets(self, pos=None, lang="eng"):
         """Iterate over all synsets with a given part of speech tag.
@@ -1840,12 +1859,14 @@ def all_eng_synsets(self, pos=None):
         # generate all synsets for each part of speech
         for pos_tag in pos_tags:
             # Open the file for reading.  Note that we can not re-use
-            # the file poitners from self._data_file_map here, because
+            # the file pointers from self._data_file_map here, because
             # we're defining an iterator, and those file pointers might
             # be moved while we're not looking.
             if pos_tag == ADJ_SAT:
-                pos_tag = ADJ
-            fileid = "data.%s" % self._FILEMAP[pos_tag]
+                pos_file = ADJ
+            else:
+                pos_file = pos_tag
+            fileid = "data.%s" % self._FILEMAP[pos_file]
             data_file = self.open(fileid)
 
             try:
@@ -1865,12 +1886,11 @@ def all_eng_synsets(self, pos=None):
                         # adjective satellites are in the same file as
                         # adjectives so only yield the synset if it's actually
                         # a satellite
-                        if synset._pos == ADJ_SAT:
+                        if pos_tag == ADJ_SAT and synset._pos == ADJ_SAT:
                             yield synset
-
                         # for all other POS tags, yield all synsets (this means
                         # that adjectives also include adjective satellites)
-                        else:
+                        elif pos_tag != ADJ_SAT:
                             yield synset
                     offset = data_file.tell()
                     line = data_file.readline()
@@ -2187,13 +2207,24 @@ def custom_lemmas(self, tab_file, lang):
                 offset_pos, label = triple[:2]
                 val = triple[-1]
                 if self.map30:
-                    if offset_pos in self.map30.keys():
+                    if offset_pos in self.map30:
                         # Map offset_pos to current Wordnet version:
                         offset_pos = self.map30[offset_pos]
                     else:
-                        # Synsets with no mapping keep their Wordnet 3.0 offset
-                        # warnings.warn(f"No map for {offset_pos}, {lang}: {lemma}")
-                        pass
+                        # Some OMW offsets were never in Wordnet:
+                        if (
+                            offset_pos not in self.nomap
+                            and offset_pos.replace("a", "s") not in self.nomap
+                        ):
+                            warnings.warn(
+                                f"{lang}: invalid offset {offset_pos} in '{line}'"
+                            )
+                        continue
+                elif offset_pos[-1] == "a":
+                    wnss = self.of2ss(offset_pos)
+                    if wnss and wnss.pos() == "s":  # Wordnet pos is "s"
+                        # Label OMW adjective satellites back to their Wordnet pos ("s")
+                        offset_pos = self.ss2of(wnss)
                 pair = label.split(":")
                 attr = pair[-1]
                 if len(pair) == 1 or pair[0] == lg:

nltk/test/wordnet.doctest

@@ -638,6 +638,24 @@ Issue 2721: WordNetCorpusReader.ic() does not add smoothing to N
     >>> information_content(word, fake_ic) > 0
     True
 
+Issue 3077: Incorrect part-of-speech filtering in all_synsets
+
+    >>> next(wn.all_synsets(pos="a"))
+    Synset('able.a.01')
+    >>> next(wn.all_synsets(pos="s"))
+    Synset('emergent.s.02')
+    >>> wn.add_omw()
+    >>> next(wn.all_synsets(lang="hrv"))
+    Synset('able.a.01')
+    >>> next(wn.all_synsets(lang="hrv", pos="n"))
+    Synset('entity.n.01')
+    >>> next(wn.all_synsets(lang="hrv", pos="v"))
+    Synset('breathe.v.01')
+    >>> next(wn.all_synsets(lang="hrv", pos="s"))
+    Synset('ideological.s.02')
+    >>> next(wn.all_synsets(lang="hrv", pos="a"))
+    Synset('able.a.01')
+
 
 ------------------------------------------------
 Endlessness vs. intractability in relation trees