Merge pull request #333 from NervanaSystems/alon/master_local

Author: peteriz

doc/source/cross_doc_coref.rst

@@ -63,11 +63,6 @@ Requirements
         {
             "topic_id": "2_ecb", #Required (a topic is a set of multiple documents that share the same subject)
             "doc_id": "1_10.xml", #Required (the article or document id this mention belong to)
-            "mention_head": "Josh", #Optional
-            "mention_head_lemma": "josh", #Optional
-            "mention_head_pos": "NOUN", #Optional (part of speech)
-            "mention_ner": null, #Optional (named entity recognition)
-            "mention_type": "HUM", #Optional (for debugging)
             "sent_id": 0, #Optional (mention sentence number in document)
             "tokens_number": [ #Optional (the token number in sentence, will be required when using Within doc entities)
                 13
@@ -77,11 +72,6 @@ Requirements
         {
             "topic_id": "2_ecb", #Required
             "doc_id": "1_11.xml",
-            "mention_head": "Reid",
-            "mention_head_lemma": "reid",
-            "mention_head_pos": "PROPN",
-            "mention_ner": "PERSON",
-            "mention_type": "HUM",
             "sent_id": 0,
             "tokens_number": [
                 3
@@ -165,8 +155,10 @@ Code Example
 
     entity_config.gold_mentions_file = '<Replace with your entity mentions json file>'
 
-    # Set the location of your resources,
-    # Use the default values if can
+    # CDCResources hold default attribute values that might need to be change,
+    # (using the defaults values in this example), use to configure attributes
+    # such as resources files location, output directory, resources init methods and other.
+    # check in class and see if any attributes require change in your set-up
     resource_location = CDCResources()
 
     # create a new cross doc resources, with all needed semantic relation models

doc/source/identifying_semantic_relation.rst

@@ -253,9 +253,8 @@ Each script requires a **mentions** file in JSON format as seen below. This file
 
     [
         { # Mention 1
-            "mention_head": "rep", #Optional
-            "mention_head_lemma": "rep", #Optional
-            "tokens_str": "rep" #Required
+            "tokens_str": "Intel" #Required,
+            "context": "Intel is the world's second largest and second highest valued semiconductor chip maker" #Optional (used in Elmo)
         },
         { # Mention 2
             "tokens_str": "Tara Reid"

examples/cross_doc_coref/cross_doc_coref_sieves.py

@@ -50,6 +50,10 @@ def run_example():
     entity_config.gold_mentions_file = LIBRARY_ROOT + \
         '/datasets/ecb/ecb_all_entity_mentions.json'
 
+    # CDCResources hold default attribute values that might need to be change,
+    # (using the defaults values in this example), use to configure attributes
+    # such as resources files location, output directory, resources init methods and other.
+    # check in class and see if any attributes require change in your set-up
     resource_location = CDCResources()
     resources = CDCSettings(resource_location, event_config, entity_config)
 

nlp_architect/data/cdc_resources/relations/referent_dict_relation_extraction.py

@@ -15,6 +15,7 @@
 # ******************************************************************************
 
 import logging
+import os
 from typing import Dict, List, Set
 
 from nlp_architect.common.cdc.mention_data import MentionDataLight
@@ -38,11 +39,15 @@ def __init__(self, method: OnlineOROfflineMethod, ref_dict: str = None):
             ref_dict (required): str Location of referent dictionary file to work with
         """
         logger.info('Loading ReferentDict module')
-        if method == OnlineOROfflineMethod.OFFLINE:
-            self.ref_dict = load_json_file(ref_dict)
-        elif method == OnlineOROfflineMethod.ONLINE:
-            self.ref_dict = self.load_reference_dict(ref_dict)
-        logger.info('ReferentDict module lead successfully')
+        if ref_dict is not None and os.path.isfile(ref_dict):
+            if method == OnlineOROfflineMethod.OFFLINE:
+                self.ref_dict = load_json_file(ref_dict)
+            elif method == OnlineOROfflineMethod.ONLINE:
+                self.ref_dict = self.load_reference_dict(ref_dict)
+            logger.info('ReferentDict module lead successfully')
+        else:
+            raise FileNotFoundError('Referent Dict file not found or not in path..')
+
         super(ReferentDictRelationExtraction, self).__init__()
 
     def extract_all_relations(self, mention_x: MentionDataLight,

nlp_architect/data/cdc_resources/relations/verbocean_relation_extraction.py

@@ -15,6 +15,7 @@
 # ******************************************************************************
 
 import logging
+import os
 from typing import Dict, Set
 
 from nlp_architect.common.cdc.mention_data import MentionDataLight
@@ -38,11 +39,14 @@ def __init__(self, method: OnlineOROfflineMethod, vo_file: str):
             vo_file (required): str Location of VerbOcean file to work with
         """
         logger.info('Loading Verb Ocean module')
-        if method == OnlineOROfflineMethod.OFFLINE:
-            self.vo = load_json_file(vo_file)
-        elif method == OnlineOROfflineMethod.ONLINE:
-            self.vo = self.load_verbocean_file(vo_file)
-        logger.info('Verb Ocean module lead successfully')
+        if vo_file is not None and os.path.isfile(vo_file):
+            if method == OnlineOROfflineMethod.OFFLINE:
+                self.vo = load_json_file(vo_file)
+            elif method == OnlineOROfflineMethod.ONLINE:
+                self.vo = self.load_verbocean_file(vo_file)
+            logger.info('Verb Ocean module lead successfully')
+        else:
+            raise FileNotFoundError('VerbOcean file not found or not in path..')
         super(VerboceanRelationExtraction, self).__init__()
 
     def extract_all_relations(self, mention_x: MentionDataLight,

nlp_architect/data/cdc_resources/relations/wikipedia_relation_extraction.py

@@ -17,6 +17,7 @@
 from __future__ import division
 
 import logging
+import os
 from typing import Set, List
 
 from nlp_architect.common.cdc.mention_data import MentionDataLight
@@ -50,13 +51,19 @@ def __init__(self, method: WikipediaSearchMethod, wiki_file: str = None, host: s
             index (required on Elastic mode): int the Elastic search index name
         """
         logger.info('Loading Wikipedia module')
+
         connectivity = method
         if connectivity == WikipediaSearchMethod.ONLINE:
             self.pywiki_impl = WikiOnline()
         elif connectivity == WikipediaSearchMethod.OFFLINE:
-            self.pywiki_impl = WikiOffline(wiki_file)
+            if wiki_file is not None and os.path.isdir(wiki_file):
+                self.pywiki_impl = WikiOffline(wiki_file)
+            else:
+                raise FileNotFoundError('Wikipedia resource file not found or not in path, '
+                                        'create it or change the initialization method')
         elif connectivity == WikipediaSearchMethod.ELASTIC:
             self.pywiki_impl = WikiElastic(host, port, index)
+
         logger.info('Wikipedia module lead successfully')
         super(WikipediaRelationExtraction, self).__init__()
 

nlp_architect/data/cdc_resources/relations/within_doc_coref_extraction.py

@@ -13,13 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ******************************************************************************
+import logging
+import os
 from typing import List, Set
 
 from nlp_architect.common.cdc.mention_data import MentionData
 from nlp_architect.data.cdc_resources.relations.relation_extraction import RelationExtraction
 from nlp_architect.data.cdc_resources.relations.relation_types_enums import RelationType
 from nlp_architect.utils.io import load_json_file
 
+logger = logging.getLogger(__name__)
+
 
 class WithinDocCoref(RelationExtraction):
     def __init__(self, wd_file: str):
@@ -29,8 +33,12 @@ def __init__(self, wd_file: str):
         Args:
             wd_file (required): str Location of within doc co-reference mentions file
         """
-        wd_mentions_json = load_json_file(wd_file)
-        self.within_doc_coref_chain = self.arrange_resource(wd_mentions_json)
+        logger.info('Loading Within doc resource')
+        if wd_file is not None and os.path.isfile(wd_file):
+            wd_mentions_json = load_json_file(wd_file)
+            self.within_doc_coref_chain = self.arrange_resource(wd_mentions_json)
+        else:
+            raise FileNotFoundError('Within-doc resource file not found or not in path')
         super(WithinDocCoref, self).__init__()
 
     @staticmethod

nlp_architect/data/cdc_resources/relations/wordnet_relation_extraction.py

@@ -15,6 +15,7 @@
 # ******************************************************************************
 
 import logging
+import os
 from typing import Set, List
 
 from nlp_architect.common.cdc.mention_data import MentionDataLight
@@ -44,7 +45,11 @@ def __init__(self, method: OnlineOROfflineMethod, wn_file: str = None):
         if self.connectivity == OnlineOROfflineMethod.ONLINE:
             self.wordnet_impl = WordnetOnline()
         elif self.connectivity == OnlineOROfflineMethod.OFFLINE:
-            self.wordnet_impl = WordnetOffline(wn_file)
+            if wn_file is not None and os.path.isdir(wn_file):
+                self.wordnet_impl = WordnetOffline(wn_file)
+            else:
+                raise FileNotFoundError('WordNet resource directory not found or not in path')
+
         logger.info('Wordnet module lead successfully')
         super(WordnetRelationExtraction, self).__init__()
 

nlp_architect/models/cross_doc_coref/cdc_config.py

@@ -92,7 +92,7 @@ def __init__(self):
         ]
 
         self.gold_mentions_file = LIBRARY_ROOT + \
-            '/datasets/cdc/gold_mentions/ecb_test_event_gold_mentions.json'
+            '/datasets/ecb/ecb_all_event_mentions.json'
 
 
 class EntityConfig(CDCConfig):
@@ -120,4 +120,4 @@ def __init__(self):
         ]
 
         self.gold_mentions_file = LIBRARY_ROOT + \
-            '/datasets/cdc/gold_mentions/ecb_test_entity_gold_mentions.json'
+            '/datasets/ecb/ecb_all_entity_mentions.json'