ignore non-sacr file when loading fr-litbank

Author: Aethor

tibert/bertcoref.py

@@ -486,21 +486,27 @@ def from_sacr_dir(
         tokenizer: PreTrainedTokenizerFast,
         max_span_size: int,
         lang: str,
+        ignored_files: Optional[List[str]] = None,
         **kwargs,
     ) -> CoreferenceDataset:
         """
         :param path: path to a directory containing .sacr files
         :param tokenizer:
         :param max_span_size:
         :param lang: MosesTokenizer language ('en', 'fr', 'de'...)
+        :param ignored_files: list of filenames to ignore
         :param kwargs: passed to ``open``
         """
         path = os.path.expanduser(path)
 
         documents = []
         m_tokenizer = MosesTokenizer(lang=lang)
 
-        for fpath in tqdm(sorted(glob.glob(f"{path}/*.sacr"))):
+        paths = sorted(glob.glob(f"{path}/*.sacr"))
+        if not ignored_files is None:
+            paths = [p for p in paths if not os.path.basename(p) in ignored_files]
+
+        for fpath in tqdm(paths):
             with open(fpath, **kwargs) as f:
                 text = f.read().replace("\n", " ")
 
@@ -614,7 +620,11 @@ def load_fr_litbank_dataset(
 ):
     root_path = os.path.expanduser(root_path.rstrip("/"))
     return CoreferenceDataset.from_sacr_dir(
-        f"{root_path}/sacr/All_Entites", tokenizer, max_span_size, "en"
+        f"{root_path}/sacr/All_Entites",
+        tokenizer,
+        max_span_size,
+        "en",
+        ignored_files=["schema.sacr"],
     )