Merge pull request #211 from TensorSpeech/add_eos_abstract_method

📝 Add abstractmethod 'setup_eos_token' to base_processor.

Author: dathudeptrai

setup.py

@@ -22,7 +22,7 @@
 # TODO(@dathudeptrai) update requirement if needed.
 requirements = {
     "install": [
-        "tensorflow-gpu==2.3.0",
+        "tensorflow-gpu>=2.2.0",
         "tensorflow-addons>=0.10.0",
         "setuptools>=38.5.1",
         "librosa>=0.7.0",

tensorflow_tts/processor/baker.py

@@ -549,6 +549,9 @@ def __post_init__(self):
         super().__post_init__()
         self.pinyin_parser = self.get_pinyin_parser()
 
+    def setup_eos_token(self):
+        return _eos[0]
+
     def create_items(self):
         items = []
         if self.data_dir:

tensorflow_tts/processor/base_processor.py

@@ -61,10 +61,16 @@ def __post_init__(self):
         self.create_symbols()
         if self.saved_mapper_path is not None:
             self._save_mapper(saved_path=self.saved_mapper_path)
-        
+
         # processor name. usefull to use it for AutoProcessor
         self._processor_name = type(self).__name__
 
+        if self.setup_eos_token():
+            self.add_symbol(
+                self.setup_eos_token()
+            )  # if this eos token not yet present in symbols list.
+            self.eos_id = self.symbol_to_id[self.setup_eos_token()]
+
     def __getattr__(self, name: str) -> Union[str, int]:
         if "_id" in name:  # map symbol to id
             return self.symbol_to_id[name.replace("_id", "")]
@@ -151,6 +157,11 @@ def get_one_sample(self, item):
     def text_to_sequence(self, text: str):
         return []
 
+    @abc.abstractmethod
+    def setup_eos_token(self):
+        """Return eos symbol of type string."""
+        return "eos"
+
     def convert_symbols_to_ids(self, symbols: Union[str, list]):
         sequence = []
         if isinstance(symbols, str):

tensorflow_tts/processor/kss.py

@@ -54,6 +54,9 @@ def split_line(self, data_dir, line, split):
         speaker_name = "kss"
         return text_norm, wav_path, speaker_name
 
+    def setup_eos_token(self):
+        return "eos"
+
     def get_one_sample(self, item):
         text, wav_path, speaker_name = item
 

tensorflow_tts/processor/libritts.py

@@ -83,6 +83,9 @@ def get_one_sample(self, item):
 
         return sample
 
+    def setup_eos_token(self):
+        return None # because we do not use this 
+
     def text_to_sequence(self, text):
         if (
             self.mode == "train"

tensorflow_tts/processor/ljspeech.py

@@ -155,6 +155,9 @@ def split_line(self, data_dir, line, split):
         speaker_name = "ljspeech"
         return text_norm, wav_path, speaker_name
 
+    def setup_eos_token(self):
+        return _eos
+
     def get_one_sample(self, item):
         text, wav_path, speaker_name = item
 

test/test_base_processor.py

@@ -21,6 +21,9 @@ def get_one_sample(self, item):
     def text_to_sequence(self, text):
         return ["0"]
 
+    def setup_eos_token(self):
+        return None
+
 
 @pytest.fixture
 def processor(tmpdir):