OV 2.0: python text_to_speech demo (#3038)

* first version

* update for melgan

* some fixes

* update rnn_width calue

* update name for create infer request function

* add tests and parsing string input

* remove mistakes

* remove mistakes2

* update readme.md

* update cases.py

* remove mistakes3

* update readme

* update requirements.txt

* remove mistakes in requirements.txt

* update requirements-demos.txt

* Update ci/requirements-demos.txt

Co-authored-by: Ekaterina Aidova <[email protected]>

* update reqs for demo

* amend

* change input and location tkinter import

* Update demos/text_to_speech_demo/python/models/forward_tacotron_ie.py

* Update demos/text_to_speech_demo/python/models/forward_tacotron_ie.py

Co-authored-by: VoronovaIntern <[email protected]>
Co-authored-by: Ekaterina Aidova <[email protected]>
Co-authored-by: Anzhella Pankratova <[email protected]>

Author: 4 people

ci/requirements-demos.txt

@@ -154,6 +154,7 @@ xmltodict==0.12.0
     # via motmetrics
 zipp==3.5.0
     # via importlib-metadata
+inflect==5.3.0
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

demos/requirements.txt

@@ -10,3 +10,4 @@ sympy>=1.8
 tokenizers~=0.10.1
 tensorboardX>=2.1
 tqdm>=4.54.1
+inflect>=5.3.0

demos/tests/cases.py

@@ -1307,6 +1307,32 @@ def single_option_cases(key, *args):
         ]
     )),
 
+    PythonDemo(name='text_to_speech_demo', device_keys=['-d'],
+               model_keys=['-m_duration', '-m_forward', '-m_upsample', '-m_rnn', '-m_melgan'], test_cases=combine_cases(
+        TestCase(options={'-i': [
+                    'The quick brown fox jumps over the lazy dog.',
+                    'The five boxing wizards jump quickly.'
+                ]}),
+        [
+            TestCase(options={
+                '-m_duration': ModelArg('forward-tacotron-duration-prediction'),
+                '-m_forward': ModelArg('forward-tacotron-regression'),
+                '-m_upsample': ModelArg('wavernn-upsampler'),
+                '-m_rnn': ModelArg('wavernn-rnn')
+            }),
+            TestCase(options={
+                '-m_duration': ModelArg('text-to-speech-en-0001-duration-prediction'),
+                '-m_forward': ModelArg('text-to-speech-en-0001-regression'),
+                '-m_melgan': ModelArg('text-to-speech-en-0001-generation')
+            }),
+            TestCase(options={
+                '-m_duration': ModelArg('text-to-speech-en-multi-0001-duration-prediction'),
+                '-m_forward': ModelArg('text-to-speech-en-multi-0001-regression'),
+                '-m_melgan': ModelArg('text-to-speech-en-multi-0001-generation')
+            }),
+        ]
+    )),
+
     PythonDemo(name='whiteboard_inpainting_demo', device_keys=['-d'],
                model_keys=['-m_i', '-m_s'], test_cases=combine_cases(
         TestCase(options={'-i': TestDataArg('msasl/global_crops/_nz_sivss20/clip_0017/img_%05d.jpg'),

demos/text_to_speech_demo/python/README.md

@@ -63,7 +63,7 @@ Options:
                         Required. Path to ForwardTacotron`s mel-spectrogram
                         regression part (*.xml format).
   -i INPUT, --input INPUT
-                        Required. Text file with text.
+                        Required. Text or path to the input file.
   -o OUT, --out OUT     Optional. Path to an output .wav file
   -d DEVICE, --device DEVICE
                         Optional. Specify the target device to infer on; CPU,

demos/text_to_speech_demo/python/models/forward_tacotron_ie.py

@@ -1,5 +1,5 @@
 """
- Copyright (c) 2020 Intel Corporation
+ Copyright (c) 2020-2022 Intel Corporation
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -23,31 +23,39 @@
 from utils.embeddings_processing import PCA
 
 
+def check_input_name(model, input_tensor_name):
+    try:
+        model.input(input_tensor_name)
+        return True
+    except RuntimeError:
+        return False
+
+
 class ForwardTacotronIE:
     def __init__(self, model_duration, model_forward, ie, device='CPU', verbose=False):
         self.verbose = verbose
         self.device = device
         self.ie = ie
 
-        self.duration_predictor_net = self.load_network(model_duration)
-        self.duration_predictor_exec = self.create_exec_network(self.duration_predictor_net, model_duration)
+        self.duration_predictor_model = self.load_network(model_duration)
+        self.duration_predictor_request = self.create_infer_request(self.duration_predictor_model, model_duration)
 
-        self.forward_net = self.load_network(model_forward)
-        self.forward_exec = self.create_exec_network(self.forward_net, model_forward)
+        self.forward_model = self.load_network(model_forward)
+        self.forward_request = self.create_infer_request(self.forward_model, model_forward)
 
         # fixed length of the sequence of symbols
-        self.duration_len = self.duration_predictor_net.input_info['input_seq'].input_data.shape[1]
+        self.duration_len = self.duration_predictor_model.input('input_seq').shape[1]
         # fixed length of the input embeddings for forward
-        self.forward_len = self.forward_net.input_info['data'].input_data.shape[1]
+        self.forward_len = self.forward_model.input('data').shape[1]
         if self.verbose:
             log.debug('Forward limitations : {0} symbols and {1} embeddings'.format(self.duration_len, self.forward_len))
-        self.is_attention = 'pos_mask' in self.forward_net.input_info
+        self.is_attention = check_input_name(self.forward_model, "pos_mask")
         if self.is_attention:
             self.init_pos_mask()
         else:
             self.pos_mask = None
 
-        self.is_multi_speaker = self.has_speaker_embeddings()
+        self.is_multi_speaker = check_input_name(self.duration_predictor_model, "speaker_embedding")
         if self.is_multi_speaker:
             self.init_speaker_information()
         else:
@@ -106,13 +114,13 @@ def load_network(self, model_xml):
         model_bin_name = ".".join(osp.basename(model_xml).split('.')[:-1]) + ".bin"
         model_bin = osp.join(osp.dirname(model_xml), model_bin_name)
         log.info('Reading ForwardTacotron model {}'.format(model_xml))
-        net = self.ie.read_network(model=model_xml, weights=model_bin)
-        return net
+        model = self.ie.read_model(model=model_xml, weights=model_bin)
+        return model
 
-    def create_exec_network(self, net, path):
-        exec_net = self.ie.load_network(network=net, device_name=self.device)
+    def create_infer_request(self, model, path):
+        compiled_model = self.ie.compile_model(model, device_name=self.device)
         log.info('The ForwardTacotron model {} is loaded to {}'.format(path, self.device))
-        return exec_net
+        return compiled_model.create_infer_request()
 
     def infer_duration(self, sequence, speaker_embedding=None, alpha=1.0, non_empty_symbols=None):
         if self.is_attention:
@@ -122,15 +130,15 @@ def infer_duration(self, sequence, speaker_embedding=None, alpha=1.0, non_empty_
                       "input_mask": input_mask,
                       "pos_mask": pos_mask}
             if speaker_embedding is not None:
-                inputs["speaker_embedding"] = speaker_embedding
-            out = self.duration_predictor_exec.infer(inputs)
+                inputs["speaker_embedding"] = np.array([speaker_embedding])
+            self.duration_predictor_request.infer(inputs)
         else:
-            out = self.duration_predictor_exec.infer(inputs={"input_seq": sequence})
-        duration = out["duration"] * alpha
+            self.duration_predictor_request.infer(inputs={"input_seq": sequence})
+        duration = self.duration_predictor_request.get_tensor("duration").data[:] * alpha
 
         duration = (duration + 0.5).astype('int').flatten()
         duration = np.expand_dims(duration, axis=0)
-        preprocessed_embeddings = out["embeddings"]
+        preprocessed_embeddings = self.duration_predictor_request.get_tensor("embeddings").data[:]
 
         if non_empty_symbols is not None:
             duration = duration[:, :non_empty_symbols]
@@ -150,11 +158,11 @@ def infer_mel(self, aligned_emb, non_empty_symbols, speaker_embedding=None):
                       "data_mask": data_mask,
                       "pos_mask": pos_mask}
             if speaker_embedding is not None:
-                inputs["speaker_embedding"] = speaker_embedding
-            out = self.forward_exec.infer(inputs)
+                inputs["speaker_embedding"] = np.array([speaker_embedding])
+            self.forward_request.infer(inputs)
         else:
-            out = self.forward_exec.infer(inputs={"data": aligned_emb})
-        return out['mel'][:, :non_empty_symbols]
+            self.forward_request.infer(inputs={"data": aligned_emb})
+        return self.forward_request.get_tensor('mel').data[:, :non_empty_symbols]
 
     def find_optimal_delimiters_position(self, sequence, delimiters, idx, window=20):
         res = {d: -1 for d in delimiters}
@@ -238,16 +246,12 @@ def forward(self, text, alpha=1.0, speaker_id=19, speaker_emb=None):
         return res
 
     def get_speaker_embeddings(self):
-        if self.has_speaker_embeddings():
+        if self.is_multi_speaker:
             return self.speaker_embeddings
-        else:
-            return None
-
-    def has_speaker_embeddings(self):
-        return 'speaker_embedding' in self.duration_predictor_net.input_info
+        return None
 
     def get_pca_speaker_embedding(self, gender, alpha):
-        if not self.has_speaker_embeddings():
+        if not self.is_multi_speaker:
             return None
 
         emb = self.male_embeddings if gender == "Male" else self.female_embeddings

demos/text_to_speech_demo/python/models/mel2wave_ie.py

@@ -1,5 +1,5 @@
 """
- Copyright (c) 2020 Intel Corporation
+ Copyright (c) 2020-2022 Intel Corporation
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@
 
 import numpy as np
 
+from openvino.runtime import PartialShape, set_batch, Layout
 from utils.wav_processing import (
     fold_with_overlap, infer_from_discretized_mix_logistic, pad_tensor, xfade_and_unfold,
 )
@@ -49,37 +50,41 @@ def __init__(self, model_upsample, model_rnn, ie, target=11000, overlap=550, hop
         self.batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256]
         self.ie = ie
 
-        self.upsample_net = self.load_network(model_upsample)
+        self.upsample_model = self.load_network(model_upsample)
         if upsampler_width > 0:
-            orig_shape = self.upsample_net.input_info['mels'].input_data.shape
-            self.upsample_net.reshape({"mels": (orig_shape[0], upsampler_width, orig_shape[2])})
+            orig_shape = self.upsample_model.input('mels').shape
+            self.upsample_model.reshape({"mels": PartialShape([orig_shape[0], upsampler_width, orig_shape[2]])})
 
-        self.upsample_exec = self.create_exec_network(self.upsample_net, model_upsample)
+        self.upsample_request = self.create_infer_requests(self.upsample_model, model_upsample)
 
-        self.rnn_net = self.load_network(model_rnn)
-        self.rnn_exec = self.create_exec_network(self.rnn_net, model_rnn, batch_sizes=self.batch_sizes)
+        self.rnn_model = self.load_network(model_rnn)
+        self.rnn_requests = self.create_infer_requests(self.rnn_model, model_rnn, batch_sizes=self.batch_sizes)
 
         # fixed number of the mels in mel-spectrogramm
-        self.mel_len = self.upsample_net.input_info['mels'].input_data.shape[1] - 2 * self.pad
-        self.rnn_width = self.rnn_net.input_info['x'].input_data.shape[1]
+        self.mel_len = self.upsample_model.input('mels').shape[1] - 2 * self.pad
+        self.rnn_width = self.rnn_model.input('h1.1').shape[1]
 
     def load_network(self, model_xml):
         model_bin_name = ".".join(osp.basename(model_xml).split('.')[:-1]) + ".bin"
         model_bin = osp.join(osp.dirname(model_xml), model_bin_name)
         log.info('Reading WaveRNN model {}'.format(model_xml))
-        net = self.ie.read_network(model=model_xml, weights=model_bin)
-        return net
+        model = self.ie.read_model(model=model_xml, weights=model_bin)
+        return model
 
-    def create_exec_network(self, net, path, batch_sizes=None):
+    def create_infer_requests(self, model, path, batch_sizes=None):
         if batch_sizes is not None:
-            exec_net = []
+            requests = []
+            for parameter in model.get_parameters():
+                parameter.set_layout(Layout("BC"))
             for b_s in batch_sizes:
-                net.batch_size = b_s
-                exec_net.append(self.ie.load_network(network=net, device_name=self.device))
+                set_batch(model, b_s)
+                compiled_model = self.ie.compile_model(model, device_name=self.device)
+                requests.append(compiled_model.create_infer_request())
         else:
-            exec_net = self.ie.load_network(network=net, device_name=self.device)
+            compiled_model = self.ie.compile_model(model, device_name=self.device)
+            requests = compiled_model.create_infer_request()
         log.info('The WaveRNN model {} is loaded to {}'.format(path, self.device))
-        return exec_net
+        return requests
 
     @staticmethod
     def get_rnn_init_states(b_size=1, rnn_dims=328):
@@ -133,8 +138,9 @@ def forward(self, mels):
     def forward_upsample(self, mels):
         mels = pad_tensor(mels, pad=self.pad)
 
-        out = self.upsample_exec.infer(inputs={"mels": mels})
-        upsample_mels, aux = out["upsample_mels"][:, self.indent:-self.indent, :], out["aux"]
+        self.upsample_request.infer(inputs={"mels": mels})
+        upsample_mels = self.upsample_request.get_tensor("upsample_mels").data[:, self.indent:-self.indent, :]
+        aux = self.upsample_request.get_tensor("aux").data[:]
         return upsample_mels, aux
 
     def forward_rnn(self, mels, upsampled_mels, aux):
@@ -160,13 +166,12 @@ def forward_rnn(self, mels, upsampled_mels, aux):
 
             a1_t, a2_t, a3_t, a4_t = \
                 (a[:, i, :] for a in aux_split)
+            self.rnn_requests[active_network].infer(inputs={"m_t": m_t, "a1_t": a1_t, "a2_t": a2_t, "a3_t": a3_t,
+                                                           "a4_t": a4_t, "h1.1": h1, "h2.1": h2, "x": x})
 
-            out = self.rnn_exec[active_network].infer(inputs={"m_t": m_t, "a1_t": a1_t, "a2_t": a2_t, "a3_t": a3_t,
-                                                              "a4_t": a4_t, "h1.1": h1, "h2.1": h2, "x": x})
-
-            logits = out["logits"]
-            h1 = out["h1"]
-            h2 = out["h2"]
+            logits = self.rnn_requests[active_network].get_tensor('logits').data[:]
+            h1 = self.rnn_requests[active_network].get_tensor('h1').data[:]
+            h2 = self.rnn_requests[active_network].get_tensor('h2').data[:]
 
             sample = infer_from_discretized_mix_logistic(logits)
 
@@ -204,38 +209,40 @@ def __init__(self, model, ie, device='CPU', default_width=800):
         self.scales = 4
         self.hop_length = 256
 
-        self.net = self.load_network(model)
-        if self.net.input_info['mel'].input_data.shape[2] != default_width:
-            orig_shape = self.net.input_info['mel'].input_data.shape
+        self.model = self.load_network(model)
+        if self.model.input('mel').shape[2] != default_width:
+            orig_shape = self.model.input('mel').shape
             new_shape = (orig_shape[0], orig_shape[1], default_width)
-            self.net.reshape({"mel": new_shape})
+            self.model.reshape({"mel": PartialShape([new_shape[0], new_shape[1], new_shape[2]])})
 
-        self.exec_net = self.create_exec_network(self.net, self.scales)
+        self.requests = self.create_infer_requests(self.model, model, self.scales)
 
         # fixed number of columns in mel-spectrogramm
-        self.mel_len = self.net.input_info['mel'].input_data.shape[2]
+        self.mel_len = self.model.input('mel').shape[2]
         self.widths = [self.mel_len * (i + 1) for i in range(self.scales)]
 
     def load_network(self, model_xml):
         model_bin_name = ".".join(osp.basename(model_xml).split('.')[:-1]) + ".bin"
         model_bin = osp.join(osp.dirname(model_xml), model_bin_name)
         log.info('Reading MelGAN model {}'.format(model_xml))
-        net = self.ie.read_network(model=model_xml, weights=model_bin)
-        return net
+        model = self.ie.read_model(model=model_xml, weights=model_bin)
+        return model
 
-    def create_exec_network(self, net, path, scales=None):
+    def create_infer_requests(self, model, path, scales=None):
         if scales is not None:
-            orig_shape = net.input_info['mel'].input_data.shape
-            exec_net = []
+            orig_shape = model.input('mel').shape
+            requests = []
             for i in range(scales):
                 new_shape = (orig_shape[0], orig_shape[1], orig_shape[2] * (i + 1))
-                net.reshape({"mel": new_shape})
-                exec_net.append(self.ie.load_network(network=net, device_name=self.device))
-                net.reshape({"mel": orig_shape})
+                model.reshape({"mel": PartialShape([new_shape[0], new_shape[1], new_shape[2]])})
+                compiled_model = self.ie.compile_model(model, device_name=self.device)
+                requests.append(compiled_model.create_infer_request())
+                model.reshape({"mel": PartialShape([orig_shape[0], orig_shape[1], orig_shape[2]])})
         else:
-            exec_net = self.ie.load_network(network=net, device_name=self.device)
+            compiled_model = self.ie.compile_model(model, device_name=self.device)
+            requests = compiled_model.create_infer_request()
         log.info('The MelGAN model {} is loaded to {}'.format(path, self.device))
-        return exec_net
+        return requests
 
     def forward(self, mel):
         mel = np.expand_dims(mel, axis=0)
@@ -261,7 +268,8 @@ def forward(self, mel):
         c_begin = 0
         c_end = cur_w
         while c_begin < cols:
-            audio = self.exec_net[active_net].infer(inputs={"mel": mel[:, :, c_begin:c_end]})["audio"]
+            self.requests[active_net].infer(inputs={"mel": mel[:, :, c_begin:c_end]})
+            audio = self.requests[active_net].get_tensor("audio").data[:]
             res_audio.extend(audio)
 
             c_begin = c_end