Remove librosa dependency from package

Author: pythonlessons

CHANGELOG.md

@@ -1,3 +1,10 @@
+## [1.1.2] - 2022-09-29
+### Changed
+- Removed `Librosa` library dependency in requirements, now it is optional and required only with modules that use librosa
+
+### Added
+- Created `Tutorials.05_sound_to_text.train_no_limit.py` that demonstrates how to train audio recognition model with `mltu` without audio length limit
+
 ## [1.1.1] - 2022-09-26
 ### Changed
 - Included `self._executor` as generator in `mltu.dataProvider.DataProvider` object, to enable functionality to modify batch preprocessing without changing original code

Tutorials/05_sound_to_text/model.py

@@ -7,7 +7,7 @@
 
 def train_model(input_dim, output_dim, activation="leaky_relu", dropout=0.2):
 
-    inputs = layers.Input(shape=input_dim, name="input")
+    inputs = layers.Input(shape=input_dim, name="input", dtype=tf.float32)
 
     # expand dims to add channel dimension
     input = layers.Lambda(lambda x: tf.expand_dims(x, axis=-1))(inputs)
@@ -46,7 +46,7 @@ def train_model(input_dim, output_dim, activation="leaky_relu", dropout=0.2):
     x = layers.Dropout(dropout)(x)
 
     # Classification layer
-    output = layers.Dense(output_dim + 1, activation="softmax")(x)
+    output = layers.Dense(output_dim + 1, activation="softmax", dtype=tf.float32)(x)
 
     model = Model(inputs=inputs, outputs=output)
     return model

Tutorials/05_sound_to_text/train_no_limit.py

@@ -0,0 +1,117 @@
+import tensorflow as tf
+try: [tf.config.experimental.set_memory_growth(gpu, True) for gpu in tf.config.experimental.list_physical_devices("GPU")]
+except: pass
+tf.keras.mixed_precision.set_global_policy('mixed_float16') # mixed precission training for faster training time
+
+import os
+import tarfile
+import pandas as pd
+from tqdm import tqdm
+from urllib.request import urlopen
+from io import BytesIO
+
+from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard
+from mltu.preprocessors import WavReader
+
+from mltu.tensorflow.dataProvider import DataProvider
+from mltu.transformers import LabelIndexer, LabelPadding, SpectrogramPadding
+from mltu.tensorflow.losses import CTCloss
+from mltu.tensorflow.callbacks import Model2onnx, TrainLogger
+from mltu.tensorflow.metrics import CERMetric, WERMetric
+
+from model import train_model
+from configs import ModelConfigs
+
+
+def download_and_unzip(url, extract_to="Datasets", chunk_size=1024*1024):
+    http_response = urlopen(url)
+
+    data = b""
+    iterations = http_response.length // chunk_size + 1
+    for _ in tqdm(range(iterations)):
+        data += http_response.read(chunk_size)
+
+    tarFile = tarfile.open(fileobj=BytesIO(data), mode="r|bz2")
+    tarFile.extractall(path=extract_to)
+    tarFile.close()
+
+
+dataset_path = os.path.join("Datasets", "LJSpeech-1.1")
+if not os.path.exists(dataset_path):
+    download_and_unzip("https://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2", extract_to="Datasets")
+
+dataset_path = "Datasets/LJSpeech-1.1"
+metadata_path = dataset_path + "/metadata.csv"
+wavs_path = dataset_path + "/wavs/"
+
+# Read metadata file and parse it
+metadata_df = pd.read_csv(metadata_path, sep="|", header=None, quoting=3)
+metadata_df.columns = ["file_name", "transcription", "normalized_transcription"]
+metadata_df = metadata_df[["file_name", "normalized_transcription"]]
+
+# structure the dataset where each row is a list of [wav_file_path, sound transcription]
+dataset = [[f"Datasets/LJSpeech-1.1/wavs/{file}.wav", label.lower()] for file, label in metadata_df.values.tolist()]
+
+# Create a ModelConfigs object to store model configurations
+configs = ModelConfigs()
+configs.save()
+
+# Create a data provider for the dataset
+data_provider = DataProvider(
+    dataset=dataset,
+    skip_validation=True,
+    batch_size=configs.batch_size,
+    data_preprocessors=[
+        WavReader(frame_length=configs.frame_length, frame_step=configs.frame_step, fft_length=configs.fft_length),
+        ],
+    transformers=[
+        LabelIndexer(configs.vocab),
+        ],
+    batch_postprocessors=[
+        SpectrogramPadding(padding_value=0, use_on_batch=True),
+        LabelPadding(padding_value=len(configs.vocab), use_on_batch=True),
+    ],
+)
+
+# Split the dataset into training and validation sets
+train_data_provider, val_data_provider = data_provider.split(split = 0.9)
+
+# Creating TensorFlow model architecture
+model = train_model(
+    input_dim = (None, 193),
+    output_dim = len(configs.vocab),
+    dropout=0.5
+)
+
+# Compile the model and print summary
+model.compile(
+    optimizer=tf.keras.optimizers.Adam(learning_rate=configs.learning_rate), 
+    loss=CTCloss(), 
+    metrics=[
+        CERMetric(vocabulary=configs.vocab),
+        WERMetric(vocabulary=configs.vocab)
+        ],
+    run_eagerly=False
+)
+model.summary(line_length=110)
+
+# Define callbacks
+earlystopper = EarlyStopping(monitor="val_CER", patience=20, verbose=1, mode="min")
+checkpoint = ModelCheckpoint(f"{configs.model_path}/model.h5", monitor="val_CER", verbose=1, save_best_only=True, mode="min")
+trainLogger = TrainLogger(configs.model_path)
+tb_callback = TensorBoard(f"{configs.model_path}/logs", update_freq=1)
+reduceLROnPlat = ReduceLROnPlateau(monitor="val_CER", factor=0.8, min_delta=1e-10, patience=5, verbose=1, mode="auto")
+model2onnx = Model2onnx(f"{configs.model_path}/model.h5")
+
+# Train the model
+model.fit(
+    train_data_provider,
+    validation_data=val_data_provider,
+    epochs=configs.train_epochs,
+    callbacks=[earlystopper, checkpoint, trainLogger, reduceLROnPlat, tb_callback, model2onnx],
+    workers=configs.train_workers,
+)
+
+# Save training and validation datasets as csv files
+train_data_provider.to_csv(os.path.join(configs.model_path, "train.csv"))
+val_data_provider.to_csv(os.path.join(configs.model_path, "val.csv"))

mltu/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.1.1"
+__version__ = "1.1.2"
 
 from .annotations.images import Image
 from .annotations.images import CVImage

mltu/dataProvider.py

@@ -216,7 +216,6 @@ def executor(batch_data):
 
     def __iter__(self):
         """ Create a generator that iterate over the Sequence."""
-        self.start_executor()
         for index in range(len(self)):
             results = self[index]
             yield results
@@ -269,6 +268,9 @@ def __getitem__(self, index: int):
         Returns:
             tuple: batch of data and batch of annotations
         """
+        if index==0:
+            self.start_executor()
+
         dataset_batch = self.get_batch_annotations(index)
 
         # First read and preprocess the batch data

mltu/preprocessors.py

@@ -1,7 +1,5 @@
 import os
 import typing
-import librosa
-import librosa.display
 import numpy as np
 import matplotlib.pyplot as plt
 import matplotlib
@@ -106,6 +104,11 @@ class WavReader:
         frame_step (int): Step size between frames in samples.
         fft_length (int): Number of FFT components.
     """
+    try:
+        import librosa
+    except ImportError:
+        raise ImportError("librosa is required to read Audio. Please install it with `pip install librosa`.")
+
     def __init__(
             self,
             frame_length: int = 256,
@@ -133,12 +136,12 @@ def get_spectrogram(wav_path: str, frame_length: int, frame_step: int, fft_lengt
             np.ndarray: Spectrogram of the WAV file.
         """
         # Load the wav file and store the audio data in the variable 'audio' and the sample rate in 'orig_sr'
-        audio, orig_sr = librosa.load(wav_path) 
+        audio, orig_sr = WavReader.librosa.load(wav_path) 
 
         # Compute the Short Time Fourier Transform (STFT) of the audio data and store it in the variable 'spectrogram'
         # The STFT is computed with a hop length of 'frame_step' samples, a window length of 'frame_length' samples, and 'fft_length' FFT components.
         # The resulting spectrogram is also transposed for convenience
-        spectrogram = librosa.stft(audio, hop_length=frame_step, win_length=frame_length, n_fft=fft_length).T
+        spectrogram = WavReader.librosa.stft(audio, hop_length=frame_step, win_length=frame_length, n_fft=fft_length).T
 
         # Take the absolute value of the spectrogram to obtain the magnitude spectrum
         spectrogram = np.abs(spectrogram)
@@ -162,7 +165,7 @@ def plot_raw_audio(wav_path: str, title: str = None, sr: int = 16000) -> None:
             title (str, optional): Title
         """
         # Load the wav file and store the audio data in the variable 'audio' and the sample rate in 'orig_sr'
-        audio, orig_sr = librosa.load(wav_path, sr=sr)
+        audio, orig_sr = WavReader.librosa.load(wav_path, sr=sr)
 
         duration = len(audio) / orig_sr
 

mltu/transformers.py

@@ -137,18 +137,34 @@ class LabelPadding(Transformer):
     """Pad label to max_word_length
     
     Attributes:
-        max_word_length (int): Maximum length of label
         padding_value (int): Value to pad
+        max_word_length (int): Maximum length of label
+        use_on_batch (bool): Whether to use on batch. Default: False
     """
     def __init__(
         self, 
-        max_word_length: int, 
-        padding_value: int
+        padding_value: int,
+        max_word_length: int = None, 
+        use_on_batch: bool = False
         ) -> None:
         self.max_word_length = max_word_length
         self.padding_value = padding_value
+        self.use_on_batch = use_on_batch
+
+        if not use_on_batch and max_word_length is None:
+            raise ValueError("max_word_length must be specified if use_on_batch is False")
 
     def __call__(self, data: np.ndarray, label: np.ndarray):
+        if self.use_on_batch:
+            max_len = max([len(a) for a in label])
+            padded_labels = []
+            for l in label:
+                padded_label = np.pad(l, (0, max_len - len(l)), "constant", constant_values=self.padding_value)
+                padded_labels.append(padded_label)
+
+            padded_labels = np.array(padded_labels)
+            return data, padded_labels
+
         label = label[:self.max_word_length]
         return data, np.pad(label, (0, self.max_word_length - len(label)), "constant", constant_values=self.padding_value)
 
@@ -157,22 +173,41 @@ class SpectrogramPadding(Transformer):
     """Pad spectrogram to max_spectrogram_length
     
     Attributes:
-        max_spectrogram_length (int): Maximum length of spectrogram
         padding_value (int): Value to pad
+        max_spectrogram_length (int): Maximum length of spectrogram. Must be specified if use_on_batch is False. Default: None
+        use_on_batch (bool): Whether to use on batch. Default: False
     """
     def __init__(
         self, 
-        max_spectrogram_length: int, 
-        padding_value: int
+        padding_value: int,
+        max_spectrogram_length: int = None, 
+        use_on_batch: bool = False
         ) -> None:
         self.max_spectrogram_length = max_spectrogram_length
         self.padding_value = padding_value
+        self.use_on_batch = use_on_batch
+
+        if not use_on_batch and max_spectrogram_length is None:
+            raise ValueError("max_spectrogram_length must be specified if use_on_batch is False")
 
     def __call__(self, spectrogram: np.ndarray, label: np.ndarray):
+        if self.use_on_batch:
+            max_len = max([len(a) for a in spectrogram])
+            padded_spectrograms = []
+            for spec in spectrogram:
+                padded_spectrogram = np.pad(spec, ((0, max_len - spec.shape[0]), (0,0)), mode="constant", constant_values=self.padding_value)
+                padded_spectrograms.append(padded_spectrogram)
+
+            padded_spectrograms = np.array(padded_spectrograms)
+            label = np.array(label)
+
+            return padded_spectrograms, label
+
         padded_spectrogram = np.pad(spectrogram, ((0, self.max_spectrogram_length - spectrogram.shape[0]),(0,0)), mode="constant", constant_values=self.padding_value)
 
         return padded_spectrogram, label
 
+
 class AudioPadding(Transformer):
     def __init__(self, max_audio_length: int, padding_value: int = 0, use_on_batch: bool = False, limit: bool = False):
         super(AudioPadding, self).__init__()

requirements.txt

@@ -5,5 +5,4 @@ numpy
 opencv-python
 Pillow>=9.4.0
 onnxruntime>=1.15.0  # onnxruntime-gpu for GPU support
-librosa>=0.9.2
 matplotlib