Merge branch 'feature/transformers' into develop

Author: pythonlessons

.gitignore

@@ -10,4 +10,6 @@ dist
 !*.md
 
 .idea
-.python-version
+.python-version
+
+test

CHANGELOG.md

@@ -1,3 +1,15 @@
+## [1.1.0] - 2022-08-28
+### Changed
+- Changed `mltu.transformers.SpectrogramPadding` object, to pad spectrogram end with zeros instead of start
+
+### Added
+- Created `Tutorials/09_translation_transformer` tutorial, that shows how to train translation transformer model
+- Created `mltu.tensorflow.tokenizers` module, that contains `CustomTokenizer` for text data
+- Created `mltu.tensorflow.transformer.attention` module, that contains `BaseAttention`, `CrossAttention`, `GlobalSelfAttention` and `CausalSelfAttention` layers
+- Created `mltu.tensorflow.transformer.layers` module, that contains `positional_encoding` function, `PositionalEmbedding`, `FeedForward`, `EncoderLayer`, `DecoderLayer`, `Encoder`, `Decoder` layers and `Transformer` model 
+- Created `mltu.tensorflow.transformer.callbacks` module, that contains `EncDecSplitCallback` callback, to split Transformer model into separate encoder and decoder models
+- Created `mltu.tensorflow.transformer.utils` module, that contains `MaskedLoss` loss and `MaskedAccuracy` metric, used for training Transformer models
+
 ## [1.0.15] - 2022-07-15
 ### Changed
 - Fixed bug in `mltu.dataProvider.DataProvider` to work with `batch_postprocessors`.

README.md

@@ -23,4 +23,5 @@ Each tutorial has its own requirements.txt file for a specific mltu version. As
 5. [Introduction to speech recognition with TensorFlow](https://pylessons.com/speech-recognition), code in ```Tutorials\05_speech_recognition``` folder;
 6. [Introduction to PyTorch in a practical way](https://pylessons.com/pytorch-introduction), code in ```Tutorials\06_pytorch_introduction``` folder;
 7. [Using custom wrapper to simplify PyTorch models training pipeline](https://pylessons.com/pytorch-introduction), code in ```Tutorials\07_pytorch_wrapper``` folder;
-8. [Handwriting words recognition with PyTorch](https://pylessons.com/handwriting-recognition-pytorch), code in ```Tutorials\08_handwriting_recognition_torch``` folder;
+8. [Handwriting words recognition with PyTorch](https://pylessons.com/handwriting-recognition-pytorch), code in ```Tutorials\08_handwriting_recognition_torch``` folder;
+9. [Transformer training with TensorFlow for Translation task](https://pylessons.com/transformers-training), code in ```Tutorials\09_translation_transformer``` folder;

mltu/tensorflow/Tests/test_metrics.py Tests/test_tensorflow_metrics.pymltu/tensorflow/Tests/test_metrics.py renamed to Tests/test_tensorflow_metrics.py

@@ -36,7 +36,7 @@ def predict(self, data: np.ndarray):
         spectrogram = WavReader.get_spectrogram(wav_path, frame_length=configs.frame_length, frame_step=configs.frame_step, fft_length=configs.fft_length)
         # WavReader.plot_raw_audio(wav_path, label)
 
-        padded_spectrogram = np.pad(spectrogram, ((configs.max_spectrogram_length - spectrogram.shape[0], 0),(0,0)), mode="constant", constant_values=0)
+        padded_spectrogram = np.pad(spectrogram, ((0, configs.max_spectrogram_length - spectrogram.shape[0]),(0,0)), mode="constant", constant_values=0)
 
         # WavReader.plot_spectrogram(spectrogram, label)
 

Tutorials/05_sound_to_text/inferenceModel.py

@@ -0,0 +1,10 @@
+# Training TensorFlow Transformer model for Spanish to English translation task
+### In this tutorial, I'll walk through a practical example of Transformer Training for Language Translation tasks from Spanish to the English language
+
+<br><br>
+# **Detailed tutorial**:
+### [Transformer training with TensorFlow for Translation task](https://pylessons.com/transformers-training)
+
+<p align="center">
+    <img src="https://pylessons.com/media/Tutorials/transformers/transformers-training/transformers-training.png">
+</p>

Tutorials/09_translation_transformer/README.md

@@ -0,0 +1,26 @@
+import os
+from datetime import datetime
+
+from mltu.configs import BaseModelConfigs
+
+
+class ModelConfigs(BaseModelConfigs):
+    def __init__(self):
+        super().__init__()
+        self.model_path = os.path.join(
+            "Models/09_translation_transformer",
+            datetime.strftime(datetime.now(), "%Y%m%d%H%M"),
+        )
+        self.num_layers = 4
+        self.d_model = 128
+        self.num_heads = 8
+        self.dff = 512
+        self.dropout_rate = 0.1
+        self.batch_size = 16
+        self.train_epochs = 50
+        # CustomSchedule parameters
+        self.init_lr = 0.00001
+        self.lr_after_warmup = 0.0005
+        self.final_lr = 0.0001
+        self.warmup_epochs = 2
+        self.decay_epochs = 18

Tutorials/09_translation_transformer/configs.py

@@ -0,0 +1,46 @@
+
+import os
+import requests
+from tqdm import tqdm
+from bs4 import BeautifulSoup
+
+# URL to the directory containing the files to be downloaded
+language = "en-es"
+url = f"https://data.statmt.org/opus-100-corpus/v1.0/supervised/{language}/"
+save_directory = f"./Datasets/{language}"
+
+# Create the save directory if it doesn't exist
+os.makedirs(save_directory, exist_ok=True)
+
+# Send a GET request to the URL
+response = requests.get(url)
+
+# Parse the HTML response
+soup = BeautifulSoup(response.content, 'html.parser')
+
+# Find all the anchor tags in the HTML
+links = soup.find_all('a')
+
+# Extract the href attribute from each anchor tag
+file_links = [link['href'] for link in links if '.' in link['href']]
+
+# Download each file
+for file_link in tqdm(file_links):
+    file_url = url + file_link
+    save_path = os.path.join(save_directory, file_link)
+    
+    print(f"Downloading {file_url}")
+    
+    # Send a GET request for the file
+    file_response = requests.get(file_url)
+    if file_response.status_code == 404:
+        print(f"Could not download {file_url}")
+        continue
+    
+    # Save the file to the specified directory
+    with open(save_path, 'wb') as file:
+        file.write(file_response.content)
+    
+    print(f"Saved {file_link}")
+
+print("All files have been downloaded.")

Tutorials/09_translation_transformer/download.py

@@ -0,0 +1,45 @@
+import tensorflow as tf
+
+from mltu.tensorflow.transformer.layers import Encoder, Decoder
+
+def Transformer(
+    input_vocab_size: int, 
+    target_vocab_size: int, 
+    encoder_input_size: int = None,
+    decoder_input_size: int = None,
+    num_layers: int=6, 
+    d_model: int=512, 
+    num_heads: int=8,
+    dff: int=2048,
+    dropout_rate: float=0.1,
+    ) -> tf.keras.Model:
+    """
+    A custom TensorFlow model that implements the Transformer architecture.
+
+    Args:
+        input_vocab_size (int): The size of the input vocabulary.
+        target_vocab_size (int): The size of the target vocabulary.
+        encoder_input_size (int): The size of the encoder input sequence.
+        decoder_input_size (int): The size of the decoder input sequence.
+        num_layers (int): The number of layers in the encoder and decoder.
+        d_model (int): The dimensionality of the model.
+        num_heads (int): The number of heads in the multi-head attention layer.
+        dff (int): The dimensionality of the feed-forward layer.
+        dropout_rate (float): The dropout rate.
+
+    Returns:
+        A TensorFlow Keras model.
+    """
+    inputs = [
+        tf.keras.layers.Input(shape=(encoder_input_size,), dtype=tf.int64), 
+        tf.keras.layers.Input(shape=(decoder_input_size,), dtype=tf.int64)
+        ]
+    
+    encoder_input, decoder_input = inputs
+
+    encoder = Encoder(num_layers=num_layers, d_model=d_model, num_heads=num_heads, dff=dff, vocab_size=input_vocab_size, dropout_rate=dropout_rate)(encoder_input)
+    decoder = Decoder(num_layers=num_layers, d_model=d_model, num_heads=num_heads, dff=dff, vocab_size=target_vocab_size, dropout_rate=dropout_rate)(decoder_input, encoder)
+
+    output = tf.keras.layers.Dense(target_vocab_size)(decoder)
+
+    return tf.keras.Model(inputs=inputs, outputs=output)

Tutorials/09_translation_transformer/model.py

@@ -0,0 +1 @@
+beautifulsoup4