Merge branch 'feature/sound_to_text'

Author: pythonlessons

README.md

@@ -14,4 +14,7 @@ Each tutorial has its own requirements.txt file for a specific mltu version. As
 
 # Tutorials and Examples:
 1. [Text Recognition With TensorFlow and CTC network](https://pylessons.com/ctc-text-recognition), code in ```Tutorials\01_image_to_word``` folder;
-2. [TensorFlow OCR model for reading Captchas](https://pylessons.com/tensorflow-ocr-captcha), code in ```Tutorials\02_captcha_to_text``` folder;
+2. [TensorFlow OCR model for reading Captchas](https://pylessons.com/tensorflow-ocr-captcha), code in ```Tutorials\02_captcha_to_text``` folder;
+3. [Handwriting words recognition with TensorFlow](https://pylessons.com/handwriting-recognition), code in ```Tutorials\03_handwriting_recognition``` folder;
+4. [Handwritten sentence recognition with TensorFlow](https://pylessons.com/handwritten-sentence-recognition), code in ```Tutorials\04_sentence_recognition``` folder;
+5. [Introduction to speech recognition with TensorFlow](https://pylessons.com/speech-recognition), code in ```Tutorials\05_speech_recognition``` folder;

Tutorials/03_handwriting_recognition/README.md

@@ -11,7 +11,7 @@ def __init__(self):
         self.height = 32
         self.width = 128
         self.max_text_length = 0
-        self.batch_size = 64
-        self.learning_rate = 0.001
+        self.batch_size = 16
+        self.learning_rate = 0.0005
         self.train_epochs = 1000
         self.train_workers = 20

Tutorials/03_handwriting_recognition/configs.py

@@ -26,11 +26,11 @@ def predict(self, image: np.ndarray):
     from tqdm import tqdm
     from mltu.configs import BaseModelConfigs
 
-    configs = BaseModelConfigs.load("Models/03_handwriting_recognition/202212290905/configs.yaml")
+    configs = BaseModelConfigs.load("Models/03_handwriting_recognition/202301111911/configs.yaml")
 
     model = ImageToWordModel(model_path=configs.model_path, char_list=configs.vocab)
 
-    df = pd.read_csv("Models/03_handwriting_recognition/202212290905/val.csv").values.tolist()
+    df = pd.read_csv("Models/03_handwriting_recognition/202301111911/val.csv").values.tolist()
 
     accum_cer = []
     for image_path, label in tqdm(df):
@@ -43,4 +43,10 @@ def predict(self, image: np.ndarray):
 
         accum_cer.append(cer)
 
+        # resize by 4x
+        image = cv2.resize(image, (image.shape[1] * 4, image.shape[0] * 4))
+        cv2.imshow("Image", image)
+        cv2.waitKey(0)
+        cv2.destroyAllWindows()
+
     print(f"Average CER: {np.average(accum_cer)}")

Tutorials/03_handwriting_recognition/inferenceModel.py

@@ -108,7 +108,6 @@ def download_and_unzip(url, extract_to='Datasets', chunk_size=1024*1024):
     optimizer=tf.keras.optimizers.Adam(learning_rate=configs.learning_rate), 
     loss=CTCloss(), 
     metrics=[CWERMetric(padding_token=len(configs.vocab))],
-    run_eagerly=False
 )
 model.summary(line_length=110)
 

Tutorials/03_handwriting_recognition/train.py

@@ -0,0 +1,10 @@
+# Handwritten sentence recognition with TensorFlow
+## Unlock the power of handwritten sentence recognition with TensorFlow and CTC loss. From digitizing notes to transcribing historical documents and automating exam grading
+
+
+## **Detailed tutorial**:
+## [Handwritten sentence recognition with TensorFlow](https://pylessons.com/handwritten-sentence-recognition)
+
+<p align="center">
+  <img src="https://pylessons.com/media/Tutorials/TensorFlow-CAPTCHA-solver/handwritten-sentence-recognition/handwritten-sentence-recognition_pbLia4E.png">
+</p>

Tutorials/04_sentence_recognition/README.md

@@ -12,6 +12,6 @@ def __init__(self):
         self.width = 1408
         self.max_text_length = 0
         self.batch_size = 32
-        self.learning_rate = 0.001
+        self.learning_rate = 0.0005
         self.train_epochs = 1000
         self.train_workers = 20

Tutorials/04_sentence_recognition/configs.py

@@ -4,14 +4,15 @@
 
 from mltu.inferenceModel import OnnxInferenceModel
 from mltu.utils.text_utils import ctc_decoder, get_cer, get_wer
+from mltu.transformers import ImageResizer
 
 class ImageToWordModel(OnnxInferenceModel):
     def __init__(self, char_list: typing.Union[str, list], *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.char_list = char_list
 
     def predict(self, image: np.ndarray):
-        image = cv2.resize(image, self.input_shape[:2][::-1])
+        image = ImageResizer.resize_maintaining_aspect_ratio(image, *self.input_shape[:2][::-1])
 
         image_pred = np.expand_dims(image, axis=0).astype(np.float32)
 
@@ -26,11 +27,11 @@ def predict(self, image: np.ndarray):
     from tqdm import tqdm
     from mltu.configs import BaseModelConfigs
 
-    configs = BaseModelConfigs.load("Models/04_sentence_recognition/202301060816/configs.yaml")
+    configs = BaseModelConfigs.load("Models/04_sentence_recognition/202301131202/configs.yaml")
 
     model = ImageToWordModel(model_path=configs.model_path, char_list=configs.vocab)
 
-    df = pd.read_csv("Models/04_sentence_recognition/202301060816/val.csv").values.tolist()
+    df = pd.read_csv("Models/04_sentence_recognition/202301131202/val.csv").values.tolist()
 
     accum_cer, accum_wer = [], []
     for image_path, label in tqdm(df):
@@ -40,9 +41,16 @@ def predict(self, image: np.ndarray):
 
         cer = get_cer(prediction_text, label)
         wer = get_wer(prediction_text, label)
-        print(f"Image: {image_path}; Label: ({label}); Prediction: ({prediction_text}); CER: {cer}; WER: {wer}")
+        print("Image: ", image_path)
+        print("Label:", label)
+        print("Prediction: ", prediction_text)
+        print(f"CER: {cer}; WER: {wer}")
 
         accum_cer.append(cer)
         accum_wer.append(wer)
 
+        cv2.imshow(prediction_text, image)
+        cv2.waitKey(0)
+        cv2.destroyAllWindows()
+
     print(f"Average CER: {np.average(accum_cer)}, Average WER: {np.average(accum_wer)}")

Tutorials/04_sentence_recognition/inferenceModel.py

@@ -18,7 +18,7 @@
 import stow
 from tqdm import tqdm
 
-# Must download and extract datasets manually from https://fki.tic.heia-fr.ch/databases/download-the-iam-handwriting-database
+# Must download and extract datasets manually from https://fki.tic.heia-fr.ch/databases/download-the-iam-handwriting-database to Datasets\IAM_Sentences
 sentences_txt_path = stow.join('Datasets', 'IAM_Sentences', 'ascii', 'sentences.txt')
 sentences_folder_path = stow.join('Datasets', 'IAM_Sentences', 'sentences')
 
@@ -63,7 +63,7 @@
     batch_size=configs.batch_size,
     data_preprocessors=[ImageReader()],
     transformers=[
-        ImageResizer(configs.width, configs.height, keep_aspect_ratio=False),
+        ImageResizer(configs.width, configs.height, keep_aspect_ratio=True),
         LabelIndexer(configs.vocab),
         LabelPadding(max_word_length=configs.max_text_length, padding_value=len(configs.vocab)),
         ],
@@ -102,7 +102,7 @@
 checkpoint = ModelCheckpoint(f"{configs.model_path}/model.h5", monitor='val_CER', verbose=1, save_best_only=True, mode='min')
 trainLogger = TrainLogger(configs.model_path)
 tb_callback = TensorBoard(f'{configs.model_path}/logs', update_freq=1)
-reduceLROnPlat = ReduceLROnPlateau(monitor='val_CER', factor=0.9, min_delta=1e-10, patience=10, verbose=1, mode='auto')
+reduceLROnPlat = ReduceLROnPlateau(monitor='val_CER', factor=0.9, min_delta=1e-10, patience=5, verbose=1, mode='auto')
 model2onnx = Model2onnx(f"{configs.model_path}/model.h5")
 
 # Train the model