Merge branch 'develop'

Author: pythonlessons

Tutorials/09_translation_transformer/requirements.txt

@@ -1 +1,3 @@
-beautifulsoup4
+beautifulsoup4
+tf2onnx==1.14.0
+onnx==1.12.0

Tutorials/09_translation_transformer/test.py

@@ -1,4 +1,5 @@
 import numpy as np
+import time
 
 from mltu.tokenizers import CustomTokenizer
 from mltu.inferenceModel import OnnxInferenceModel
@@ -12,6 +13,7 @@ def __init__(self, *args, **kwargs):
         self.detokenizer = CustomTokenizer.load(self.metadata["detokenizer"])
 
     def predict(self, sentence):
+        start = time.time()
         tokenized_sentence = self.tokenizer.texts_to_sequences([sentence])[0]
         encoder_input = np.pad(tokenized_sentence, (0, self.tokenizer.max_length - len(tokenized_sentence)), constant_values=0).astype(np.int64)
 
@@ -30,8 +32,7 @@ def predict(self, sentence):
                 break
 
         results = self.detokenizer.detokenize([tokenized_results])
-        return results[0]
-
+        return results[0], time.time() - start
 
 def read_files(path):
     with open(path, "r", encoding="utf-8") as f:
@@ -49,11 +50,12 @@ def read_files(path):
 max_lenght = 500
 val_examples = [[es_sentence, en_sentence] for es_sentence, en_sentence in zip(es_validation_data, en_validation_data) if len(es_sentence) <= max_lenght and len(en_sentence) <= max_lenght]
 
-translator = PtEnTranslator("Models/09_translation_transformer/202307241748/model.onnx")
+translator = PtEnTranslator("Models/09_translation_transformer/202308241514/model.onnx")
 
 val_dataset = []
 for es, en in val_examples:
-    results = translator.predict(es)
-    print(en)
+    results, duration = translator.predict(es)
+    print(en.lower())
     print(results)
+    print(duration)
     print()

Tutorials/09_translation_transformer/train.py

@@ -123,6 +123,8 @@ def preprocess_inputs(data_batch, label_batch):
 model2onnx = Model2onnx(f"{configs.model_path}/model.h5", metadata={"tokenizer": tokenizer.dict(), "detokenizer": detokenizer.dict()}, save_on_epoch_end=False)
 encDecSplitCallback = EncDecSplitCallback(configs.model_path, encoder_metadata={"tokenizer": tokenizer.dict()}, decoder_metadata={"detokenizer": detokenizer.dict()})
 
+configs.save()
+
 # Train the model
 transformer.fit(
     train_dataProvider,