Translation transformer tutorial

Author: pythonlessons

Tutorials/09_translation_transformer/test.py

@@ -3,30 +3,26 @@
 try: [tf.config.experimental.set_memory_growth(gpu, True) for gpu in tf.config.experimental.list_physical_devices("GPU")]
 except: pass
 import numpy as np
-import json
-import tensorflow_datasets as tfds
 
 from mltu.tokenizers import CustomTokenizer
 
-examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True, as_supervised=True)
-
 class PtEnTranslator(OnnxInferenceModel):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         self.new_inputs = self.model.get_inputs()
-        self.pt_tokenizer = CustomTokenizer.load(self.metadata["pt_tokenizer"])
-        self.eng_tokenizer = CustomTokenizer.load(self.metadata["eng_tokenizer"])
+        self.tokenizer = CustomTokenizer.load(self.metadata["tokenizer"])
+        self.detokenizer = CustomTokenizer.load(self.metadata["detokenizer"])
         # self.eng_tokenizer = CustomTokenizer.load("Tutorials/09_transformers/eng_tokenizer.json")
         # self.pt_tokenizer = CustomTokenizer.load("Tutorials/09_transformers/pt_tokenizer.json")  
 
     def predict(self, sentence):
-        tokenized_sentence = self.pt_tokenizer.texts_to_sequences([sentence])[0]
-        encoder_input = np.pad(tokenized_sentence, (0, self.pt_tokenizer.max_length - len(tokenized_sentence)), constant_values=0).astype(np.int64)
+        tokenized_sentence = self.tokenizer.texts_to_sequences([sentence])[0]
+        encoder_input = np.pad(tokenized_sentence, (0, self.tokenizer.max_length - len(tokenized_sentence)), constant_values=0).astype(np.int64)
 
-        tokenized_results = [self.eng_tokenizer.start_token_index]
-        for index in range(self.eng_tokenizer.max_length - 1):
-            decoder_input = np.pad(tokenized_results, (0, self.eng_tokenizer.max_length - len(tokenized_results)), constant_values=0).astype(np.int64)
+        tokenized_results = [self.detokenizer.start_token_index]
+        for index in range(self.detokenizer.max_length - 1):
+            decoder_input = np.pad(tokenized_results, (0, self.detokenizer.max_length - len(tokenized_results)), constant_values=0).astype(np.int64)
             input_dict = {
                 self.model._inputs_meta[0].name: np.expand_dims(encoder_input, axis=0),
                 self.model._inputs_meta[1].name: np.expand_dims(decoder_input, axis=0),
@@ -35,24 +31,41 @@ def predict(self, sentence):
             pred_results = np.argmax(preds, axis=2)
             tokenized_results.append(pred_results[0][index])
 
-            if tokenized_results[-1] == self.eng_tokenizer.end_token_index:
+            if tokenized_results[-1] == self.detokenizer.end_token_index:
                 break
 
-        results = self.eng_tokenizer.detokenize([tokenized_results])
+        results = self.detokenizer.detokenize([tokenized_results])
         return results[0]
 
 
-translator = PtEnTranslator("Models/09_translation_transformer/202307101211/model.onnx")
+def read_files(path):
+    with open(path, "r", encoding="utf-8") as f:
+        en_train_dataset = f.read().split("\n")[:-1]
+    return en_train_dataset
+
+# Path to dataset
+en_validation_data_path = "Datasets/en-es/opus.en-es-dev.en"
+es_validation_data_path = "Datasets/en-es/opus.en-es-dev.es"
+
+en_validation_data = read_files(en_validation_data_path)
+es_validation_data = read_files(es_validation_data_path)
+
+# Consider only sentences with length <= 500
+max_lenght = 500
+val_examples = [[es_sentence, en_sentence] for es_sentence, en_sentence in zip(es_validation_data, en_validation_data) if len(es_sentence) <= max_lenght and len(en_sentence) <= max_lenght]
+# es_validation_data, en_validation_data = zip(*val_dataset)
+
+
+
+
 
+translator = PtEnTranslator("Models/09_translation_transformer/202307241748/model.onnx")
 
-train_examples, val_examples = examples['train'], examples['validation']
 
 val_dataset = []
-for pt, en in val_examples:
-    pt_sentence = pt.numpy().decode('utf-8')
-    en_sentence = en.numpy().decode('utf-8')
-    results = translator.predict(pt_sentence)
-    print(en_sentence)
+for es, en in val_examples:
+    results = translator.predict(es)
+    print(en)
     print(results)
     print()
     # val_dataset.append([pt.numpy().decode('utf-8'), en.numpy().decode('utf-8')])

Tutorials/09_translation_transformer/train.py

@@ -136,6 +136,7 @@ def preprocess_inputs(data_batch, label_batch):
         checkpoint, 
         tb_callback, 
         reduceLROnPlat,
-        model2onnx
+        model2onnx,
+        encDecSplitCallback
         ]
     )