Fixed no oov token error in vocab for WordPieceTokenizer (#136)

* Fixed no oov token error in vocab for WordPieceTokenizer

* Raise no oov_token error during explicit checking for WordPieceTokenizer

* Edits

* Fix

Co-authored-by: Matt Watson <[email protected]>

Author: adhadse

keras_nlp/tokenizers/word_piece_tokenizer.py

@@ -198,6 +198,8 @@ def __init__(
                 "Vocabulary must be an file path or list of terms. "
                 f"Received: vocabulary={vocabulary}"
             )
+        if oov_token is None:
+            raise ValueError("`oov_token` cannot be None.")
 
         self.sequence_length = sequence_length
         self.lowercase = lowercase
@@ -207,6 +209,15 @@ def __init__(
         self.suffix_indicator = suffix_indicator
         self.oov_token = oov_token
 
+        if oov_token not in self.vocabulary:
+            raise RuntimeError(
+                f'Cannot find `oov_token="{self.oov_token}"` in the '
+                "vocabulary.\n"
+                "You can either update the vocabulary to include "
+                f'`"{self.oov_token}"`, or pass a different value for '
+                "the `oov_token` argument when creating the tokenizer."
+            )
+
         self._fast_word_piece = tf_text.FastWordpieceTokenizer(
             vocab=self.vocabulary,
             token_out_type=self.compute_dtype,

keras_nlp/tokenizers/word_piece_tokenizer_test.py

@@ -223,3 +223,25 @@ def test_saving(self):
             model(input_data),
             restored_model(input_data),
         )
+
+    def test_no_oov_token_in_vocabulary(self):
+        vocab_data = ["qu", "@@ick", "br", "@@OWN", "fox"]
+        with self.assertRaises(RuntimeError):
+            WordPieceTokenizer(
+                vocabulary=vocab_data,
+            )
+
+        vocab_data = ["@UNK@", "qu", "@@ick", "br", "@@OWN", "fox"]
+        with self.assertRaises(RuntimeError):
+            WordPieceTokenizer(
+                vocabulary=vocab_data,
+            )
+
+        vocab_data = ["UNK", "qu", "@@ick", "br", "@@OWN", "fox"]
+        with self.assertRaises(RuntimeError):
+            WordPieceTokenizer(
+                vocabulary=vocab_data,
+            )
+
+        with self.assertRaises(ValueError):
+            WordPieceTokenizer(vocabulary=vocab_data, oov_token=None)