Fix byte pair detokenization of 2d arrays (#423)

Before this fix, the detokenize function would squish everything
down into a single string. So it would not preserve the structure of
what was passed in.

Author: mattdangerw

keras_nlp/tokenizers/byte_pair_tokenizer.py

@@ -516,16 +516,13 @@ def detokenize(self, inputs):
             inputs = tf.expand_dims(inputs, 0)
 
         unicode_text = tf.strings.reduce_join(
-            self.id_to_token_map.lookup(inputs), axis=1
+            self.id_to_token_map.lookup(inputs), axis=-1
         )
         split_unicode_text = tf.strings.unicode_split(unicode_text, "UTF-8")
         byte_text = tf.strings.reduce_join(
-            self.unicode2byte.lookup(split_unicode_text)
+            self.unicode2byte.lookup(split_unicode_text), axis=-1
         )
 
-        if not scalar_input:
-            byte_text = tf.expand_dims(byte_text, 0)
-
         return byte_text
 
     def _transform_bytes(self, tokens):

keras_nlp/tokenizers/byte_pair_tokenizer_test.py

@@ -57,8 +57,14 @@ def test_tokenize_scalar_input(self):
         encoded = self.tokenizer.tokenize(input_data)
         self.assertAllEqual(encoded, [31876, 4])
 
-    def test_detokenize(self):
-        input_data = ["brown."]
+    def test_detokenize_scalar_input(self):
+        input_data = ["quick brown fox."]
+        encoded = self.tokenizer.tokenize(input_data)
+        decoded = self.tokenizer.detokenize(encoded)
+        self.assertAllEqual(input_data, decoded)
+
+    def test_detokenize_list_input(self):
+        input_data = ["quick brown fox.", "slow black bear."]
         encoded = self.tokenizer.tokenize(input_data)
         decoded = self.tokenizer.detokenize(encoded)
         self.assertAllEqual(input_data, decoded)