Disable cleaning of spaces

Now the outputs match the ones before AutoTokenizer was introduced.

Author: Marcin Kardas

galai/model.py

@@ -306,7 +306,11 @@ def generate(
         if not return_full_text:
             out_tokens = out_tokens[:, input_v.shape[1]:]
         # we keep special tokens such as [START_REF] or <work>
-        decoded = self.tokenizer.batch_decode(out_tokens, skip_special_tokens=False)
+        decoded = self.tokenizer.batch_decode(
+            out_tokens,
+            skip_special_tokens=False,
+            clean_up_tokenization_spaces=False,
+        )
         # so we manually remove </s> and <pad>
         decoded = [
             text.replace(self.tokenizer.eos_token, "").replace(self.tokenizer.pad_token, "")
@@ -431,7 +435,11 @@ def generate_reference(
             )
         # cut-off the prompts
         generated_tokens = out["sequences"][:, prompt_length:]
-        decoded = self.tokenizer.batch_decode(generated_tokens, skip_special_tokens=False)
+        decoded = self.tokenizer.batch_decode(
+            generated_tokens,
+            skip_special_tokens=False,
+            clean_up_tokenization_spaces=False,
+        )
         references = []
         unfinished_generation = False
         for text in decoded:

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 PACKAGE_NAME = 'galai'
-VERSION = "1.1.5"
+VERSION = "1.1.6"
 DESCRIPTION = "API for the GALACTICA model"
 KEYWORDS = "Scientific Intelligence"
 URL = 'https://github.com/paperswithcode/galai'