Update falcon tokenizer (#344)

* Update generate_tests.py

* Do not add token types for `FalconTokenizer`

Author: xenova

src/tokenizers.js

@@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { }
 export class XLMRobertaTokenizer extends PreTrainedTokenizer { }
 export class MPNetTokenizer extends PreTrainedTokenizer { }
 
-export class FalconTokenizer extends PreTrainedTokenizer {
-    /** @type {add_token_types} */
-    prepare_model_inputs(inputs) {
-        return add_token_types(inputs);
-    }
-}
+export class FalconTokenizer extends PreTrainedTokenizer { }
 
 export class GPTNeoXTokenizer extends PreTrainedTokenizer { }
 

tests/generate_tests.py

@@ -10,7 +10,7 @@
 
 # List of tokenizers where the model isn't yet supported, but the tokenizer is
 ADDITIONAL_TOKENIZERS_TO_TEST = {
-    'RefinedWebModel': [
+    'falcon': [
         'tiiuae/falcon-7b',
     ],
     "llama": [