Add support for Split pretokenizer w/ behavior=removed & invert=false

Author: xenova

src/tokenizers.js

@@ -1518,6 +1518,8 @@ class SplitPreTokenizer extends PreTokenizer {
 
         if (this.config.invert) {
             return text.match(this.pattern) || [];
+        } else if (this.config.behavior?.toLowerCase() === 'removed') {
+            return text.split(this.pattern).filter(x => x);
         } else {
             return regexSplit(text, this.pattern);
         }

tests/models/roberta/tokenization.js

@@ -691,4 +691,14 @@ export const TEST_CONFIG = {
       decoded: "<s> \tH\u00e4LLo!how  \n Are yoU?  </s>",
     },
   },
+
+  // Split tokenizer with behavior="Removed" and invert=false
+  "onnx-community/camembertv2-base": {
+    SIMPLE: {
+      text: BASE_TEST_STRINGS.SIMPLE,
+      tokens: ['How', 'are', 'you', 'doi', '##ng', '?'],
+      ids: [1, 14473, 9556, 10577, 6471, 9274, 38, 2],
+      decoded: "[CLS] How are you doing? [SEP]",
+    }
+  },
 };