update stub for typing

Author: ArthurZucker

bindings/python/py_src/tokenizers/__init__.pyi

@@ -52,6 +52,16 @@ class BPEDecoder(Decoder):
         """
         pass
 
+    @property
+    def suffix(self):
+        """ """
+        pass
+
+    @suffix.setter
+    def suffix(self, value):
+        """ """
+        pass
+
 class ByteFallback(Decoder):
     """
     ByteFallback Decoder
@@ -115,6 +125,16 @@ class CTC(Decoder):
     def __init__(self, pad_token="<pad>", word_delimiter_token="|", cleanup=True):
         pass
 
+    @property
+    def cleanup(self):
+        """ """
+        pass
+
+    @cleanup.setter
+    def cleanup(self, value):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -128,6 +148,26 @@ class CTC(Decoder):
         """
         pass
 
+    @property
+    def pad_token(self):
+        """ """
+        pass
+
+    @pad_token.setter
+    def pad_token(self, value):
+        """ """
+        pass
+
+    @property
+    def word_delimiter_token(self):
+        """ """
+        pass
+
+    @word_delimiter_token.setter
+    def word_delimiter_token(self, value):
+        """ """
+        pass
+
 class Fuse(Decoder):
     """
     Fuse Decoder
@@ -182,6 +222,36 @@ class Metaspace(Decoder):
         """
         pass
 
+    @property
+    def prepend_scheme(self):
+        """ """
+        pass
+
+    @prepend_scheme.setter
+    def prepend_scheme(self, value):
+        """ """
+        pass
+
+    @property
+    def replacement(self):
+        """ """
+        pass
+
+    @replacement.setter
+    def replacement(self, value):
+        """ """
+        pass
+
+    @property
+    def split(self):
+        """ """
+        pass
+
+    @split.setter
+    def split(self, value):
+        """ """
+        pass
+
 class Replace(Decoder):
     """
     Replace Decoder
@@ -237,6 +307,16 @@ class Strip(Decoder):
     def __init__(self, content, left=0, right=0):
         pass
 
+    @property
+    def content(self):
+        """ """
+        pass
+
+    @content.setter
+    def content(self, value):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -250,6 +330,26 @@ class Strip(Decoder):
         """
         pass
 
+    @property
+    def start(self):
+        """ """
+        pass
+
+    @start.setter
+    def start(self, value):
+        """ """
+        pass
+
+    @property
+    def stop(self):
+        """ """
+        pass
+
+    @stop.setter
+    def stop(self, value):
+        """ """
+        pass
+
 class WordPiece(Decoder):
     """
     WordPiece Decoder
@@ -265,6 +365,16 @@ class WordPiece(Decoder):
     def __init__(self, prefix="##", cleanup=True):
         pass
 
+    @property
+    def cleanup(self):
+        """ """
+        pass
+
+    @cleanup.setter
+    def cleanup(self, value):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -277,3 +387,13 @@ class WordPiece(Decoder):
             :obj:`str`: The decoded string
         """
         pass
+
+    @property
+    def prefix(self):
+        """ """
+        pass
+
+    @prefix.setter
+    def prefix(self, value):
+        """ """
+        pass

bindings/python/py_src/tokenizers/decoders/__init__.pyi

@@ -131,6 +131,46 @@ class BPE(Model):
     ):
         pass
 
+    @property
+    def byte_fallback(self):
+        """ """
+        pass
+
+    @byte_fallback.setter
+    def byte_fallback(self, value):
+        """ """
+        pass
+
+    @property
+    def continuing_subword_prefix(self):
+        """ """
+        pass
+
+    @continuing_subword_prefix.setter
+    def continuing_subword_prefix(self, value):
+        """ """
+        pass
+
+    @property
+    def dropout(self):
+        """ """
+        pass
+
+    @dropout.setter
+    def dropout(self, value):
+        """ """
+        pass
+
+    @property
+    def end_of_word_suffix(self):
+        """ """
+        pass
+
+    @end_of_word_suffix.setter
+    def end_of_word_suffix(self, value):
+        """ """
+        pass
+
     @staticmethod
     def from_file(cls, vocab, merge, **kwargs):
         """
@@ -157,6 +197,16 @@ class BPE(Model):
         """
         pass
 
+    @property
+    def fuse_unk(self):
+        """ """
+        pass
+
+    @fuse_unk.setter
+    def fuse_unk(self, value):
+        """ """
+        pass
+
     def get_trainer(self):
         """
         Get the associated :class:`~tokenizers.trainers.Trainer`
@@ -182,6 +232,16 @@ class BPE(Model):
         """
         pass
 
+    @property
+    def ignore_merges(self):
+        """ """
+        pass
+
+    @ignore_merges.setter
+    def ignore_merges(self, value):
+        """ """
+        pass
+
     @staticmethod
     def read_file(self, vocab, merges):
         """
@@ -250,6 +310,16 @@ class BPE(Model):
         """
         pass
 
+    @property
+    def unk_token(self):
+        """ """
+        pass
+
+    @unk_token.setter
+    def unk_token(self, value):
+        """ """
+        pass
+
 class Unigram(Model):
     """
     An implementation of the Unigram algorithm
@@ -460,6 +530,16 @@ class WordLevel(Model):
         """
         pass
 
+    @property
+    def unk_token(self):
+        """ """
+        pass
+
+    @unk_token.setter
+    def unk_token(self, value):
+        """ """
+        pass
+
 class WordPiece(Model):
     """
     An implementation of the WordPiece algorithm
@@ -477,6 +557,16 @@ class WordPiece(Model):
     def __init__(self, vocab, unk_token, max_input_chars_per_word):
         pass
 
+    @property
+    def continuing_subword_prefix(self):
+        """ """
+        pass
+
+    @continuing_subword_prefix.setter
+    def continuing_subword_prefix(self, value):
+        """ """
+        pass
+
     @staticmethod
     def from_file(vocab, **kwargs):
         """
@@ -525,6 +615,16 @@ class WordPiece(Model):
         """
         pass
 
+    @property
+    def max_input_chars_per_word(self):
+        """ """
+        pass
+
+    @max_input_chars_per_word.setter
+    def max_input_chars_per_word(self, value):
+        """ """
+        pass
+
     @staticmethod
     def read_file(vocab):
         """
@@ -589,3 +689,13 @@ class WordPiece(Model):
             A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
         """
         pass
+
+    @property
+    def unk_token(self):
+        """ """
+        pass
+
+    @unk_token.setter
+    def unk_token(self, value):
+        """ """
+        pass