Remove warning for duplicates in string vocab (#38)

* remove warning for duplicates in string vocab

* lint

Author: benlebrun

genlm/backend/tokenization/vocab.py

@@ -1,6 +1,5 @@
 """Functions to get and check HuggingFace tokenizer vocabularies"""
 
-import warnings
 from transformers import AutoTokenizer
 
 from genlm.backend.tokenization.bytes import ByteVocabError, get_byte_vocab
@@ -91,13 +90,4 @@ def bytes_to_strs(tokenizer, byte_vocab, byte2str_fallback):
 
         str_vocab.append(token)
 
-    duplicates = {
-        token: indices for token, indices in seen_tokens.items() if len(indices) > 1
-    }
-    if duplicates:
-        warnings.warn(
-            "Duplicate tokens found in string vocabulary. "
-            "This may lead to downstream issues with the string vocabulary; we recommend using the byte vocabulary."
-        )
-
     return str_vocab