Add _string to CharTokenizer (nltk#3156)

Author: tomaarsen

nltk/test/unit/test_tokenize.py

@@ -16,6 +16,7 @@
     sent_tokenize,
     word_tokenize,
 )
+from nltk.tokenize.simple import CharTokenizer
 
 
 def load_stanford_segmenter():
@@ -865,3 +866,21 @@ class ExtLangVars(punkt.PunktLanguageVars):
     )
     def test_sent_tokenize(self, sentences: str, expected: List[str]):
         assert sent_tokenize(sentences) == expected
+
+    def test_string_tokenizer(self) -> None:
+        sentence = "Hello there"
+        tokenizer = CharTokenizer()
+        assert tokenizer.tokenize(sentence) == list(sentence)
+        assert list(tokenizer.span_tokenize(sentence)) == [
+            (0, 1),
+            (1, 2),
+            (2, 3),
+            (3, 4),
+            (4, 5),
+            (5, 6),
+            (6, 7),
+            (7, 8),
+            (8, 9),
+            (9, 10),
+            (10, 11),
+        ]

nltk/tokenize/simple.py

@@ -70,6 +70,8 @@ class CharTokenizer(StringTokenizer):
     is ever required directly, use ``for char in string``.
     """
 
+    _string = None
+
     def tokenize(self, s):
         return list(s)