feat: change english_words to set for performance gain (#380)

Author: cragwolfe

CHANGELOG.md

@@ -1,7 +1,9 @@
-## 0.5.5-dev0
+## 0.5.5-dev1
 
 ### Enhancements
 
+* `contains_english_word()`, used heavily in text processing, is 10x faster.
+
 ### Features
 
 * Add `clean_non_ascii_chars` to remove non-ascii characters from unicode string

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.5-dev0"  # pragma: no cover
+__version__ = "0.5.5-dev1"  # pragma: no cover

unstructured/nlp/english_words.py

@@ -1,6 +1,6 @@
 import os
 import pathlib
-from typing import List
+from typing import List, Set
 
 DIRECTORY = pathlib.Path(__file__).parent.resolve()
 # NOTE(robinson) - the list of English words is based on the nlkt.corpus.words corpus
@@ -14,4 +14,4 @@
 
 # NOTE(robinson) - add new words that we want to pass for the English check in here
 ADDITIONAL_ENGLISH_WORDS: List[str] = []
-ENGLISH_WORDS: List[str] = BASE_ENGLISH_WORDS + ADDITIONAL_ENGLISH_WORDS
+ENGLISH_WORDS: Set[str] = set(BASE_ENGLISH_WORDS + ADDITIONAL_ENGLISH_WORDS)