v0.10.0 (#492)

* Use any dimensionality reduction technique instead of UMAP
* Use any clustering technique instead of HDBSCAN
* Add a CountVectorizer page with tips and tricks on how to create topic representations that fit your use case
* Added pages on how to use other dimensionality reduction and clustering algorithms
* Additional instructions on how to reduce outliers in the FAQ
* Fixed `None` being returned for probabilities when transforming unseen documents
* Replaced all instances of `arg:` with `Arguments:` for consistency
* Before saving a fitted BERTopic instance, we remove the stopwords in the fitted CountVectorizer model as it can get quite large due to the number of words that end in stopwords if `min_df` is set to a value larger than 1
* Set `"hdbscan>=0.8.28"` to prevent numpy issues
* Update gensim dependency to `>=4.0.0` (#371)
* Fix topic 0 not appearing in visualizations (#472)
* Fix #506
* Fix #429

Author: Maarten Grootendorst

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 160

.gitignore

@@ -75,3 +75,4 @@ venv.bak/
 
 .idea
 .idea/
+.vscode

bertopic/__init__.py

@@ -1,6 +1,6 @@
 from bertopic._bertopic import BERTopic
 
-__version__ = "0.9.4"
+__version__ = "0.10.0"
 
 __all__ = [
     "BERTopic",

bertopic/_bertopic.py

@@ -53,10 +53,13 @@ def check_embeddings_shape(embeddings, docs):
 
 def check_is_fitted(model):
     """ Checks if the model was fitted by verifying the presence of self.matches
+
     Arguments:
         model: BERTopic instance for which the check is performed.
+
     Returns:
         None
+
     Raises:
         ValueError: If the matches were not found.
     """

bertopic/_utils.py

@@ -48,8 +48,8 @@ def embed(self,
             Document/words embeddings with shape (n, m) with `n` documents/words
             that each have an embeddings size of `m`
         """
-        vector_shape = self.embedding_model.word_vec(list(self.embedding_model.vocab.keys())[0]).shape
-        empty_vector = np.zeros(vector_shape[0])
+        vector_shape = self.embedding_model.get_vector(list(self.embedding_model.index_to_key)[0]).shape[0]
+        empty_vector = np.zeros(vector_shape)
 
         embeddings = []
         for doc in tqdm(documents, disable=not verbose, position=0, leave=True):
@@ -58,7 +58,7 @@ def embed(self,
             # Extract word embeddings
             for word in doc.split(" "):
                 try:
-                    word_embedding = self.embedding_model.word_vec(word)
+                    word_embedding = self.embedding_model.get_vector(word)
                     doc_embedding.append(word_embedding)
                 except KeyError:
                     doc_embedding.append(empty_vector)

bertopic/backend/_gensim.py

@@ -46,12 +46,14 @@ def visualize_barchart(topic_model,
     colors = itertools.cycle(["#D55E00", "#0072B2", "#CC79A7", "#E69F00", "#56B4E9", "#009E73", "#F0E442"])
 
     # Select topics based on top_n and topics args
+    freq_df = topic_model.get_topic_freq()
+    freq_df = freq_df.loc[freq_df.Topic != -1, :]
     if topics is not None:
         topics = list(topics)
     elif top_n_topics is not None:
-        topics = topic_model.get_topic_freq().Topic.to_list()[1:top_n_topics + 1]
+        topics = sorted(freq_df.Topic.to_list()[:top_n_topics])
     else:
-        topics = topic_model.get_topic_freq().Topic.to_list()[1:7]
+        topics = sorted(freq_df.Topic.to_list()[0:6])
 
     # Initialize figure
     subplot_titles = [f"Topic {topic}" for topic in topics]

bertopic/plotting/_barchart.py

@@ -56,12 +56,14 @@ def visualize_heatmap(topic_model,
         embeddings = topic_model.c_tf_idf
 
     # Select topics based on top_n and topics args
+    freq_df = topic_model.get_topic_freq()
+    freq_df = freq_df.loc[freq_df.Topic != -1, :]
     if topics is not None:
         topics = list(topics)
     elif top_n_topics is not None:
-        topics = sorted(topic_model.get_topic_freq().Topic.to_list()[1:top_n_topics + 1])
+        topics = sorted(freq_df.Topic.to_list()[:top_n_topics])
     else:
-        topics = sorted(list(topic_model.get_topics().keys()))
+        topics = sorted(freq_df.Topic.to_list())
 
     # Order heatmap by similar clusters of topics
     if n_clusters:

bertopic/plotting/_heatmap.py

@@ -57,12 +57,14 @@ def visualize_hierarchy(topic_model,
         embeddings = topic_model.c_tf_idf
 
     # Select topics based on top_n and topics args
+    freq_df = topic_model.get_topic_freq()
+    freq_df = freq_df.loc[freq_df.Topic != -1, :]
     if topics is not None:
-        topics = sorted(list(topics))
+        topics = list(topics)
     elif top_n_topics is not None:
-        topics = sorted(topic_model.get_topic_freq().Topic.to_list()[1:top_n_topics + 1])
+        topics = sorted(freq_df.Topic.to_list()[:top_n_topics])
     else:
-        topics = sorted(list(topic_model.get_topics().keys()))
+        topics = sorted(freq_df.Topic.to_list())
 
     # Select embeddings
     all_topics = sorted(list(topic_model.get_topics().keys()))

bertopic/plotting/_hierarchy.py

@@ -43,12 +43,14 @@ def visualize_topics(topic_model,
     style="width:1000px; height: 680px; border: 0px;""></iframe>
     """
     # Select topics based on top_n and topics args
+    freq_df = topic_model.get_topic_freq()
+    freq_df = freq_df.loc[freq_df.Topic != -1, :]
     if topics is not None:
         topics = list(topics)
     elif top_n_topics is not None:
-        topics = sorted(topic_model.get_topic_freq().Topic.to_list()[1:top_n_topics + 1])
+        topics = sorted(freq_df.Topic.to_list()[:top_n_topics])
     else:
-        topics = sorted(list(topic_model.get_topics().keys()))
+        topics = sorted(freq_df.Topic.to_list())
 
     # Extract topic words and their frequencies
     topic_list = sorted(topics)