change to previous tokenizer

Author: christinadionysio

src/main/python/systemds/scuro/representations/tfidf.py

@@ -19,17 +19,11 @@
 #
 # -------------------------------------------------------------
 import numpy as np
-from textblob import TextBlob
 
+from sklearn.feature_extraction.text import TfidfVectorizer
 from systemds.scuro.modality.transformed import TransformedModality
 from systemds.scuro.representations.unimodal import UnimodalRepresentation
 from systemds.scuro.representations.utils import save_embeddings
-from gensim import models
-from gensim.corpora import Dictionary
-
-import nltk
-
-nltk.download("punkt_tab")
 
 
 class TfIdf(UnimodalRepresentation):
@@ -43,12 +37,10 @@ def transform(self, modality):
             modality.modality_type, self, modality.metadata
         )
 
-        tokens = [list(TextBlob(s).words) for s in modality.data]
-        dictionary = Dictionary()
-        BoW_corpus = [dictionary.doc2bow(doc, allow_update=True) for doc in tokens]
-        tfidf = models.TfidfModel(BoW_corpus, smartirs="ntc")
-        X = tfidf[BoW_corpus]
-        X = [np.array(x)[:, 1].reshape(1, -1) for x in X]
+        vectorizer = TfidfVectorizer(min_df=self.min_df)
+
+        X = vectorizer.fit_transform(modality.data)
+        X = [np.array(x).reshape(1, -1) for x in X.toarray()]
 
         if self.output_file is not None:
             save_embeddings(X, self.output_file)

src/main/python/systemds/scuro/representations/word2vec.py

@@ -23,7 +23,7 @@
 from systemds.scuro.representations.unimodal import UnimodalRepresentation
 from systemds.scuro.representations.utils import save_embeddings
 from gensim.models import Word2Vec
-from textblob import TextBlob
+from gensim.utils import tokenize
 
 import nltk
 
@@ -51,7 +51,7 @@ def transform(self, modality):
         transformed_modality = TransformedModality(
             modality.modality_type, self, modality.metadata
         )
-        t = [list(TextBlob(s).words) for s in modality.data]
+        t = [list(tokenize(s.lower())) for s in modality.data]
         model = Word2Vec(
             sentences=t,
             vector_size=self.vector_size,
@@ -60,7 +60,7 @@ def transform(self, modality):
         )
         embeddings = []
         for sentences in modality.data:
-            tokens = list(TextBlob(sentences).words)
+            tokens = list(tokenize(sentences.lower()))
             embeddings.append(np.array(get_embedding(tokens, model)).reshape(1, -1))
 
         if self.output_file is not None: