Corpus - automated input/output summaries

Author: PrimozGodec

orangecontrib/text/corpus.py

@@ -9,6 +9,7 @@
 import numpy as np
 import scipy.sparse as sp
 from gensim import corpora
+from orangewidget.utils.signals import summarize, PartialSummary
 
 from Orange.data import (
     Variable,
@@ -657,3 +658,20 @@ def arrays_equal(a, b):
                 np.array_equal(self.pos_tags, other.pos_tags) and
                 self.domain == other.domain and
                 self.ngram_range == other.ngram_range)
+
+
+@summarize.register(Corpus)
+def summarize_(corpus: Corpus) -> PartialSummary:
+    """
+    Provides automated input and output summaries for Corpus
+    """
+    table_summary = summarize.dispatch(Table)(corpus)
+    extras = (
+        (
+            f"<br/><nobr>Total tokens: {sum(map(len, corpus.tokens))}, "
+            f"Total types: {len(corpus.dictionary)}</nobr>"
+        )
+        if corpus.has_tokens()
+        else "<br/><nobr>Corpus is not preprocessed</nobr>"
+    )
+    return PartialSummary(table_summary.summary, table_summary.details + extras)

orangecontrib/text/tests/test_corpus.py

@@ -4,6 +4,8 @@
 
 import numpy as np
 from numpy.testing import assert_array_equal
+from orangecontrib.text.preprocess import RegexpTokenizer
+from orangewidget.utils.signals import Input, summarize
 from scipy.sparse import csr_matrix, issparse
 
 from Orange.data import Table, DiscreteVariable, StringVariable, Domain, ContinuousVariable
@@ -629,5 +631,40 @@ def test_pickle_corpus(self):
         pickle.dumps(c)
 
 
+class TestCorpusSummaries(unittest.TestCase):
+    def test_corpus_not_preprocessed(self):
+        """Check if details part of the summary is formatted correctly"""
+        corpus = Corpus.from_file("book-excerpts")
+
+        n_features = len(corpus.domain.variables) + len(corpus.domain.metas)
+        details = (
+            f"<nobr>{len(corpus)} instances, {n_features} variables</nobr><br/>"
+            f"<nobr>Features: — (no missing values)</nobr><br/>"
+            f"<nobr>Target: categorical</nobr><br/>"
+            f"<nobr>Metas: string</nobr><br/>"
+            f"<nobr>Corpus is not preprocessed</nobr>"
+        )
+        table_summary = summarize.dispatch(Corpus)(corpus)
+        self.assertEqual(140, table_summary.summary)
+        self.assertEqual(details, table_summary.details)
+
+    def test_corpus_preprocessed(self):
+        """Check if details part of the summary is formatted correctly"""
+        corpus = Corpus.from_file("book-excerpts")
+        corpus = RegexpTokenizer()(corpus)
+
+        n_features = len(corpus.domain.variables) + len(corpus.domain.metas)
+        details = (
+            f"<nobr>{len(corpus)} instances, {n_features} variables</nobr><br/>"
+            f"<nobr>Features: — (no missing values)</nobr><br/>"
+            f"<nobr>Target: categorical</nobr><br/>"
+            f"<nobr>Metas: string</nobr><br/>"
+            f"<nobr>Total tokens: 128020, Total types: 11712</nobr>"
+        )
+        table_summary = summarize.dispatch(Corpus)(corpus)
+        self.assertEqual(140, table_summary.summary)
+        self.assertEqual(details, table_summary.details)
+
+
 if __name__ == "__main__":
     unittest.main()