Corpus - automated input/output summaries

Author: PrimozGodec

orangecontrib/text/corpus.py

@@ -26,6 +26,13 @@
 from orangecontrib.text.vectorization.base import get_unique_names
 from orangecontrib.text.vectorization import BowVectorizer
 
+try:
+    from orangewidget.utils.signals import summarize, PartialSummary
+    # import to check if Table summary is available
+    from Orange.widgets.utils import state_summary
+except ImportError:
+    summarize, PartialSummary = None, None
+
 
 def get_sample_corpora_dir():
     path = os.path.dirname(__file__)
@@ -657,3 +664,23 @@ def arrays_equal(a, b):
                 np.array_equal(self.pos_tags, other.pos_tags) and
                 self.domain == other.domain and
                 self.ngram_range == other.ngram_range)
+
+
+if summarize:
+    # summarize is not available in older versions of orange-widget-base
+    # skip if not available
+    @summarize.register(Corpus)
+    def summarize_(corpus: Corpus) -> PartialSummary:
+        """
+        Provides automated input and output summaries for Corpus
+        """
+        table_summary = summarize.dispatch(Table)(corpus)
+        extras = (
+            (
+                f"<br/><nobr>Total tokens: {sum(map(len, corpus.tokens))}, "
+                f"Total types: {len(corpus.dictionary)}</nobr>"
+            )
+            if corpus.has_tokens()
+            else "<br/><nobr>Corpus is not preprocessed</nobr>"
+        )
+        return PartialSummary(table_summary.summary, table_summary.details + extras)

orangecontrib/text/tests/test_corpus.py

@@ -1,9 +1,11 @@
 import os
 import pickle
 import unittest
+from unittest import skipIf
 
 import numpy as np
 from numpy.testing import assert_array_equal
+from orangecontrib.text.preprocess import RegexpTokenizer
 from scipy.sparse import csr_matrix, issparse
 
 from Orange.data import Table, DiscreteVariable, StringVariable, Domain, ContinuousVariable
@@ -12,6 +14,11 @@
 from orangecontrib.text.corpus import Corpus
 from orangecontrib.text.tag import AveragedPerceptronTagger
 
+try:
+    from orangewidget.utils.signals import summarize
+except ImportError:
+    summarize = None
+
 
 class CorpusTests(unittest.TestCase):
     def setUp(self):
@@ -629,5 +636,41 @@ def test_pickle_corpus(self):
         pickle.dumps(c)
 
 
+@skipIf(summarize is None, "summarize is not available for orange-widget-base<4.13")
+class TestCorpusSummaries(unittest.TestCase):
+    def test_corpus_not_preprocessed(self):
+        """Check if details part of the summary is formatted correctly"""
+        corpus = Corpus.from_file("book-excerpts")
+
+        n_features = len(corpus.domain.variables) + len(corpus.domain.metas)
+        details = (
+            f"<nobr>{len(corpus)} instances, {n_features} variables</nobr><br/>"
+            f"<nobr>Features: — (no missing values)</nobr><br/>"
+            f"<nobr>Target: categorical</nobr><br/>"
+            f"<nobr>Metas: string</nobr><br/>"
+            f"<nobr>Corpus is not preprocessed</nobr>"
+        )
+        summary = summarize.dispatch(Corpus)(corpus)
+        self.assertEqual(140, summary.summary)
+        self.assertEqual(details, summary.details)
+
+    def test_corpus_preprocessed(self):
+        """Check if details part of the summary is formatted correctly"""
+        corpus = Corpus.from_file("book-excerpts")
+        corpus = RegexpTokenizer()(corpus)
+
+        n_features = len(corpus.domain.variables) + len(corpus.domain.metas)
+        details = (
+            f"<nobr>{len(corpus)} instances, {n_features} variables</nobr><br/>"
+            f"<nobr>Features: — (no missing values)</nobr><br/>"
+            f"<nobr>Target: categorical</nobr><br/>"
+            f"<nobr>Metas: string</nobr><br/>"
+            f"<nobr>Total tokens: 128020, Total types: 11712</nobr>"
+        )
+        summary = summarize.dispatch(Corpus)(corpus)
+        self.assertEqual(140, summary.summary)
+        self.assertEqual(details, summary.details)
+
+
 if __name__ == "__main__":
     unittest.main()