add test to check fit and rename bag-of-words transformer to count transformer

Author: pazzo83

README.md

@@ -89,19 +89,19 @@ BM25Transformer(
 ```
 Please see [http://ethen8181.github.io/machine-learning/search/bm25_intro.html](http://ethen8181.github.io/machine-learning/search/bm25_intro.html) for more details about how these parameters affect the matrix that is generated.
 
-## Bag-of-Words Transformer
+## Count Transformer
 The `MLJText` package also offers a way to represent documents using the simpler bag-of-words representation. This returns a document-term matrix (as you would get in `TextAnalysis`) that consists of the count for every word in the corpus for each document in the corpus.
 
 ### Usage
 ```julia
 using MLJ, MLJText, TextAnalysis
 
 docs = ["Hi my name is Sam.", "How are you today?"]
-bagofwords_transformer = BagOfWordsTransformer()
-mach = machine(bagofwords_transformer, tokenize.(docs))
+count_transformer = CountTransformer()
+mach = machine(count_transformer, tokenize.(docs))
 MLJ.fit!(mach)
 
-bagofwords_mat = transform(mach, tokenize.(docs))
+count_mat = transform(mach, tokenize.(docs))
 ```
 
 The resulting matrix looks like:

src/MLJText.jl

@@ -21,9 +21,9 @@ include("scitypes.jl")
 include("utils.jl")
 include("abstract_text_transformer.jl")
 include("tfidf_transformer.jl")
-include("bagofwords_transformer.jl")
+include("count_transformer.jl")
 include("bm25_transformer.jl")
 
-export TfidfTransformer, BM25Transformer, BagOfWordsTransformer
+export TfidfTransformer, BM25Transformer, CountTransformer
 
 end # module

src/abstract_text_transformer.jl

@@ -34,17 +34,17 @@ function _fit(transformer::AbstractTextTransformer, verbosity::Int, X::Corpus)
     if transformer.max_doc_freq < 1 || transformer.min_doc_freq > 0
         high = round(Int, transformer.max_doc_freq * n)
         low = round(Int, transformer.min_doc_freq * n)
-        new_dtm, vocab = limit_features(dtm_matrix, high, low)
+        new_doc_term_mat, vocab = limit_features(dtm_matrix, high, low)
     else
-        new_dtm = dtm_matrix.dtm
+        new_doc_term_mat = dtm_matrix.dtm
         vocab = dtm_matrix.terms
     end
 
     # calculate IDF
-    idf = compute_idf(transformer.smooth_idf, new_dtm)
+    idf = compute_idf(transformer.smooth_idf, new_doc_term_mat)
 
     # prepare result
-    fitresult = get_result(transformer, idf, vocab, new_dtm)
+    fitresult = get_result(transformer, idf, vocab, new_doc_term_mat)
     cache = nothing
 
     return fitresult, cache, NamedTuple()

src/bm25_transformer.jl

@@ -106,9 +106,9 @@ end
 function _transform(transformer::BM25Transformer, 
                     result::BMI25TransformerResult,
                     v::Corpus)
-    dtm_matrix = build_dtm(v, result.vocab)
-    bm25 = similar(dtm_matrix.dtm, eltype(result.idf_vector))
-    build_bm25!(dtm_matrix.dtm, bm25, result.idf_vector, result.mean_words_in_docs; κ=transformer.κ, β=transformer.β)
+    doc_terms = build_dtm(v, result.vocab)
+    bm25 = similar(doc_terms.dtm, eltype(result.idf_vector))
+    build_bm25!(doc_terms.dtm, bm25, result.idf_vector, result.mean_words_in_docs; κ=transformer.κ, β=transformer.β)
 
     # here we return the `adjoint` of our sparse matrix to conform to 
     # the `n x p` dimensions throughout MLJ

src/bagofwords_transformer.jl renamed to src/count_transformer.jl

@@ -1,7 +1,8 @@
 """
-    BagOfWordsTransformer()
+CountTransformer()
 
-Convert a collection of raw documents to matrix representing a bag-of-words structure.
+Convert a collection of raw documents to matrix representing a bag-of-words structure from
+word counts.
 
 Essentially, a bag-of-words approach to representing documents in a matrix is comprised of
 a count of every word in the document corpus/collection for every document. This is a simple
@@ -21,64 +22,64 @@ will be removed. Similarly, the `min_doc_freq` parameter restricts terms in the
 other direction. A value of 0.01 means that only terms that are at least in 1% of
 documents will be included.
 """
-mutable struct BagOfWordsTransformer <: AbstractTextTransformer
+mutable struct CountTransformer <: AbstractTextTransformer
     max_doc_freq::Float64
     min_doc_freq::Float64
 end
 
-function BagOfWordsTransformer(; max_doc_freq::Float64 = 1.0, min_doc_freq::Float64 = 0.0)    
-    transformer = BagOfWordsTransformer(max_doc_freq, min_doc_freq)
+function CountTransformer(; max_doc_freq::Float64 = 1.0, min_doc_freq::Float64 = 0.0)    
+    transformer = CountTransformer(max_doc_freq, min_doc_freq)
     message = MMI.clean!(transformer)
     isempty(message) || @warn message
     return transformer
 end
 
-struct BagOfWordsTransformerResult
+struct CountTransformerResult
     vocab::Vector{String}
 end
 
-function _fit(transformer::BagOfWordsTransformer, verbosity::Int, X::Corpus)
+function _fit(transformer::CountTransformer, verbosity::Int, X::Corpus)
     # process corpus vocab
     update_lexicon!(X)    
 
     # calculate min and max doc freq limits
     if transformer.max_doc_freq < 1 || transformer.min_doc_freq > 0
         # we need to build out the DTM
-        dtm_matrix = build_dtm(X)
-        n = size(dtm_matrix.dtm, 2) # docs are columns
+        doc_terms = build_dtm(X)
+        n = size(doc_terms.dtm, 2) # docs are columns
         high = round(Int, transformer.max_doc_freq * n)
         low = round(Int, transformer.min_doc_freq * n)
-        _, vocab = limit_features(dtm_matrix, high, low)
+        _, vocab = limit_features(doc_terms, high, low)
     else
         vocab = sort(collect(keys(lexicon(X))))
     end
 
     # prepare result
-    fitresult = BagOfWordsTransformerResult(vocab)
+    fitresult = CountTransformerResult(vocab)
     cache = nothing
 
     return fitresult, cache, NamedTuple()
 end
 
-function _transform(::BagOfWordsTransformer, 
-                    result::BagOfWordsTransformerResult,
+function _transform(::CountTransformer, 
+                    result::CountTransformerResult,
                     v::Corpus)
-    dtm_matrix = build_dtm(v, result.vocab)
+    doc_terms = build_dtm(v, result.vocab)
 
     # here we return the `adjoint` of our sparse matrix to conform to 
     # the `n x p` dimensions throughout MLJ
-    return adjoint(dtm_matrix.dtm)
+    return adjoint(doc_terms.dtm)
 end
 
 # for returning user-friendly form of the learned parameters:
-function MMI.fitted_params(::BagOfWordsTransformer, fitresult::BagOfWordsTransformerResult)
+function MMI.fitted_params(::CountTransformer, fitresult::CountTransformerResult)
     vocab = fitresult.vocab
     return (vocab = vocab,)
 end
 
 ## META DATA
 
-MMI.metadata_pkg(BagOfWordsTransformer,
+MMI.metadata_pkg(CountTransformer,
              name="$PKG",
              uuid="7876af07-990d-54b4-ab0e-23690620f79a",
              url="https://github.com/JuliaAI/MLJText.jl",
@@ -87,13 +88,13 @@ MMI.metadata_pkg(BagOfWordsTransformer,
              is_wrapper=false
 )
 
-MMI.metadata_model(BagOfWordsTransformer,
+MMI.metadata_model(CountTransformer,
                input_scitype = Union{
                    AbstractVector{<:AbstractVector{STB.Textual}},
                    AbstractVector{<:STB.Multiset{<:ScientificNGram}},
                    AbstractVector{<:STB.Multiset{STB.Textual}}
                    },
                output_scitype = AbstractMatrix{STB.Continuous},
-               docstring = "Build Bag-of-Words matrix for corpus of documents",
-               path = "MLJText.BagOfWordsTransformer"
+               docstring = "Build Bag-of-Words matrix for corpus of documents based on word counts",
+               path = "MLJText.CountTransformer"
                )

src/tfidf_transformer.jl

@@ -90,9 +90,9 @@ end
 function _transform(::TfidfTransformer, 
                     result::TfidfTransformerResult,
                     v::Corpus)
-    dtm_matrix = build_dtm(v, result.vocab)
-    tfidf = similar(dtm_matrix.dtm, eltype(result.idf_vector))
-    build_tfidf!(dtm_matrix.dtm, tfidf, result.idf_vector)
+    doc_terms = build_dtm(v, result.vocab)
+    tfidf = similar(doc_terms.dtm, eltype(result.idf_vector))
+    build_tfidf!(doc_terms.dtm, tfidf, result.idf_vector)
 
     # here we return the `adjoint` of our sparse matrix to conform to 
     # the `n x p` dimensions throughout MLJ

src/utils.jl

@@ -1,7 +1,7 @@
-function limit_features(doc_term_matrix::DocumentTermMatrix,
+function limit_features(doc_terms::DocumentTermMatrix,
                         high::Int,
                         low::Int)
-    doc_freqs = vec(sum(doc_term_matrix.dtm, dims=2))
+    doc_freqs = vec(sum(doc_terms.dtm, dims=2))
 
     # build mask to restrict terms
     mask = trues(length(doc_freqs))
@@ -12,9 +12,9 @@ function limit_features(doc_term_matrix::DocumentTermMatrix,
         mask .&= (doc_freqs .>= low)
     end
 
-    new_terms = doc_term_matrix.terms[mask]
+    new_terms = doc_terms.terms[mask]
 
-    return (doc_term_matrix.dtm[mask, :], new_terms)
+    return (doc_terms.dtm[mask, :], new_terms)
 end
 
 ## Helper functions to build Corpus ##
@@ -55,11 +55,11 @@ function build_dtm(docs::Corpus, terms::Vector{T}) where {T}
         end
     end
     if length(rows) > 0
-        doc_term_matrix = sparse(rows, columns, values, m, n)
+        doc_term_mat = sparse(rows, columns, values, m, n)
     else
-        doc_term_matrix = spzeros(Int, m, n)
+        doc_term_mat = spzeros(Int, m, n)
     end
-    DocumentTermMatrix(doc_term_matrix, terms, row_indices)
+    DocumentTermMatrix(doc_term_mat, terms, row_indices)
 end
 
 ## General method to calculate IDF vector ##

test/abstract_text_transformer.jl

@@ -13,17 +13,17 @@ using TextAnalysis
     test_tfidf_machine = @test_logs machine(tfidf_transformer, ngram_vec)
     MLJBase.fit!(test_tfidf_machine)
 
-    # train bag_of_words transformer
-    bagofwords_vectorizer = MLJText.BagOfWordsTransformer()
-    test_bow_machine = @test_logs machine(bagofwords_vectorizer, ngram_vec)
-    MLJBase.fit!(test_bow_machine)
+    # train count transformer
+    count_transformer = MLJText.CountTransformer()
+    test_count_machine = @test_logs machine(count_transformer, ngram_vec)
+    MLJBase.fit!(test_count_machine)
 
     # train bm25 transformer
     bm25_transformer = MLJText.BM25Transformer()
     test_bm25_machine = @test_logs machine(bm25_transformer, ngram_vec)
     MLJBase.fit!(test_bm25_machine)
 
-    test_machines = [test_tfidf_machine, test_bow_machine, test_bm25_machine]
+    test_machines = [test_tfidf_machine, test_count_machine, test_bm25_machine]
 
     # test single doc
     test_doc1 = ngrams(NGramDocument("Another sentence ok"))
@@ -60,6 +60,16 @@ using TextAnalysis
         @test sum(test_doc_transform, dims=2)[2] > 0.0
         @test size(test_doc_transform) == (2, 11)
     end
+
+    # test proper fit:
+    # here we are testing to make sure the size of the corpus to be
+    # transformed does not alter the transformation that the model
+    # is doing.
+    for mach = test_machines
+        single_doc_transform = transform(mach, [test_doc2])
+        multiple_doc_transform = transform(mach, [test_doc2, test_doc2])
+        @test single_doc_transform[1, :] == multiple_doc_transform[1, :]
+    end
 end
 
 @testset "bag of words use" begin
@@ -81,18 +91,18 @@ end
     test_tfidf_machine2 = @test_logs machine(tfidf_transformer, [bag])
     MLJBase.fit!(test_tfidf_machine2)
 
-    # train bag_of_words transformer
-    bagofwords_vectorizer = MLJText.BagOfWordsTransformer()
-    test_bow_machine2 = @test_logs machine(bagofwords_vectorizer, [bag])
-    MLJBase.fit!(test_bow_machine2)
+    # train count transformer
+    count_transformer = MLJText.CountTransformer()
+    test_count_machine2 = @test_logs machine(count_transformer, [bag])
+    MLJBase.fit!(test_count_machine2)
 
     # train bm25 transformer
     bm25_transformer = MLJText.BM25Transformer()
     test_bm25_machine2 = @test_logs machine(bm25_transformer, [bag])
     MLJBase.fit!(test_bm25_machine2)
 
     test_doc5 = ["How about a cat in a hat"]
-    for mach = [test_tfidf_machine2, test_bow_machine2, test_bm25_machine2]
+    for mach = [test_tfidf_machine2, test_count_machine2, test_bm25_machine2]
         test_doc_transform = transform(mach, test_doc5)
         @test sum(test_doc_transform, dims=2)[1] > 0.0
         @test size(test_doc_transform) == (1, 8)
@@ -117,9 +127,9 @@ end
     MLJBase.fit!(test_tfidf_machine3)
 
     # train bag_of_words transformer
-    bagofwords_vectorizer = MLJText.BagOfWordsTransformer(max_doc_freq=0.8)
-    test_bow_machine3 = @test_logs machine(bagofwords_vectorizer, ngram_vec)
-    MLJBase.fit!(test_bow_machine3)
+    count_transformer = MLJText.CountTransformer(max_doc_freq=0.8)
+    test_count_machine3 = @test_logs machine(count_transformer, ngram_vec)
+    MLJBase.fit!(test_count_machine3)
 
     # train bm25 transformer
     bm25_transformer = MLJText.BM25Transformer(max_doc_freq=0.8, min_doc_freq=0.2)
@@ -130,9 +140,9 @@ end
     test_doc_transform = transform(test_tfidf_machine3, ngram_vec)
     @test (Vector(vec(sum(test_doc_transform, dims=2))) .> 0.2) == Bool[1, 1, 1, 1, 1, 1]
 
-    test_doc_transform = transform(test_bow_machine3, ngram_vec)
+    test_doc_transform = transform(test_count_machine3, ngram_vec)
     @test Vector(vec(sum(test_doc_transform, dims=2))) == [14, 10, 14, 9, 13, 7]
 
     test_doc_transform = transform(test_bm25_machine3, ngram_vec)
     @test (Vector(vec(sum(test_doc_transform, dims=2))) .> 0.8) == Bool[1, 1, 1, 1, 1, 1]    
-end
+end