SpamScorer: normalise whitespace in comparable form

Collapse Unicode whitespace runs to a single space and trim in SpamScorer::RichText#to_comparable_form to improve SpammyPhrase matching across newlines/tabs.

Author: Firefishy

lib/spam_scorer/rich_text.rb

@@ -38,7 +38,7 @@ def score
     attr_reader :text
 
     def to_comparable_form(str)
-      str.downcase(:fold).unicode_normalize(:nfkc)
+      str.downcase(:fold).unicode_normalize(:nfkc).gsub(/\s+/u, " ").strip
     end
   end
 end

test/lib/spam_scorer_test.rb

@@ -44,4 +44,19 @@ def test_spammy_phrases
     scorer = SpamScorer.new_from_rich_text(r)
     assert_equal 160, scorer.score.round
   end
+
+  def test_to_comparable_form_collapses_unicode_whitespace_and_trims
+    r = RichText.new("text", "x")
+    scorer = SpamScorer.new_from_rich_text(r)
+
+    input = "  A\u00A0\tB\n\nC  "
+    assert_equal "a b c", scorer.send(:to_comparable_form, input)
+  end
+
+  def test_spammy_phrase_can_match_across_newlines_after_normalization
+    create(:spammy_phrase, :phrase => "foo bar")
+    r = RichText.new("markdown", "foo\nbar")
+    scorer = SpamScorer.new_from_rich_text(r)
+    assert_equal 40, scorer.score.round
+  end
 end