introduce optimization to classifier model

1. weighted token probability
2. adaptive token selection
3. false positive bias

inspired by this post: https://www.paulgraham.com/better.html

Author: ramsayleung

app/services/spam_classifier_service.rb

@@ -47,11 +47,13 @@ def train_only(trained_message)
         @classifier_state.spam_counts[token] = @classifier_state.spam_counts.fetch(token, 0) + 1
         @vocabulary.add(token)
       end
-    else # :ham
+    else # :ham - FALSE POSITIVE BIAS: count ham tokens double
+      # https://www.paulgraham.com/better.html
       @classifier_state.total_ham_messages += 1
-      @classifier_state.total_ham_words += tokens.size
+      @classifier_state.total_ham_words += tokens.size * 2 # Double
+      # count for bias
       tokens.each do |token|
-        @classifier_state.ham_counts[token] = @classifier_state.ham_counts.fetch(token, 0) + 1
+        @classifier_state.ham_counts[token] = @classifier_state.ham_counts.fetch(token, 0) + 2 # Double weight
         @vocabulary.add(token)
       end
     end
@@ -70,49 +72,44 @@ def train_batch(trained_messages)
     end
     @classifier_state.save!
   end
-
   def classify(message_text)
-    # P(Spam|Words) = P(Words|Spam) * P(Spam) / P(Words)
-    # Return false if the model isn't trained enough
     @classifier_state.reload
-    return [ false, 0.0, 0.0 ] if @classifier_state.total_ham_messages == 0 || @classifier_state.total_spam_messages == 0
+    return [ false, 0.0 ] if @classifier_state.total_ham_messages.zero? || @classifier_state.total_spam_messages.zero?
 
-    tokens = tokenize(message_text)
     total_messages = @classifier_state.total_spam_messages + @classifier_state.total_ham_messages
 
-    # Calculate prior probabilities in log space
-    # Use Math.log to resolve numerical underflow problem
-    prob_spam_prior = Math.log(@classifier_state.total_spam_messages.to_f / total_messages)
-    prob_ham_prior = Math.log(@classifier_state.total_ham_messages.to_f / total_messages)
+    # These are the actual priors
+    prob_spam_prior = @classifier_state.total_spam_messages.to_f / total_messages
+    prob_ham_prior = @classifier_state.total_ham_messages.to_f / total_messages
+
+    tokens = tokenize(message_text)
 
-    spam_score = prob_spam_prior
-    ham_score = prob_ham_prior
+    # Pass the priors to the selection method for consistent logic
+    significant_tokens = get_significant_tokens(tokens, prob_spam_prior, prob_ham_prior)
 
-    vocab_size = @classifier_state.vocabulary_size
+    # Start scores with the log of the priors
+    spam_score = Math.log(prob_spam_prior)
+    ham_score = Math.log(prob_ham_prior)
 
-    tokens.each do |token|
-      # Add 1 for Laplace smoothing, Laplace smoothing is tailored to solve zero probability problem
-      spam_count = @classifier_state.spam_counts.fetch(token, 0) + 1
-      spam_score += Math.log(spam_count.to_f / (@classifier_state.total_spam_words + vocab_size))
+    significant_tokens.each do |token|
+      spam_likelihood, ham_likelihood = get_likelihoods(token)
 
-      ham_count = @classifier_state.ham_counts.fetch(token, 0) + 1
-      ham_score += Math.log(ham_count.to_f / (@classifier_state.total_ham_words + vocab_size))
+      spam_score += Math.log(spam_likelihood)
+      ham_score += Math.log(ham_likelihood)
     end
 
     diff = spam_score - ham_score
-    # stable logistic conversion
-    p_spam = if diff.abs > 700
-               diff > 0 ? 1.0 : 0.0
-    else
-               1.0 / (1.0 + Math.exp(-diff))
-    end
+    p_spam = 1.0 / (1.0 + Math.exp(-diff))
 
     confidence_threshold = Rails.application.config.probability_threshold
     is_spam = p_spam >= confidence_threshold
-    Rails.logger.info "classified_result: #{is_spam ? "maybe_spam": "maybe_ham"}, p_spam: #{p_spam}, message_text: #{message_text}"
+
+    Rails.logger.info "classified_result: #{is_spam ? "maybe_spam": "maybe_ham"}, p_spam: #{p_spam.round(4)}, tokens: #{significant_tokens.join(', ')}"
+
     [ is_spam, spam_score, ham_score ]
   end
 
+
   def tokenize(text)
     cleaned_text = clean_text(text)
     # This regex pre-tokenizes the string into 4 groups:
@@ -188,6 +185,50 @@ def pure_numbers?(token)
     token.match?(/^[0-9一二三四五六七八九十百千万亿零]+$/)
   end
 
+  # It correctly calculates P(token|class) for all cases using Laplace smoothing.
+  def get_likelihoods(token)
+    vocab_size = @classifier_state.vocabulary_size
+
+    # For a spam-only word, ham_count is 0, so ham_likelihood will be very small.
+    # This is the correct, mathematically consistent way to handle it.
+    spam_count = @classifier_state.spam_counts.fetch(token, 0)
+    spam_likelihood = (spam_count + 1.0) / (@classifier_state.total_spam_words + vocab_size)
+
+    ham_count = @classifier_state.ham_counts.fetch(token, 0)
+    ham_likelihood = (ham_count + 1.0) / (@classifier_state.total_ham_words + vocab_size)
+
+    [ spam_likelihood, ham_likelihood ]
+  end
+
+  # Corrected to use the actual priors when determining "interestingness"
+  def get_significant_tokens(tokens, prob_spam_prior, prob_ham_prior)
+    # Use a Set to consider each unique token only once
+    unique_tokens = tokens.to_set
+
+    token_scores = unique_tokens.map do |token|
+      spam_likelihood, ham_likelihood = get_likelihoods(token)
+
+      # Calculate the actual P(Spam|token) using the real priors
+      # P(S|W) = P(W|S)P(S) / (P(W|S)P(S) + P(W|H)P(H))
+      prob_word_given_spam = spam_likelihood * prob_spam_prior
+      prob_word_given_ham = ham_likelihood * prob_ham_prior
+
+      # Avoid division by zero if both are 0
+      denominator = prob_word_given_spam + prob_word_given_ham
+      next [ token, 0.5 ] if denominator == 0
+
+      prob = prob_word_given_spam / denominator
+      interestingness = (prob - 0.5).abs
+
+      [ token, interestingness ]
+    end
+
+    # Select the top 15 most interesting tokens
+    token_scores.sort_by { |_, interest| -interest }
+      .first(15)
+      .map { |token, _| token }
+  end
+
   class << self
     def rebuild_all_public
       Rails.logger.info "Starting rebuild for all public classifiers..."

vendor/dictionaries/user.dict.utf8

@@ -308,4 +308,12 @@ K 线
 学生党
 分享群
 冷静期
-大魔王
+大魔王
+副卡
+文爱
+解绑
+反差婊
+黄毛
+模拟仓
+吃干抹净
+防刷屏