Add a new page to generate random trained message

to verify the quantity of Jieba tokenization

Author: ramsayleung

app/controllers/tokenization_samples_controller.rb

@@ -0,0 +1,17 @@
+class TokenizationSamplesController < ApplicationController
+  def new
+    @trained_messages = TrainedMessage.order("RANDOM()").limit(5)
+    if @trained_messages.any?
+      service = SpamClassifierService.new(0, "dummy")
+      @samples = @trained_messages.map do |msg|
+        {
+          original: msg.message,
+          cleaned: service.clean_text(msg.message),
+          tokens: service.tokenize(msg.message)
+        }
+      end
+    else
+      @samples = []
+    end
+  end
+end

app/views/layouts/application.html.erb

@@ -47,6 +47,8 @@
 
                 <%= link_to "Banned Users", banned_users_path, 
                 class: "#{current_page?(banned_users_path) ? 'border-indigo-500 text-gray-900' : 'border-transparent text-gray-500 hover:border-gray-300 hover:text-gray-700'} inline-flex items-center px-1 pt-1 border-b-2 text-sm font-medium" %>
+                <%= link_to "Tokenization Samples", new_tokenization_sample_path, 
+                class: "#{current_page?(new_tokenization_sample_path) ? 'border-indigo-500 text-gray-900' : 'border-transparent text-gray-500 hover:border-gray-300 hover:text-gray-700'} inline-flex items-center px-1 pt-1 border-b-2 text-sm font-medium" %>
                 <%= link_to "Solid Queue Dashboard", solid_queue_dashboard.root_path, 
                 class: "#{current_page?(solid_queue_dashboard.root_path) ? 'border-indigo-500 text-gray-900' : 'border-transparent text-gray-500 hover:border-gray-300 hover:text-gray-700'} inline-flex items-center px-1 pt-1 border-b-2 text-sm font-medium" %>                
             </div>

app/views/tokenization_samples/new.html.erb

@@ -0,0 +1,41 @@
+<div class="container mx-auto px-4">
+  <h1 class="text-2xl font-bold mb-4">Tokenization Samples</h1>
+
+  <%= link_to "Get Another Random Samples",
+              new_tokenization_sample_path,
+              data: { turbo_frame: "sample" },
+              class: "bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded" %>
+
+  <turbo-frame id="sample">
+    <% if @samples.any? %>
+      <% @samples.each_with_index do |sample, index| %>
+        <div class="mt-6 p-4 border rounded shadow bg-white">
+          <h2 class="text-xl font-semibold">Sample <%= index + 1 %></h2>
+
+          <div class="mt-2">
+            <h3 class="text-lg font-medium">Original Message</h3>
+            <p class="bg-gray-100 p-2 rounded"><%= sample[:original] %></p>
+          </div>
+
+          <div class="mt-2">
+            <h3 class="text-lg font-medium">Cleaned Message</h3>
+            <p class="bg-gray-100 p-2 rounded"><%= sample[:cleaned] %></p>
+          </div>
+
+          <div class="mt-2">
+            <h3 class="text-lg font-medium">Tokens</h3>
+            <div class="bg-gray-100 p-2 rounded flex flex-wrap">
+              <% sample[:tokens].each do |token| %>
+                <span class="inline-block bg-gray-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2">
+                  <%= token %>
+                </span>
+              <% end %>
+            </div>
+          </div>
+        </div>
+      <% end %>
+    <% else %>
+      <p class="mt-4">No trained messages found in the database.</p>
+    <% end %>
+  </turbo-frame>
+</div>

config/routes.rb

@@ -5,6 +5,7 @@
   resources :trained_messages do
     post :bulk_update, on: :collection
   end
+  resources :tokenization_samples, only: [ :new ]
   # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html
 
   # Reveal health status on /up that returns 200 if the app boots with no exceptions, otherwise 500.