Merge pull request #67 from amatsuda/multipart

Handle multipart mails

Author: amatsuda

app/models/active_storage/file_blob.rb

@@ -0,0 +1,2 @@
+class ActiveStorage::FileBlob < ApplicationRecord
+end

app/models/message.rb

@@ -10,48 +10,83 @@ class Message < ApplicationRecord
   # https://blade.ruby-lang.org/ruby-talk/410000 is not.
   self.skip_time_zone_conversion_for_attributes = [:published_at]
 
+  has_many_attached :attachments
+
   attr_accessor :children
 
   class << self
     def from_mail(mail, list, list_seq)
-      body = Kconv.toutf8 mail.body.raw_source
-      if ((list.name == 'ruby-dev') && list_seq.in?([13859, 26229, 39731, 39734])) || ((list.name == 'ruby-core') && list_seq.in?([5231])) || ((list.name == 'ruby-list') && list_seq.in?([29637, 29711, 30148])) || ((list.name == 'ruby-talk') && list_seq.in?([5198, 61316]))
-        body.gsub!("\u0000", '')
-      end
-      if (list.name == 'ruby-list') && list_seq.in?([37565, 38116, 43106])
-        mail.header[:subject].value.chop!
-      end
-      if (list.name == 'ruby-list') && (list_seq.in?([41850, 43710]))
-        mail.header[:subject].value = Kconv.toutf8 mail.header[:subject].value
-      end
-      subject = mail.subject
-      subject = Kconv.toutf8 subject if subject
-      from = Kconv.toutf8 mail.from_address&.raw
-      if !from && (list.name == 'ruby-core') && (list_seq == 161)
-        from = mail.from.encode Encoding::UTF_8, Encoding::KOI8_R
-      end
+      new.from_mail(mail, list, list_seq)
+    end
+  end
 
-      message_id = mail.message_id&.encode Encoding::UTF_8, invalid: :replace, undef: :replace
+  def from_mail(mail, list, list_seq)
+    self.list_id, self.list_seq, self.published_at = list.id, list_seq, mail.date
 
-      # mail.in_reply_to returns strange Array object in some cases (?), so let's use the raw value
-      parent_message_id_header = extract_message_id_from_in_reply_to(mail.header[:in_reply_to]&.value)
-      parent_message_id = Message.where(list_id: list.id, message_id_header: parent_message_id_header).pick(:id) if parent_message_id_header
-      if !parent_message_id && (String === mail.references)
-        parent_message_id = Message.where(list_id: list.id, message_id_header: mail.references).pick(:id)
-      end
-      if !parent_message_id && (Array === mail.references)
-        mail.references.compact.each do |ref|
-          break if (parent_message_id = Message.where(list_id: list.id, message_id_header: ref).pick(:id))
-        end
+    if mail.multipart?
+      mail.parts.each do |p|
+        handle_multipart p
       end
+    else
+      self.body = Kconv.toutf8 mail.body.raw_source
+    end
+
+    if ((list.name == 'ruby-dev') && list_seq.in?([13859, 26229, 39731, 39734])) || ((list.name == 'ruby-core') && list_seq.in?([5231])) || ((list.name == 'ruby-list') && list_seq.in?([29637, 29711, 30148])) || ((list.name == 'ruby-talk') && list_seq.in?([5198, 61316]))
+      self.body.gsub!("\u0000", '')
+    end
+
+    if (list.name == 'ruby-list') && list_seq.in?([37565, 38116, 43106])
+      mail.header[:subject].value.chop!
+    end
+    if (list.name == 'ruby-list') && (list_seq.in?([41850, 43710]))
+      mail.header[:subject].value = Kconv.toutf8 mail.header[:subject].value
+    end
+    self.subject = mail.subject
+    self.subject = Kconv.toutf8 subject if self.subject
 
-      new list_id: list.id, list_seq: list_seq, body: body, subject: subject, from: from, published_at: mail.date, message_id_header: message_id, parent_id: parent_message_id
+    self.from = Kconv.toutf8 mail.from_address&.raw
+    if !self.from && (list.name == 'ruby-core') && (list_seq == 161)
+      self.from = mail.from.encode Encoding::UTF_8, Encoding::KOI8_R
     end
 
-    private def extract_message_id_from_in_reply_to(header)
-      header && header.strip.scan(/<([^>]+)>/).flatten.first
+    self.message_id_header = mail.message_id&.encode Encoding::UTF_8, invalid: :replace, undef: :replace
+
+    # mail.in_reply_to returns strange Array object in some cases (?), so let's use the raw value
+    parent_message_id_header = extract_message_id_from_in_reply_to(mail.header[:in_reply_to]&.value)
+    self.parent_id = Message.where(list_id: list.id, message_id_header: parent_message_id_header).pick(:id) if parent_message_id_header
+    if !self.parent_id && (String === mail.references)
+      self.parent_id = Message.where(list_id: list.id, message_id_header: mail.references).pick(:id)
+    end
+    if !self.parent_id && (Array === mail.references)
+      mail.references.compact.each do |ref|
+        break if (self.parent_id = Message.where(list_id: list.id, message_id_header: ref).pick(:id))
+      end
     end
 
+    self
+  end
+
+  private def handle_multipart(part)
+    if part.attachment?
+      file = StringIO.new(part.decoded)
+      attachments.attach(io: file, filename: part.filename, content_type: part.content_type)
+    else
+      case part.content_type.downcase
+      when /^text\/plain/
+        (self.body ||= '') << Kconv.toutf8(part.body.raw_source)
+      when /^text\/html;/
+        (self.html_body ||= '') << Kconv.toutf8(part.body.raw_source)
+      else
+        puts "Unknown content_type: #{part.content_type}"
+      end
+    end
+  end
+
+  private def extract_message_id_from_in_reply_to(header)
+    header && header.strip.scan(/<([^>]+)>/).flatten.first
+  end
+
+  class << self
     def from_s3(list_name, list_seq, s3_client = Aws::S3::Client.new(region: BLADE_BUCKET_REGION))
       obj = s3_client.get_object(bucket: BLADE_BUCKET_NAME, key: "#{list_name}/#{list_seq}")
 

config/storage.yml

@@ -1,10 +1,8 @@
 test:
-  service: Disk
-  root: <%= Rails.root.join("tmp/storage") %>
+  service: Database
 
 local:
-  service: Disk
-  root: <%= Rails.root.join("storage") %>
+  service: Database
 
 # Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
 # amazon:

db/migrate/20251012141446_add_html_body_to_messages.rb

@@ -0,0 +1,5 @@
+class AddHtmlBodyToMessages < ActiveRecord::Migration[8.0]
+  def change
+    add_column :messages, :html_body, :text
+  end
+end

db/migrate/20251014140646_create_active_storage_file_blobs.rb

@@ -0,0 +1,11 @@
+class CreateActiveStorageFileBlobs < ActiveRecord::Migration[8.0]
+  def change
+    create_table :active_storage_file_blobs do |t|
+      t.string :key
+      t.binary :data
+
+      t.timestamps
+    end
+    add_index :active_storage_file_blobs, :key, unique: true
+  end
+end

db/migrate/20251014141822_create_active_storage_tables.active_storage.rb

@@ -0,0 +1,57 @@
+# This migration comes from active_storage (originally 20170806125915)
+class CreateActiveStorageTables < ActiveRecord::Migration[7.0]
+  def change
+    # Use Active Record's configured type for primary and foreign keys
+    primary_key_type, foreign_key_type = primary_and_foreign_key_types
+
+    create_table :active_storage_blobs, id: primary_key_type do |t|
+      t.string   :key,          null: false
+      t.string   :filename,     null: false
+      t.string   :content_type
+      t.text     :metadata
+      t.string   :service_name, null: false
+      t.bigint   :byte_size,    null: false
+      t.string   :checksum
+
+      if connection.supports_datetime_with_precision?
+        t.datetime :created_at, precision: 6, null: false
+      else
+        t.datetime :created_at, null: false
+      end
+
+      t.index [ :key ], unique: true
+    end
+
+    create_table :active_storage_attachments, id: primary_key_type do |t|
+      t.string     :name,     null: false
+      t.references :record,   null: false, polymorphic: true, index: false, type: foreign_key_type
+      t.references :blob,     null: false, type: foreign_key_type
+
+      if connection.supports_datetime_with_precision?
+        t.datetime :created_at, precision: 6, null: false
+      else
+        t.datetime :created_at, null: false
+      end
+
+      t.index [ :record_type, :record_id, :name, :blob_id ], name: :index_active_storage_attachments_uniqueness, unique: true
+      t.foreign_key :active_storage_blobs, column: :blob_id
+    end
+
+    create_table :active_storage_variant_records, id: primary_key_type do |t|
+      t.belongs_to :blob, null: false, index: false, type: foreign_key_type
+      t.string :variation_digest, null: false
+
+      t.index [ :blob_id, :variation_digest ], name: :index_active_storage_variant_records_uniqueness, unique: true
+      t.foreign_key :active_storage_blobs, column: :blob_id
+    end
+  end
+
+  private
+    def primary_and_foreign_key_types
+      config = Rails.configuration.generators
+      setting = config.options[config.orm][:primary_key_type]
+      primary_key_type = setting || :primary_key
+      foreign_key_type = setting || :bigint
+      [ primary_key_type, foreign_key_type ]
+    end
+end

db/schema.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'active_storage/service'
+
+module ActiveStorage
+  class Service::DatabaseService < Service
+    def upload(key, io, checksum: nil, **)
+      instrument :upload, key: key, checksum: checksum do
+        ActiveStorage::FileBlob.find_or_initialize_by(key: key) do
+          it.data = io.read
+          it.save!
+        end
+      end
+    end
+
+    def download(key)
+      instrument :download, key: key do
+        ActiveStorage::FileBlob.where(key: key).pick(:data)
+      end
+    end
+
+    def delete(key)
+      instrument :delete, key: key do
+        ActiveStorage::FileBlob.find_by(key: key)&.destroy
+      end
+    end
+
+    def exist?(key)
+      instrument :exist, key: key do |payload|
+        payload[:exist] = ActiveStorage::FileBlob.exists?(key: key)
+      end
+    end
+
+    private
+
+    def service_name
+      'Database'
+    end
+  end
+end

lib/active_storage/service/database_service.rb

@@ -0,0 +1,7 @@
+require "test_helper"
+
+class ActiveStorageFileBlobTest < ActiveSupport::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end