Merge pull request #63 from amatsuda/import_from_raw_mail

Yet another email importer that imports from raw mail text

Author: amatsuda

app/helpers/messages_helper.rb

@@ -1,6 +1,6 @@
 module MessagesHelper
   def without_list_prefix(subject)
-    subject.sub(/^\[.+?\]\s*/, '')
+    subject&.sub(/^\[.+?\]\s*/, '')
   end
 
   MARGIN = 50

app/models/message.rb

@@ -11,6 +11,45 @@ class Message < ApplicationRecord
   self.skip_time_zone_conversion_for_attributes = [:published_at]
 
   class << self
+    def from_mail(mail, list, list_seq)
+      body = Kconv.toutf8 mail.body.raw_source
+      if ((list.name == 'ruby-dev') && list_seq.in?([13859, 26229, 39731, 39734])) || ((list.name == 'ruby-core') && list_seq.in?([5231])) || ((list.name == 'ruby-list') && list_seq.in?([29637, 29711, 30148])) || ((list.name == 'ruby-talk') && list_seq.in?([5198, 61316]))
+        body.gsub!("\u0000", '')
+      end
+      if (list.name == 'ruby-list') && list_seq.in?([37565, 38116, 43106])
+        mail.header[:subject].value.chop!
+      end
+      if (list.name == 'ruby-list') && (list_seq.in?([41850, 43710]))
+        mail.header[:subject].value = Kconv.toutf8 mail.header[:subject].value
+      end
+      subject = mail.subject
+      subject = Kconv.toutf8 subject if subject
+      from = Kconv.toutf8 mail.from_address&.raw
+      if !from && (list.name == 'ruby-core') && (list_seq == 161)
+        from = mail.from.encode Encoding::UTF_8, Encoding::KOI8_R
+      end
+
+      message_id = mail.message_id&.encode Encoding::UTF_8, invalid: :replace, undef: :replace
+
+      # mail.in_reply_to returns strange Array object in some cases (?), so let's use the raw value
+      parent_message_id_header = extract_message_id_from_in_reply_to(mail.header[:in_reply_to]&.value)
+      parent_message_id = Message.where(message_id_header: parent_message_id_header).pick(:id) if parent_message_id_header
+      if !parent_message_id && (String === mail.references)
+        parent_message_id = Message.where(message_id_header: mail.references).pick(:id)
+      end
+      if !parent_message_id && (Array === mail.references)
+        mail.references.compact.each do |ref|
+          break if (parent_message_id = Message.where(message_id_header: ref).pick(:id))
+        end
+      end
+
+      new list_id: list.id, list_seq: list_seq, body: body, subject: subject, from: from, published_at: mail.date, message_id_header: message_id, parent_id: parent_message_id
+    end
+
+    private def extract_message_id_from_in_reply_to(header)
+      header && header.strip.scan(/<([^>]+)>/).flatten.first
+    end
+
     def from_s3(list_name, list_seq, s3_client = Aws::S3::Client.new(region: BLADE_BUCKET_REGION))
       obj = s3_client.get_object(bucket: BLADE_BUCKET_NAME, key: "#{list_name}/#{list_seq}")
 

bin/import_mails

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'mail'
+
+BASE_DIR = Rails.root.join('tmp')
+
+params = {}
+OptionParser.new do |opts|
+  opts.on('--list LIST')
+  opts.on('--from FROM', Integer)
+  opts.on('--to TO', Integer)
+end.parse!(into: params)
+
+list = List.find_by_name(params[:list])
+
+errors = []
+
+Rails.logger.level = Logger::INFO
+
+Message.transaction do
+  (params[:from]..params[:to]).each do |seq|
+    begin
+      filepath = BASE_DIR.join(list.name, seq.to_s)
+      next unless filepath.exist?
+
+      str = File.binread filepath
+      next if str.blank?
+
+      mail = Mail.read_from_string str
+      message = Message.from_mail mail, list, seq
+      message.save!
+    rescue ActiveRecord::RecordNotUnique
+      STDERR.puts("#{list}:#{seq} already exists in Postgres")
+    rescue StandardError => e
+      errors << [seq, e]
+      STDERR.puts("failed to import #{list}:#{seq}: #{e}")
+    end
+  end
+end
+
+pp errors if errors.any?

db/migrate/20251017161507_add_index_messages_message_id_header.rb

@@ -0,0 +1,4 @@
+class AddIndexMessagesMessageIdHeader < ActiveRecord::Migration[8.0]
+  def change
+  end
+end

db/schema.rb

@@ -1,6 +1,20 @@
 require "test_helper"
 
 class MessageTest < ActiveSupport::TestCase
+  test 'from_mail' do
+    mail = Mail.read_from_string(<<END_OF_BODY)
+Subject: [ruby-list:1] Hello
+From: [email protected]
+Date: 2005-12-15T19:32:40+09:00
+
+Hello, world!
+END_OF_BODY
+    m = Message.from_mail(mail, List.find_by_name('ruby-list'), 1)
+    assert_equal "Hello, world!\r\n", m.body
+
+    assert_equal DateTime.parse('2005-12-15T19:32:40+09:00'), m.published_at
+  end
+
   test 'from_string' do
     m = Message.from_string(<<END_OF_BODY)
 Subject: [ruby-list:1] Hello