Fix encoding issues with the mbox import

We force-encoded to ascii, because early emails either used that or
base64. This didn't work properly with non english characters and also
resulted in artifacts with incorrect utf8 characters.

Instead we treat it as utf8 and ignore unknown characters. It is
possible that we do have some emails that use another encoding
internally - in this case we can fix that later and reimport again.

The commit also updates the import script so that it can reimport the
email bodies, and adds another script that only imports one single
email, for testing import changes on specific emails showing artifacts.

Author: dutow

app/services/email_ingestor.rb

@@ -9,16 +9,19 @@ def ingest_raw(raw_message, fallback_threading: false)
     return nil unless message_id
     sent_at = sanitize_email_date(m.date, m[:date], message_id)
 
-    return nil if message_id && Message.find_by_message_id(message_id)
+    body = normalize_body(extract_body(m))
+    existing_message = Message.find_by_message_id(message_id)
+    if existing_message
+      existing_message.update_columns(body: body)
+      return existing_message
+    end
 
     import_log = ''
 
     from = build_from_aliases(m, sent_at)
     to = create_users(m[:to], sent_at)
     cc = create_users(m[:cc], sent_at)
 
-    body = extract_body(m)
-
     subject = m.subject || 'No title'
 
     reply_to_msg, import_log = resolve_threading(m, import_log)
@@ -30,8 +33,6 @@ def ingest_raw(raw_message, fallback_threading: false)
     topic = reply_to_msg ? reply_to_msg.topic : Topic.create!(creator: from[0], title: subject, created_at: sent_at)
     import_log = nil if import_log == ''
 
-    body = normalize_body(body)
-
     msg = Message.create!(
       topic: topic,
       sender: from[0],

script/mbox_import.rb

@@ -116,7 +116,7 @@ def parse_message(message)
     f.each_line do |line|
 
       # Some old lines contain illegal characters
-      line = line.force_encoding("ISO-8859-1").encode("utf-8", replace: nil)
+      line = line.encode("utf-8", :invalid => :replace)
 
       # all new messages refer to lists.postgresql.org, but not old emails
       # And we can't simply check for From, as it also matches inline attachments containing git diffs

script/mbox_single_import.rb

@@ -0,0 +1,65 @@
+require_relative "../config/environment"
+require_relative "../app/services/email_ingestor"
+
+if ARGV.length != 2
+  puts "Usage: #{$PROGRAM_NAME} /path/to/mbox <message-id>"
+  exit 1
+end
+
+mbox_path = ARGV[0]
+target_id = MessageIdNormalizer.normalize(ARGV[1])
+
+if target_id.nil? || target_id.empty?
+  puts "ERROR: message-id is blank after normalization"
+  exit 1
+end
+
+def normalize_message_id(message)
+  MessageIdNormalizer.normalize(Mail.new(message).message_id)
+rescue => e
+  warn "WARN: failed to parse message id (#{e.class}: #{e.message})"
+  ''
+end
+
+def process_message(message, target_id)
+  return false if message.empty?
+
+  message_id = normalize_message_id(message)
+  return false if message_id.empty?
+  return false unless message_id == target_id
+
+  msg = EmailIngestor.new.ingest_raw(message, fallback_threading: true)
+  if msg
+    puts "Reimported #{msg.message_id}"
+  else
+    puts "Message #{target_id} not imported (invalid message id?)"
+  end
+  true
+end
+
+found = false
+message = ""
+
+puts "Scanning #{mbox_path} for #{target_id}..."
+
+File.open(mbox_path, "r") do |f|
+  f.each_line do |line|
+    line = line.encode("utf-8", :invalid => :replace)
+
+    if line.match(/^From [^@]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+/i)
+      if process_message(message, target_id)
+        found = true
+        break
+      end
+      message = ""
+    else
+      message << line
+    end
+  end
+end
+
+if !found
+  found = process_message(message, target_id)
+end
+
+puts "Message not found in #{mbox_path}" unless found