Note import functionality

This commit adds a new import functionality to the settings menu, where
users can upload CSV files to mark which messages they have read
already.

This is mainly useful to import existing reading status from mailboxes,
but for now we provide no export scripts.

The format also supports specifying notes, which allows users to import
messages with tags.

Author: dutow

Gemfile

@@ -22,6 +22,7 @@ gem "jbuilder"
 gem "slim"
 gem "kaminari"
 gem "nokogiri"
+gem "csv"
 
 # Use Active Model has_secure_password [https://guides.rubyonrails.org/active_model_basics.html#securepassword]
 gem "bcrypt", "~> 3.1"

Gemfile.lock

@@ -108,6 +108,7 @@ GEM
     crass (1.0.6)
     cssbundling-rails (1.4.3)
       railties (>= 6.0.0)
+    csv (3.3.5)
     date (3.5.0)
     debug (1.11.0)
       irb (~> 1.10)
@@ -484,6 +485,7 @@ DEPENDENCIES
   bullet
   capybara
   cssbundling-rails
+  csv
   debug
   factory_bot_rails
   faker

app/controllers/read_status_imports_controller.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "csv"
+require "set"
+
+class ReadStatusImportsController < ApplicationController
+  before_action :require_authentication
+
+  def new
+    @summary = nil
+  end
+
+  def create
+    uploaded = params[:import_file]
+    unless uploaded.respond_to?(:read)
+      flash.now[:alert] = "Please choose a CSV file to upload."
+      @summary = nil
+      return render :new
+    end
+
+    @summary = import_csv(uploaded)
+    render :new
+  rescue CSV::MalformedCSVError => e
+    flash.now[:alert] = "Invalid CSV file: #{e.message}"
+    @summary = nil
+    render :new
+  end
+
+  private
+
+  def import_csv(uploaded)
+    content = uploaded.read.to_s
+    rows = parse_csv_rows(content)
+
+    result = {
+      new_notes: 0,
+      replaced_notes: 0,
+      marked_read: 0,
+      skipped_message_ids: [],
+      warnings: []
+    }
+
+    read_message_ids = Set.new
+    note_builder = NoteBuilder.new(author: current_user)
+
+    rows.each do |message_id_raw, note_mode_raw, note_text_raw|
+      message_id = message_id_raw.to_s.strip
+      next if message_id.blank?
+
+      message = Message.find_by(message_id: message_id)
+      unless message
+        result[:skipped_message_ids] << message_id
+        next
+      end
+
+      if read_message_ids.add?(message.id)
+        MessageReadRange.add_range(user: current_user, topic: message.topic, start_id: message.id, end_id: message.id)
+        ThreadAwareness.mark_until(user: current_user, topic: message.topic, until_message_id: message.id)
+      end
+
+      note_mode = note_mode_raw.to_s.strip.downcase
+      next if note_mode.blank? || note_mode == "none"
+
+      note_text = note_text_raw.to_s
+      if note_text.strip.blank?
+        result[:warnings] << "Message #{message.message_id}: note text missing; skipped note import."
+        next
+      end
+
+      target_message =
+        case note_mode
+        when "message"
+          message
+        when "topic"
+          nil
+        else
+          result[:warnings] << "Message #{message.message_id}: unknown note mode '#{note_mode_raw}'."
+          next
+        end
+
+      note_body = note_text.sub(/\A!autoimport\s*/i, "").strip
+      note_body = "!autoimport #{note_body}".strip
+
+      imported_notes = Note.active
+                           .where(author: current_user, topic_id: message.topic_id, message_id: target_message&.id)
+                           .where("notes.body LIKE ?", "!autoimport%")
+                           .order(:id)
+
+      imported_note = imported_notes.first
+
+      if imported_note
+        note_builder.update!(note: imported_note, body: note_body)
+        result[:replaced_notes] += 1
+      else
+        note_builder.create!(topic: message.topic, message: target_message, body: note_body)
+        result[:new_notes] += 1
+      end
+
+      imported_notes.offset(1).each do |extra|
+        extra.transaction do
+          extra.update!(deleted_at: Time.current)
+          extra.note_mentions.delete_all
+          extra.note_tags.delete_all
+          extra.activities.update_all(hidden: true)
+        end
+      end
+    rescue NoteBuilder::Error, ActiveRecord::RecordInvalid => e
+      result[:warnings] << "Message #{message.message_id}: #{e.message}"
+    end
+
+    result[:marked_read] = read_message_ids.size
+    result
+  end
+
+  def parse_csv_rows(content)
+    return [] if content.blank?
+
+    with_headers = CSV.parse(content, headers: true)
+    headers = with_headers.headers.compact.map { |h| h.to_s.strip.downcase }
+
+    if headers.include?("message_id")
+      with_headers.map do |row|
+        [
+          row["message_id"],
+          row["notemode"] || row["note_mode"],
+          row["note"]
+        ]
+      end
+    else
+      CSV.parse(content, headers: false).map do |row|
+        [row[0], row[1], row[2]]
+      end
+    end
+  end
+end

app/views/read_status_imports/new.html.slim

@@ -0,0 +1,47 @@
+- content_for :title, "Import read status"
+
+.settings-page
+  h1 Import existing read status
+
+  .settings-section
+    ul
+      li
+        p Upload a CSV file with rows like: 
+        pre message_id,notemode,note
+      li All mentioned message_ids that are present in hackorum will be marked as read.
+      li If a proper note is present for the message_id, it will be also created or updated
+      li This is mainly intended for importing existing mailbox folders/tags
+      li The header row is optional. If present, use message_id, notemode, note as column names.
+      li message_id is required and should be the email Message-ID value without the < and > characters (same as the postgresql.org archive message ID)
+      li notemode is optional and can be none, message, or topic (case-insensitive).
+      li note is optional. If note is empty or missing, no note is imported even when notemode is set.
+      li For notemode=message, notes attach to that message. For notemode=topic, notes attach to the topic (thread).
+      li Notes are stored as "!autoimport <your note>" so you can include @mentions and #tags.
+      li Each message/topic can only have one autoimport message - if there's already one, the upload will overwrite the existing note.
+
+    = form_with url: read_status_imports_path, method: :post, local: true, multipart: true, data: { turbo: false } do
+      .form-group
+        = label_tag :import_file, "CSV file"
+        = file_field_tag :import_file, accept: ".csv,text/csv", required: true
+      = submit_tag "Import", class: "button-primary"
+
+  - if @summary
+    .settings-section
+      h2 Import results
+      ul
+        li New notes: #{@summary[:new_notes]}
+        li Replaced notes: #{@summary[:replaced_notes]}
+        li Message IDs marked as read: #{@summary[:marked_read]}
+        li Skipped message IDs: #{@summary[:skipped_message_ids].size}
+
+      - if @summary[:warnings].any?
+        h3 Warnings
+        ul
+          - @summary[:warnings].each do |warning|
+            li = warning
+
+      - if @summary[:skipped_message_ids].any?
+        h3 Skipped message IDs
+        ul
+          - @summary[:skipped_message_ids].each do |mid|
+            li = mid

app/views/settings/show.html.slim

@@ -93,3 +93,7 @@
   .settings-section
     h2 Teams
     = link_to "Manage teams", teams_path, class: "button-secondary"
+
+  .settings-section
+    h2 Import
+    = link_to "Import existing read status", new_read_status_import_path, class: "button-secondary"

config/routes.rb

@@ -39,6 +39,7 @@
     post :mark_all_read, on: :collection
   end
   resources :notes, only: [:create, :update, :destroy]
+  resources :read_status_imports, only: [:new, :create]
   get "stats", to: "stats#show", as: :stats
   get "stats/data", to: "stats#data", as: :stats_data
   get "person/*email/contributions/:year", to: "people#contributions", as: :person_contributions, format: false

spec/requests/read_status_imports_spec.rb

@@ -0,0 +1,109 @@
+require "rails_helper"
+require "cgi"
+
+RSpec.describe "Read status imports", type: :request do
+  def sign_in(email:, password: "secret")
+    post session_path, params: { email: email, password: password }
+    expect(response).to redirect_to(root_path)
+  end
+
+  def attach_verified_alias(user, email:, primary: true)
+    al = create(:alias, user: user, email: email)
+    if primary && user.person&.default_alias_id.nil?
+      user.person.update!(default_alias_id: al.id)
+    end
+    Alias.by_email(email).update_all(verified_at: Time.current)
+    al
+  end
+
+  def upload_csv(content)
+    file = Tempfile.new(["read-status", ".csv"])
+    file.write(content)
+    file.rewind
+    Rack::Test::UploadedFile.new(file.path, "text/csv")
+  ensure
+    file.close
+  end
+
+  it "requires authentication" do
+    get new_read_status_import_path
+    expect(response).to redirect_to(new_session_path)
+  end
+
+  it "imports read status and notes with warnings and skips" do
+    user = create(:user, password: "secret", password_confirmation: "secret")
+    attach_verified_alias(user, email: "[email protected]")
+    sign_in(email: "[email protected]")
+
+    message1 = create(:message)
+    message2 = create(:message)
+
+    csv = <<~CSV
+      message_id,notemode,note
+      #{message1.message_id},message,Imported #tag
+      #{message2.message_id},message,
+      <[email protected]>,message,Missing message
+    CSV
+
+    post read_status_imports_path, params: { import_file: upload_csv(csv) }
+
+    note = Note.active.find_by(author: user, message_id: message1.id)
+    expect(note).to be_present
+    expect(note.body).to start_with("!autoimport")
+    expect(note.note_tags.pluck(:tag)).to include("tag")
+
+    expect(Note.active.find_by(author: user, message_id: message2.id)).to be_nil
+
+    expect(MessageReadRange.covering?(user: user, topic: message1.topic, message_id: message1.id)).to be(true)
+    expect(MessageReadRange.covering?(user: user, topic: message2.topic, message_id: message2.id)).to be(true)
+    expect(ThreadAwareness.covering?(user: user, topic: message1.topic, message_id: message1.id)).to be(true)
+    expect(ThreadAwareness.covering?(user: user, topic: message2.topic, message_id: message2.id)).to be(true)
+
+    expect(response.body).to include("New notes: 1")
+    expect(response.body).to include("Replaced notes: 0")
+    expect(response.body).to include("Message IDs marked as read: 2")
+    expect(response.body).to include("Skipped message IDs: 1")
+    escaped_message_id = CGI.escapeHTML(message2.message_id)
+    expect(response.body).to include("Message #{escaped_message_id}: note text missing; skipped note import.")
+    expect(response.body).to include(CGI.escapeHTML("<[email protected]>"))
+  end
+
+  it "replaces existing imported notes" do
+    user = create(:user, password: "secret", password_confirmation: "secret")
+    attach_verified_alias(user, email: "[email protected]")
+    sign_in(email: "[email protected]")
+
+    message = create(:message)
+    Note.create!(topic: message.topic, message: message, author: user, body: "!autoimport old text")
+
+    csv = <<~CSV
+      #{message.message_id},message,New text
+    CSV
+
+    post read_status_imports_path, params: { import_file: upload_csv(csv) }
+
+    note = Note.active.find_by(author: user, message_id: message.id)
+    expect(note.body).to eq("!autoimport New text")
+    expect(response.body).to include("New notes: 0")
+    expect(response.body).to include("Replaced notes: 1")
+  end
+
+  it "imports topic-level notes for a message's topic" do
+    user = create(:user, password: "secret", password_confirmation: "secret")
+    attach_verified_alias(user, email: "[email protected]")
+    sign_in(email: "[email protected]")
+
+    message = create(:message)
+
+    csv = <<~CSV
+      #{message.message_id},topic,Thread note #tag
+    CSV
+
+    post read_status_imports_path, params: { import_file: upload_csv(csv) }
+
+    note = Note.active.find_by(author: user, topic_id: message.topic_id, message_id: nil)
+    expect(note).to be_present
+    expect(note.body).to eq("!autoimport Thread note #tag")
+    expect(note.note_tags.pluck(:tag)).to include("tag")
+  end
+end