Persist RFC822 Message-ID for cross-sync IMAP dedup; harden EML export

IMAP composite IDs (mailbox|uid) change when messages move between
mailboxes across syncs. Without persistent dedup, a message moved
from All Mail to Trash would be re-imported as a duplicate.

Add rfc822_message_id column to messages table, populated from the
parsed MIME Message-ID header during ingest. For IMAP sources,
check this column before persisting to skip messages already
imported under a different composite ID. Dedup skips are counted
as skipped (not errors) via errDuplicateRFC822 sentinel.

Harden sanitizeEMLFilename with filepath.Base to ensure the output
is a plain filename with no directory components, guarding against
IMAP mailbox names containing path separators.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: wesm

cmd/msgvault/cmd/export_eml.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -72,7 +73,11 @@ func sanitizeEMLFilename(sourceMessageID string) string {
 		}
 		return r
 	}, sourceMessageID)
-	if safe == "" {
+	// Ensure the result is a plain filename with no directory
+	// components, guarding against IMAP mailbox names with
+	// path separators or traversal sequences.
+	safe = filepath.Base(safe)
+	if safe == "" || safe == "." {
 		safe = "message"
 	}
 	return safe + ".eml"

internal/store/messages.go

@@ -45,7 +45,8 @@ type Message struct {
 	ConversationID  int64
 	SourceID        int64
 	SourceMessageID string
-	MessageType     string // "email"
+	RFC822MessageID sql.NullString // RFC822 Message-ID header for cross-mailbox dedup
+	MessageType     string         // "email"
 	SentAt          sql.NullTime
 	ReceivedAt      sql.NullTime
 	InternalDate    sql.NullTime
@@ -85,6 +86,20 @@ func (s *Store) MessageExistsBatch(sourceID int64, sourceMessageIDs []string) (m
 	return result, nil
 }
 
+// MessageExistsByRFC822ID checks if a message with the given
+// RFC822 Message-ID already exists for this source.
+func (s *Store) MessageExistsByRFC822ID(
+	sourceID int64, rfc822ID string,
+) (bool, error) {
+	var count int
+	err := s.db.QueryRow(
+		`SELECT COUNT(*) FROM messages
+		 WHERE source_id = ? AND rfc822_message_id = ?`,
+		sourceID, rfc822ID,
+	).Scan(&count)
+	return count > 0, err
+}
+
 // MessageExistsWithRawBatch checks which message IDs already exist in the database
 // and have raw MIME data stored.
 // Returns a map of source_message_id -> internal message_id.
@@ -144,13 +159,15 @@ func (s *Store) EnsureConversation(sourceID int64, sourceConversationID, title s
 
 const upsertMessageSQL = `
 	INSERT INTO messages (
-		conversation_id, source_id, source_message_id, message_type,
+		conversation_id, source_id, source_message_id,
+		rfc822_message_id, message_type,
 		sent_at, received_at, internal_date, sender_id, is_from_me,
 		subject, snippet, size_estimate,
 		has_attachments, attachment_count, archived_at
-	) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
+	) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
 	ON CONFLICT(source_id, source_message_id) DO UPDATE SET
 		conversation_id = excluded.conversation_id,
+		rfc822_message_id = excluded.rfc822_message_id,
 		sent_at = excluded.sent_at,
 		received_at = excluded.received_at,
 		internal_date = excluded.internal_date,
@@ -169,7 +186,8 @@ func (s *Store) UpsertMessage(msg *Message) (int64, error) {
 
 func upsertMessage(q querier, msg *Message) (int64, error) {
 	args := []any{
-		msg.ConversationID, msg.SourceID, msg.SourceMessageID, msg.MessageType,
+		msg.ConversationID, msg.SourceID, msg.SourceMessageID,
+		msg.RFC822MessageID, msg.MessageType,
 		msg.SentAt, msg.ReceivedAt, msg.InternalDate, msg.SenderID, msg.IsFromMe,
 		msg.Subject, msg.Snippet, msg.SizeEstimate,
 		msg.HasAttachments, msg.AttachmentCount,

internal/store/schema.sql

@@ -108,6 +108,9 @@ CREATE TABLE IF NOT EXISTS messages (
     -- Platform-specific ID for dedup
     source_message_id TEXT,
 
+    -- RFC822 Message-ID for cross-mailbox dedup (IMAP)
+    rfc822_message_id TEXT,
+
     -- Message classification
     message_type TEXT NOT NULL,  -- 'email', 'imessage', 'sms', 'mms', 'rcs', 'whatsapp'
 

internal/store/store.go

@@ -226,12 +226,20 @@ func (s *Store) InitSchema() error {
 		return fmt.Errorf("execute schema.sql: %w", err)
 	}
 
-	// Migration: add sync_config column to sources for databases created before
-	// IMAP support was added. SQLite returns "duplicate column name" if the
-	// column already exists, which we treat as success.
-	if _, err := s.db.Exec(`ALTER TABLE sources ADD COLUMN sync_config JSON`); err != nil {
-		if !isSQLiteError(err, "duplicate column name") {
-			return fmt.Errorf("migrate schema (sync_config): %w", err)
+	// Migrations: add columns for databases created before these features.
+	// SQLite returns "duplicate column name" if the column already exists,
+	// which we treat as success.
+	for _, m := range []struct {
+		sql  string
+		desc string
+	}{
+		{`ALTER TABLE sources ADD COLUMN sync_config JSON`, "sync_config"},
+		{`ALTER TABLE messages ADD COLUMN rfc822_message_id TEXT`, "rfc822_message_id"},
+	} {
+		if _, err := s.db.Exec(m.sql); err != nil {
+			if !isSQLiteError(err, "duplicate column name") {
+				return fmt.Errorf("migrate schema (%s): %w", m.desc, err)
+			}
 		}
 	}
 

internal/sync/sync.go

@@ -212,6 +212,10 @@ func (s *Syncer) processBatch(ctx context.Context, sourceID int64, listResp *gma
 
 			threadID := threadIDs[newIDs[i]]
 			if err := s.ingestMessage(ctx, sourceID, raw, threadID, labelMap); err != nil {
+				if errors.Is(err, errDuplicateRFC822) {
+					result.skipped++
+					continue
+				}
 				s.logger.Warn("failed to ingest message", "id", raw.ID, "error", err)
 				checkpoint.ErrorsCount++
 				continue
@@ -511,10 +515,17 @@ func (s *Syncer) parseToModel(sourceID int64, raw *gmail.RawMessage, threadID st
 	}
 
 	// Build message record
+	rfc822ID := sql.NullString{}
+	if parsed.MessageID != "" {
+		rfc822ID = sql.NullString{
+			String: parsed.MessageID, Valid: true,
+		}
+	}
 	msg := &store.Message{
 		ConversationID:  conversationID,
 		SourceID:        sourceID,
 		SourceMessageID: raw.ID,
+		RFC822MessageID: rfc822ID,
 		MessageType:     "email",
 		SenderID:        senderID,
 		Subject:         sql.NullString{String: subject, Valid: subject != ""},
@@ -634,13 +645,35 @@ func (s *Syncer) persistMessage(data *messageData, labelMap map[string]int64) er
 	return nil
 }
 
+// errDuplicateRFC822 signals that a message was skipped because
+// another message with the same RFC822 Message-ID already exists
+// for this source. Used for cross-sync dedup on IMAP where
+// composite IDs change when messages move between mailboxes.
+var errDuplicateRFC822 = errors.New("duplicate RFC822 Message-ID")
+
 // ingestMessage parses and stores a single message.
 func (s *Syncer) ingestMessage(ctx context.Context, sourceID int64, raw *gmail.RawMessage, threadID string, labelMap map[string]int64) error {
 	data, err := s.parseToModel(sourceID, raw, threadID)
 	if err != nil {
 		return err
 	}
 
+	// For IMAP sources, check if a message with the same RFC822
+	// Message-ID already exists under a different composite ID.
+	// This handles messages that moved between mailboxes across
+	// syncs (e.g. All Mail → Trash changes the mailbox|uid key).
+	if s.opts.SourceType == "imap" &&
+		data.message.RFC822MessageID.Valid {
+		exists, err := s.store.MessageExistsByRFC822ID(
+			sourceID, data.message.RFC822MessageID.String)
+		if err != nil {
+			return fmt.Errorf("check rfc822 dedup: %w", err)
+		}
+		if exists {
+			return errDuplicateRFC822
+		}
+	}
+
 	return s.persistMessage(data, labelMap)
 }
 

internal/sync/sync_test.go

@@ -1686,6 +1686,47 @@ func TestIMAPThreading(t *testing.T) {
 	}
 }
 
+// TestIMAPCrossSyncDedup verifies that a message imported from one mailbox
+// is not re-imported when it appears under a different mailbox|uid on a
+// subsequent sync (e.g. moved from All Mail to Trash).
+func TestIMAPCrossSyncDedup(t *testing.T) {
+	env := newTestEnv(t)
+	env.SetOptions(t, func(o *Options) {
+		o.SourceType = "imap"
+	})
+
+	msg := testemail.NewMessage().
+		Subject("Dedup test").
+		Header("Message-ID", "<dedup@example.com>").
+		Body("Same message, different mailbox.").
+		Bytes()
+
+	// First sync: message is in All Mail
+	env.Mock.Profile.MessagesTotal = 1
+	env.Mock.Profile.HistoryID = 100
+	env.Mock.AddMessage("AllMail|42", msg, []string{"AllMail"})
+	summary := runFullSync(t, env)
+	assertSummary(t, summary, WantSummary{Added: intPtr(1)})
+
+	// Second sync: message moved to Trash (different composite ID)
+	delete(env.Mock.Messages, "AllMail|42")
+	env.Mock.AddMessage("Trash|99", msg, []string{"Trash"})
+	summary = runFullSync(t, env)
+	// Should be skipped via RFC822 Message-ID dedup, not re-imported
+	assertSummary(t, summary, WantSummary{Added: intPtr(0)})
+
+	// Only one message should exist in the database
+	var count int
+	err := env.Store.DB().QueryRow(
+		`SELECT COUNT(*) FROM messages`).Scan(&count)
+	if err != nil {
+		t.Fatalf("count messages: %v", err)
+	}
+	if count != 1 {
+		t.Errorf("expected 1 message, got %d (duplicate imported)", count)
+	}
+}
+
 // TestIncrementalSyncLabelRemovedWithMissingRaw verifies that removing a label
 // from a message whose raw MIME data is missing still succeeds. The label-removal
 // path operates on the message_labels table directly and never touches raw data.