diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 9990a687729..d0e8b33f23b 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -898,15 +898,15 @@ importers: '@rush-temp/pod-gmail': specifier: file:./projects/pod-gmail.tgz version: file:projects/pod-gmail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(encoding@0.1.13)(utf-8-validate@6.0.4) - '@rush-temp/pod-inbound-mail': - specifier: file:./projects/pod-inbound-mail.tgz - version: file:projects/pod-inbound-mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9)) '@rush-temp/pod-love': specifier: file:./projects/pod-love.tgz version: file:projects/pod-love.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(utf-8-validate@6.0.4) '@rush-temp/pod-mail': specifier: file:./projects/pod-mail.tgz version: file:projects/pod-mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9)) + '@rush-temp/pod-mail-worker': + specifier: file:./projects/pod-mail-worker.tgz + version: file:projects/pod-mail-worker.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9)) '@rush-temp/pod-media': specifier: file:./projects/pod-media.tgz version: file:projects/pod-media.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9)) @@ -5331,14 +5331,14 @@ packages: resolution: {integrity: sha512-gIKoyoL1gSWpWR35vPCMOpYY/Z+TkO0YJvKHU9ysRpH5Uqm8vjBD91RckK86xNoAHCC6Tuk/xsHnLQfSuQTv3w==, tarball: file:projects/pod-gmail.tgz} version: 0.0.0 - '@rush-temp/pod-inbound-mail@file:projects/pod-inbound-mail.tgz': - resolution: {integrity: sha512-HYLxttD6rjp/qp/dljLvbNScWbKeu6bgjNEd4KQUvZpex5OrQLglgQILIgNKegp9bBqtiGbKnZQ8Nd4xi0Viiw==, tarball: file:projects/pod-inbound-mail.tgz} - version: 0.0.0 - '@rush-temp/pod-love@file:projects/pod-love.tgz': resolution: {integrity: sha512-ii97uZ2rg+aZ44gF7p0RLuW5Qs7S/Skb5RgAxuAgIRqydgNVaUelwBeGxgbpr5JnNfvp+jdWxRaTcN6S9o8tJQ==, tarball: file:projects/pod-love.tgz} version: 0.0.0 + '@rush-temp/pod-mail-worker@file:projects/pod-mail-worker.tgz': + resolution: {integrity: sha512-pGpDe2VXivzWayLm0pr99f+f5vD5eirwLnB1JTWUUTXtZtbDm6QOaAAEMlzWNwXTZfZGn74Qd7zl1AUCLp6O0A==, tarball: file:projects/pod-mail-worker.tgz} + version: 0.0.0 + '@rush-temp/pod-mail@file:projects/pod-mail.tgz': resolution: {integrity: sha512-WhDC0I0bu1HeY3HN/yz6xQHsanXYf0ZAWEToJuKb3IAuLTsq1k/um2url73TyBYmWT+XbzKDRMh+AWM5xkav+w==, tarball: file:projects/pod-mail.tgz} version: 0.0.0 @@ -23913,22 +23913,19 @@ snapshots: - supports-color - utf-8-validate - '@rush-temp/pod-inbound-mail@file:projects/pod-inbound-mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))': + '@rush-temp/pod-love@file:projects/pod-love.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(utf-8-validate@6.0.4)': dependencies: '@tsconfig/node16': 1.0.4 '@types/cors': 2.8.17 '@types/express': 4.17.21 '@types/jest': 29.5.12 '@types/node': 22.15.29 - '@types/sanitize-html': 2.15.0 - '@types/turndown': 5.0.5 '@types/uuid': 8.3.4 + '@types/ws': 8.5.11 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.3))(eslint@8.56.0)(typescript@5.8.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.8.3) cors: 2.8.5 - cross-env: 7.0.3 dotenv: 16.0.3 - eml-parse-js: 1.2.0-beta.0 esbuild: 0.24.2 eslint: 8.56.0 eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.3))(eslint@8.56.0)(typescript@5.8.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.8.3) @@ -23938,13 +23935,14 @@ snapshots: eslint-plugin-promise: 6.1.1(eslint@8.56.0) express: 4.21.2 jest: 29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3)) + jwt-simple: 0.5.6 + livekit-server-sdk: 2.11.0 prettier: 3.2.5 - sanitize-html: 2.15.0 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3)))(typescript@5.8.3) ts-node: 10.9.2(@types/node@22.15.29)(typescript@5.8.3) - turndown: 7.2.0 typescript: 5.8.3 uuid: 8.3.2 + ws: 8.18.2(bufferutil@4.0.8)(utf-8-validate@6.0.4) transitivePeerDependencies: - '@babel/core' - '@jest/types' @@ -23952,22 +23950,27 @@ snapshots: - '@swc/wasm' - babel-jest - babel-plugin-macros + - bufferutil - node-notifier - supports-color + - utf-8-validate - '@rush-temp/pod-love@file:projects/pod-love.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(utf-8-validate@6.0.4)': + '@rush-temp/pod-mail-worker@file:projects/pod-mail-worker.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))': dependencies: '@tsconfig/node16': 1.0.4 '@types/cors': 2.8.17 '@types/express': 4.17.21 '@types/jest': 29.5.12 '@types/node': 22.15.29 + '@types/sanitize-html': 2.15.0 + '@types/turndown': 5.0.5 '@types/uuid': 8.3.4 - '@types/ws': 8.5.11 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.3))(eslint@8.56.0)(typescript@5.8.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.8.3) cors: 2.8.5 + cross-env: 7.0.3 dotenv: 16.0.3 + eml-parse-js: 1.2.0-beta.0 esbuild: 0.24.2 eslint: 8.56.0 eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.3))(eslint@8.56.0)(typescript@5.8.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.8.3) @@ -23977,14 +23980,13 @@ snapshots: eslint-plugin-promise: 6.1.1(eslint@8.56.0) express: 4.21.2 jest: 29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3)) - jwt-simple: 0.5.6 - livekit-server-sdk: 2.11.0 prettier: 3.2.5 + sanitize-html: 2.16.0 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3)))(typescript@5.8.3) ts-node: 10.9.2(@types/node@22.15.29)(typescript@5.8.3) + turndown: 7.2.0 typescript: 5.8.3 uuid: 8.3.2 - ws: 8.18.2(bufferutil@4.0.8)(utf-8-validate@6.0.4) transitivePeerDependencies: - '@babel/core' - '@jest/types' @@ -23992,10 +23994,8 @@ snapshots: - '@swc/wasm' - babel-jest - babel-plugin-macros - - bufferutil - node-notifier - supports-color - - utf-8-validate '@rush-temp/pod-mail@file:projects/pod-mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))': dependencies: diff --git a/common/scripts/docker.sh b/common/scripts/docker.sh index 8df3dd1e8f6..16c2cb9c86b 100755 --- a/common/scripts/docker.sh +++ b/common/scripts/docker.sh @@ -17,7 +17,7 @@ rush docker:build -p 20 \ --to @hcengineering/pod-love \ --to @hcengineering/pod-mail \ --to @hcengineering/pod-datalake \ ---to @hcengineering/pod-inbound-mail \ +--to @hcengineering/pod-mail-worker \ --to @hcengineering/pod-export \ --to @hcengineering/pod-msg2file \ --to @hcengineering/pod-media \ diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 5d84c2d645c..84c883a2475 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -161,6 +161,8 @@ services: - BRANDING_PATH=/var/cfg/branding.json # - DISABLE_SIGNUP=true - OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318/v1/traces + - MAILBOX_DOMAINS=huly.dev.local + - MAILBOX_MAX_COUNT_PER_ACCOUNT=2 restart: unless-stopped stats: image: hardcoreeng/stats diff --git a/rush.json b/rush.json index ec61d4dec0c..3855d182e07 100644 --- a/rush.json +++ b/rush.json @@ -2504,8 +2504,8 @@ "shouldPublish": false }, { - "packageName": "@hcengineering/pod-inbound-mail", - "projectFolder": "services/mail/pod-inbound-mail", + "packageName": "@hcengineering/pod-mail-worker", + "projectFolder": "services/mail/pod-mail-worker", "shouldPublish": false }, { diff --git a/server/account/src/serviceOperations.ts b/server/account/src/serviceOperations.ts index 65faefd7146..5709876dfc8 100644 --- a/server/account/src/serviceOperations.ts +++ b/server/account/src/serviceOperations.ts @@ -555,7 +555,7 @@ export async function getPersonInfo ( ): Promise { const { account } = params const { extra } = decodeTokenVerbose(ctx, token) - verifyAllowedServices(['workspace', 'tool', 'gmail'], extra) + verifyAllowedServices(['workspace', 'tool', 'gmail', 'huly-mail'], extra) if (account == null || account === '') { throw new PlatformError(new Status(Severity.ERROR, platform.status.BadRequest, {})) @@ -917,7 +917,7 @@ export async function findFullSocialIds ( ): Promise { const { socialIds } = params const { extra } = decodeTokenVerbose(ctx, token) - verifyAllowedServices(['gmail', 'tool', 'workspace'], extra) + verifyAllowedServices(['gmail', 'tool', 'workspace', 'huly-mail'], extra) if (socialIds == null || socialIds.length === 0) { throw new PlatformError(new Status(Severity.ERROR, platform.status.BadRequest, {})) diff --git a/server/account/src/utils.ts b/server/account/src/utils.ts index 0bc9cf55662..c9b3cdeded2 100644 --- a/server/account/src/utils.ts +++ b/server/account/src/utils.ts @@ -1746,7 +1746,8 @@ export const integrationServices = [ 'mailbox', 'caldav', 'gmail', - 'google-calendar' + 'google-calendar', + 'huly-mail' ] export async function findExistingIntegration ( diff --git a/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts b/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts index 7aa34f09c61..85f48b2a5b9 100644 --- a/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts +++ b/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts @@ -74,7 +74,10 @@ jest.mock('@hcengineering/core', () => { jest.mock('@hcengineering/mail-common', () => ({ createMessages: jest.fn().mockResolvedValue(undefined), getChannel: jest.fn().mockResolvedValue({ _id: 'test-channel-id' }), - isSyncedMessage: jest.fn().mockReturnValue(false) + isSyncedMessage: jest.fn().mockReturnValue(false), + getMessageExtra: jest.fn().mockReturnValue({}), + getMailHeaders: jest.fn().mockReturnValue([]), + isHulyMessage: jest.fn().mockReturnValue(false) })) jest.mock('googleapis', () => ({ diff --git a/services/gmail/pod-gmail/src/gmail.ts b/services/gmail/pod-gmail/src/gmail.ts index 676a1ede022..d222be83343 100644 --- a/services/gmail/pod-gmail/src/gmail.ts +++ b/services/gmail/pod-gmail/src/gmail.ts @@ -35,21 +35,20 @@ import { isWorkspaceLoginInfo, AccountClient } from '@hcengineering/account-client' -import { MailRecipient, type SyncOptions, getChannel, isSyncedMessage } from '@hcengineering/mail-common' +import { + MailRecipient, + type SyncOptions, + getChannel, + getMailHeaders, + isSyncedMessage +} from '@hcengineering/mail-common' import chat from '@hcengineering/chat' import { encode64 } from './base64' import config from './config' import { GmailController } from './gmailController' import { RateLimiter } from './rateLimiter' -import { - type ProjectCredentials, - type Token, - type User, - type SyncState, - HulyMailHeader, - HulyMessageIdHeader -} from './types' +import { type ProjectCredentials, type Token, type User, type SyncState, GmailMessageType } from './types' import { addFooter, isToken, serviceToken, getKvsClient, createGmailSearchQuery } from './utils' import type { WorkspaceClient } from './workspaceClient' import { getOrCreateSocialId } from './accounts' @@ -73,8 +72,7 @@ function makeHTMLBody (message: NewMessage, from: string): string { 'Content-Transfer-Encoding: 7bit\n', `To: ${message.to} \n`, `From: ${from} \n`, - `${HulyMailHeader}: true\n`, - `${HulyMessageIdHeader}: ${message._id}\n` + ...getMailHeaders(GmailMessageType, message._id) ] if (message.replyTo != null) { @@ -385,7 +383,7 @@ export class GmailClient { } this.ctx.info('Sending gmail message', { id: message._id, email }) - const gmailBody = await makeHTMLBodyV2(this.accountClient, message, thread, this.socialId._id, email) + const gmailBody = await makeHTMLBodyV2(this.ctx, this.accountClient, message, thread, this.socialId._id, email) await this.rateLimiter.take(100) await this.gmail.messages.send({ userId: 'me', diff --git a/services/gmail/pod-gmail/src/message/v2/message.ts b/services/gmail/pod-gmail/src/message/v2/message.ts index 25ad957f68a..ae73073ef1d 100644 --- a/services/gmail/pod-gmail/src/message/v2/message.ts +++ b/services/gmail/pod-gmail/src/message/v2/message.ts @@ -24,7 +24,9 @@ import { EmailMessage, getProducer, MailRecipient, - getMessageExtra + getMessageExtra, + HulyMailHeader, + HulyMessageIdHeader } from '@hcengineering/mail-common' import { type KeyValueClient } from '@hcengineering/kvs-client' import { AccountClient, isWorkspaceLoginInfo, WorkspaceLoginInfo } from '@hcengineering/account-client' @@ -33,7 +35,7 @@ import { IMessageManager } from '../types' import config from '../../config' import { AttachmentHandler } from '../attachments' import { decode64 } from '../../base64' -import { GmailMessageType, HulyMailHeader, HulyMessageIdHeader } from '../../types' +import { GmailMessageType } from '../../types' export class MessageManagerV2 implements IMessageManager { private wsInfo: WorkspaceLoginInfo | undefined = undefined diff --git a/services/gmail/pod-gmail/src/message/v2/send.ts b/services/gmail/pod-gmail/src/message/v2/send.ts index 1faaf52e27e..e33d3753379 100644 --- a/services/gmail/pod-gmail/src/message/v2/send.ts +++ b/services/gmail/pod-gmail/src/message/v2/send.ts @@ -1,45 +1,42 @@ import { CreateMessageEvent } from '@hcengineering/communication-sdk-types' import { type GaxiosResponse } from 'gaxios' import { gmail_v1 } from 'googleapis' -import { markdownToHtml, getReplySubject } from '@hcengineering/mail-common' +import { + markdownToHtml, + getReplySubject, + getRecipients, + getMailHeaders, + HulyMailHeader, + HulyMessageIdHeader +} from '@hcengineering/mail-common' +import { Card } from '@hcengineering/card' +import { MeasureContext, PersonId } from '@hcengineering/core' +import { AccountClient } from '@hcengineering/account-client' import { encode64 } from '../../base64' import { addFooter } from '../../utils' -import { Card } from '@hcengineering/card' -import { PersonId, SocialIdType } from '@hcengineering/core' -import { AccountClient } from '@hcengineering/account-client' -import { HulyMailHeader, HulyMessageIdHeader } from '../../types' +import { GmailMessageType } from '../../types' export async function makeHTMLBodyV2 ( + ctx: MeasureContext, accountClient: AccountClient, message: CreateMessageEvent, thread: Card, personId: PersonId, from: string ): Promise { - const collaborators: PersonId[] = (thread as any).members ?? [] - if (collaborators.length === 0) { + const recipients = await getRecipients(ctx, accountClient, thread, personId) + if (recipients === undefined) { return undefined } - const recipients = collaborators.length > 1 ? collaborators.filter((c) => c !== personId) : collaborators - const mailSocialIds = (await accountClient.findFullSocialIds(recipients)).filter( - (id) => id.type === SocialIdType.EMAIL - ) - if (mailSocialIds.length === 0) { - console.warn('No social IDs found for recipients', { recipients }) - return undefined - } - const to = mailSocialIds[0].value - const copy = mailSocialIds.length > 1 ? mailSocialIds.slice(1).map((s) => s.value) : [] - + const { to, copy } = recipients const str = [ 'Content-Type: text/html; charset="UTF-8"\n', 'MIME-Version: 1.0\n', 'Content-Transfer-Encoding: 7bit\n', `To: ${to} \n`, `From: ${from} \n`, - `${HulyMailHeader}: true\n`, - `${HulyMessageIdHeader}: ${message._id}\n` + ...getMailHeaders(GmailMessageType, message._id) ] // TODO: get reply-to from channel diff --git a/services/gmail/pod-gmail/src/types.ts b/services/gmail/pod-gmail/src/types.ts index f4dde63c34f..b3872e202d6 100644 --- a/services/gmail/pod-gmail/src/types.ts +++ b/services/gmail/pod-gmail/src/types.ts @@ -94,5 +94,3 @@ export interface SyncState { } export const GmailMessageType = 'gmail-message' -export const HulyMailHeader = 'X-Huly-Sent' -export const HulyMessageIdHeader = 'X-Huly-Message-Id' diff --git a/services/mail/mail-common/src/__tests__/thread.test.ts b/services/mail/mail-common/src/__tests__/thread.test.ts index 6eda2880f54..70374e0f21e 100644 --- a/services/mail/mail-common/src/__tests__/thread.test.ts +++ b/services/mail/mail-common/src/__tests__/thread.test.ts @@ -25,7 +25,8 @@ describe('ThreadLookupService', () => { const MAIL_ID = 'test-mail-id' const SPACE_ID = 'test-space-id' as Ref const THREAD_ID = 'test-thread-id' as Ref - const KEY = `mail-thread-lookup:${MAIL_ID}:${SPACE_ID}` + const EMAIL = 'test@example.com' + const KEY = `mail-thread-lookup:${MAIL_ID}:${SPACE_ID}:${EMAIL}` // Mocks let mockCtx: MeasureContext @@ -98,7 +99,7 @@ describe('ThreadLookupService', () => { // Get service and call method const service = ThreadLookupService.getInstance(mockCtx, mockKeyValueClient, TOKEN) - const result = await service.getThreadId(MAIL_ID, SPACE_ID) + const result = await service.getThreadId(MAIL_ID, SPACE_ID, EMAIL) // Verify behavior expect(mockKeyValueClient.getValue).toHaveBeenCalledWith(KEY) @@ -107,7 +108,9 @@ describe('ThreadLookupService', () => { 'Found existing thread mapping', expect.objectContaining({ mailId: MAIL_ID, - threadId: THREAD_ID + spaceId: SPACE_ID, + threadId: THREAD_ID, + email: EMAIL }) ) }) @@ -118,7 +121,7 @@ describe('ThreadLookupService', () => { // Get service and call method const service = ThreadLookupService.getInstance(mockCtx, mockKeyValueClient, TOKEN) - const result = await service.getThreadId(MAIL_ID, SPACE_ID) + const result = await service.getThreadId(MAIL_ID, SPACE_ID, EMAIL) // Verify behavior expect(mockKeyValueClient.getValue).toHaveBeenCalledWith(KEY) @@ -133,7 +136,7 @@ describe('ThreadLookupService', () => { // Get service and call method const service = ThreadLookupService.getInstance(mockCtx, mockKeyValueClient, TOKEN) - const result = await service.getThreadId(MAIL_ID, SPACE_ID) + const result = await service.getThreadId(MAIL_ID, SPACE_ID, EMAIL) // Verify behavior expect(mockKeyValueClient.getValue).toHaveBeenCalledWith(KEY) @@ -142,6 +145,7 @@ describe('ThreadLookupService', () => { 'Failed to lookup thread for email', expect.objectContaining({ mailId: MAIL_ID, + spaceId: SPACE_ID, error: mockError }) ) @@ -152,7 +156,7 @@ describe('ThreadLookupService', () => { it('should store thread mapping in KVS', async () => { // Get service and call method const service = ThreadLookupService.getInstance(mockCtx, mockKeyValueClient, TOKEN) - await service.setThreadId(MAIL_ID, SPACE_ID, THREAD_ID) + await service.setThreadId(MAIL_ID, SPACE_ID, THREAD_ID, EMAIL) // Verify behavior expect(mockKeyValueClient.setValue).toHaveBeenCalledWith(KEY, { @@ -176,7 +180,7 @@ describe('ThreadLookupService', () => { // Get service and call method const service = ThreadLookupService.getInstance(mockCtx, mockKeyValueClient, TOKEN) - await service.setThreadId(MAIL_ID, SPACE_ID, THREAD_ID) + await service.setThreadId(MAIL_ID, SPACE_ID, THREAD_ID, EMAIL) // Verify behavior expect(mockKeyValueClient.setValue).toHaveBeenCalledWith(KEY, expect.any(Object)) @@ -196,7 +200,7 @@ describe('ThreadLookupService', () => { it('should return undefined when inReplyTo is undefined', async () => { // Get service and call method const service = ThreadLookupService.getInstance(mockCtx, mockKeyValueClient, TOKEN) - const result = await service.getParentThreadId(undefined, SPACE_ID) + const result = await service.getParentThreadId(undefined, SPACE_ID, EMAIL) // Verify behavior expect(result).toBeUndefined() @@ -214,10 +218,10 @@ describe('ThreadLookupService', () => { // Get service and call method const service = ThreadLookupService.getInstance(mockCtx, mockKeyValueClient, TOKEN) - const result = await service.getParentThreadId(REPLY_TO, SPACE_ID) + const result = await service.getParentThreadId(REPLY_TO, SPACE_ID, EMAIL) // Verify behavior - expect(mockKeyValueClient.getValue).toHaveBeenCalledWith(`mail-thread-lookup:${REPLY_TO}:${SPACE_ID}`) + expect(mockKeyValueClient.getValue).toHaveBeenCalledWith(`mail-thread-lookup:${REPLY_TO}:${SPACE_ID}:${EMAIL}`) expect(result).toBe(PARENT_THREAD_ID) }) }) @@ -226,7 +230,7 @@ describe('ThreadLookupService', () => { it('should delete mapping from KVS', async () => { // Get service and call method const service = ThreadLookupService.getInstance(mockCtx, mockKeyValueClient, TOKEN) - await service.deleteMapping(MAIL_ID, SPACE_ID) + await service.deleteMapping(MAIL_ID, SPACE_ID, EMAIL) // Verify behavior expect(mockKeyValueClient.deleteKey).toHaveBeenCalledWith(KEY) @@ -246,7 +250,7 @@ describe('ThreadLookupService', () => { // Get service and call method const service = ThreadLookupService.getInstance(mockCtx, mockKeyValueClient, TOKEN) - await service.deleteMapping(MAIL_ID, SPACE_ID) + await service.deleteMapping(MAIL_ID, SPACE_ID, EMAIL) // Verify behavior expect(mockKeyValueClient.deleteKey).toHaveBeenCalledWith(KEY) diff --git a/services/mail/mail-common/src/message.ts b/services/mail/mail-common/src/message.ts index 54ce09f2317..72dd2a07fdf 100644 --- a/services/mail/mail-common/src/message.ts +++ b/services/mail/mail-common/src/message.ts @@ -206,14 +206,14 @@ async function saveMessageToSpaces ( const spaceId = space._id let isReply = false await rateLimiter.add(async () => { - let threadId = await threadLookup.getThreadId(mailId, spaceId) + let threadId = await threadLookup.getThreadId(mailId, spaceId, recipient.email) if (threadId !== undefined) { ctx.info('Message is already in the thread, skip', { mailId, threadId, spaceId }) return } if (inReplyTo !== undefined) { - threadId = await threadLookup.getParentThreadId(inReplyTo, spaceId) + threadId = await threadLookup.getParentThreadId(inReplyTo, spaceId, recipient.email) isReply = threadId !== undefined } const channel = await channelCache.getOrCreateChannel(spaceId, participants, recipient.email, recipient.socialId) @@ -262,7 +262,7 @@ async function saveMessageToSpaces ( const messageId = await createMailMessage(producer, config, messageData, threadId, options) await createFiles(ctx, producer, config, attachments, messageData, threadId, messageId) - await threadLookup.setThreadId(mailId, space._id, threadId) + await threadLookup.setThreadId(mailId, space._id, threadId, recipient.email) }) } await rateLimiter.waitProcessing() diff --git a/services/mail/mail-common/src/thread.ts b/services/mail/mail-common/src/thread.ts index 5b92e5c102b..57499251357 100644 --- a/services/mail/mail-common/src/thread.ts +++ b/services/mail/mail-common/src/thread.ts @@ -54,20 +54,21 @@ export class ThreadLookupService { ThreadLookupService.instances.delete(token) } - async getThreadId (mailId: string, spaceId: Ref): Promise | undefined> { + async getThreadId (mailId: string, spaceId: Ref, email: string): Promise | undefined> { try { if (mailId == null || spaceId == null) { this.ctx.warn('Invalid parameters for thread lookup', { mailId, spaceId }) return undefined } - const key = this.getLookupKey(mailId, spaceId) + const key = this.getLookupKey(mailId, spaceId, email) const lookup = await this.keyValueClient.getValue(key) if (lookup !== null) { this.ctx.info('Found existing thread mapping', { mailId, spaceId, - threadId: lookup.threadId + threadId: lookup.threadId, + email }) return lookup.threadId } @@ -79,9 +80,9 @@ export class ThreadLookupService { } } - async setThreadId (mailId: string, spaceId: Ref, threadId: Ref): Promise { + async setThreadId (mailId: string, spaceId: Ref, threadId: Ref, email: string): Promise { try { - const key = this.getLookupKey(mailId, spaceId) + const key = this.getLookupKey(mailId, spaceId, email) const lookupInfo: ThreadInfo = { threadId @@ -99,17 +100,21 @@ export class ThreadLookupService { } } - async getParentThreadId (inReplyTo: string | undefined, spaceId: Ref): Promise | undefined> { + async getParentThreadId ( + inReplyTo: string | undefined, + spaceId: Ref, + email: string + ): Promise | undefined> { if (inReplyTo === undefined) { return undefined } - return await this.getThreadId(inReplyTo, spaceId) + return await this.getThreadId(inReplyTo, spaceId, email) } - async deleteMapping (mailId: string, spaceId: Ref): Promise { + async deleteMapping (mailId: string, spaceId: Ref, email: string): Promise { try { - const key = this.getLookupKey(mailId, spaceId) + const key = this.getLookupKey(mailId, spaceId, email) await this.keyValueClient.deleteKey(key) this.ctx.info('Deleted thread mapping', { mailId, spaceId }) } catch (error) { @@ -117,7 +122,7 @@ export class ThreadLookupService { } } - private getLookupKey (mailId: string, spaceId: Ref): string { - return `mail-thread-lookup:${mailId}:${spaceId}` + private getLookupKey (mailId: string, spaceId: Ref, email: string): string { + return `mail-thread-lookup:${mailId}:${spaceId}:${email}` } } diff --git a/services/mail/mail-common/src/txHandler.ts b/services/mail/mail-common/src/txHandler.ts index ebfddccfb44..9fba73c5675 100644 --- a/services/mail/mail-common/src/txHandler.ts +++ b/services/mail/mail-common/src/txHandler.ts @@ -13,7 +13,15 @@ // limitations under the License. // -import core, { Tx, TxDomainEvent, TxOperations } from '@hcengineering/core' +import core, { + Tx, + TxDomainEvent, + TxOperations, + TxCreateDoc, + PersonId, + SocialIdType, + MeasureContext +} from '@hcengineering/core' import { CreateMessageEvent, MessageEventType } from '@hcengineering/communication-sdk-types' import chat from '@hcengineering/chat' @@ -22,7 +30,8 @@ import { Card } from '@hcengineering/card' import mail from '@hcengineering/mail' import { normalizeEmail } from './utils' -import { COMMUNICATION_DOMAIN } from './types' +import { COMMUNICATION_DOMAIN, MailRecipients } from './types' +import { AccountClient } from '@hcengineering/account-client' export function toMessageEvent (tx: Tx): CreateMessageEvent | undefined { if (tx._class !== core.class.TxDomainEvent) { @@ -42,7 +51,41 @@ export function toMessageEvent (tx: Tx): CreateMessageEvent | undefined { return event } +export function isNewChannelTx (tx: Tx): boolean { + if (tx._class !== core.class.TxCreateDoc) { + return false + } + const createTx = tx as TxCreateDoc + return createTx.objectClass === chat.masterTag.Channel +} + export async function getChannel (client: TxOperations, email: string): Promise { const normalizedEmail = normalizeEmail(email) return await client.findOne(mail.tag.MailChannel, { title: normalizedEmail }) } + +export async function getRecipients ( + ctx: MeasureContext, + accountClient: AccountClient, + thread: Card, + personId: PersonId +): Promise { + const collaborators: PersonId[] = (thread as any).members ?? [] + if (collaborators.length === 0) { + return undefined + } + const recipients = collaborators.length > 1 ? collaborators.filter((c) => c !== personId) : collaborators + const mailSocialIds = (await accountClient.findFullSocialIds(recipients)).filter( + (id) => id.type === SocialIdType.EMAIL + ) + if (mailSocialIds.length === 0) { + ctx.warn('No social IDs found for recipients', { recipients }) + return undefined + } + const to = mailSocialIds[0].value + const copy = mailSocialIds.length > 1 ? mailSocialIds.slice(1).map((s) => s.value) : undefined + return { + to, + copy + } +} diff --git a/services/mail/mail-common/src/types.ts b/services/mail/mail-common/src/types.ts index 8793e092c39..6951b6aa1f7 100644 --- a/services/mail/mail-common/src/types.ts +++ b/services/mail/mail-common/src/types.ts @@ -82,4 +82,13 @@ export interface SyncOptions { noNotify?: boolean } +export interface MailRecipients { + to: string + copy?: string[] +} + export const COMMUNICATION_DOMAIN = 'communication' as OperationDomain + +export const HulyMailHeader = 'X-Huly-Sent' +export const HulyMessageIdHeader = 'X-Huly-Message-Id' +export const HulyMessageTypeHeader = 'X-Huly-Message-Type' diff --git a/services/mail/mail-common/src/utils.ts b/services/mail/mail-common/src/utils.ts index e0034532de6..fb5e07e4a2e 100644 --- a/services/mail/mail-common/src/utils.ts +++ b/services/mail/mail-common/src/utils.ts @@ -17,7 +17,14 @@ import sanitizeHtml from 'sanitize-html' import { imageSize } from 'image-size' import { BlobMetadata, MeasureContext } from '@hcengineering/core' -import { Attachment, EmailContact, EmailMessage } from './types' +import { + Attachment, + EmailContact, + EmailMessage, + HulyMailHeader, + HulyMessageIdHeader, + HulyMessageTypeHeader +} from './types' import { MessageExtra } from '@hcengineering/communication-types' import { CreateMessageEvent } from '@hcengineering/communication-sdk-types' @@ -194,3 +201,20 @@ export function getReplySubject (threadName: string | undefined): string | undef return `Re: ${trimmedSubject}` } + +export function getMailHeaders (messageType: string, messageId?: string | undefined): string[] { + const headers = [`${HulyMailHeader}: true\n`, `${HulyMessageTypeHeader}: ${messageType}\n`] + if (messageId !== undefined) { + headers.push(`${HulyMessageIdHeader}: ${messageId}\n`) + } + return headers +} + +export function isHulyMessage (headers: string[]): boolean { + return headers.some( + (header) => + header.startsWith(HulyMailHeader) || + header.startsWith(HulyMessageIdHeader) || + header.startsWith(HulyMessageTypeHeader) + ) +} diff --git a/services/mail/pod-inbound-mail/.eslintrc.js b/services/mail/pod-mail-worker/.eslintrc.js similarity index 100% rename from services/mail/pod-inbound-mail/.eslintrc.js rename to services/mail/pod-mail-worker/.eslintrc.js diff --git a/services/mail/pod-inbound-mail/.gitignore b/services/mail/pod-mail-worker/.gitignore similarity index 100% rename from services/mail/pod-inbound-mail/.gitignore rename to services/mail/pod-mail-worker/.gitignore diff --git a/services/mail/pod-inbound-mail/.npmignore b/services/mail/pod-mail-worker/.npmignore similarity index 100% rename from services/mail/pod-inbound-mail/.npmignore rename to services/mail/pod-mail-worker/.npmignore diff --git a/services/mail/pod-inbound-mail/Dockerfile b/services/mail/pod-mail-worker/Dockerfile similarity index 100% rename from services/mail/pod-inbound-mail/Dockerfile rename to services/mail/pod-mail-worker/Dockerfile diff --git a/services/mail/pod-mail-worker/README.md b/services/mail/pod-mail-worker/README.md new file mode 100644 index 00000000000..c25cdddec39 --- /dev/null +++ b/services/mail/pod-mail-worker/README.md @@ -0,0 +1,82 @@ +# Pod Mail Worker + +Pod Mail Worker is a service that provides bidirectional synchronization between Huly messages and email servers. + +## Purpose + +This service acts as a bridge between Huly's internal messaging system and external email infrastructure, enabling: + +- **Incoming Email Processing**: Receives emails via MTA hooks and converts them into Huly messages +- **Outgoing Email Synchronization**: Processes Huly messages and sends them as emails +- **Queue Processing**: Handles asynchronous message processing via Kafka queues + +## Key Components + +### MTA Hook Handler (`/mta-hook`) +- Receives incoming emails from mail transfer agents +- Parses email content (plain text, HTML, attachments) +- Converts emails to Huly message format +- Handles email threading via In-Reply-To headers + +### Mail Worker +- Processes queued mail operations +- Manages workspace client connections +- Handles bulk email processing operations + +## Configuration + +Key environment variables: + +- `PORT`: Service port (default: 4050) +- `WORKSPACE_URL`: Target Huly workspace URL +- `ACCOUNTS_URL`: Huly accounts service URL +- `KVS_URL`: Key-value store URL for thread mapping +- `QUEUE_CONFIG`: Kafka queue configuration +- `HOOK_TOKEN`: Authentication token for MTA hooks +- `IGNORED_ADDRESSES`: Comma-separated list of email addresses to ignore +- `MAIL_SIZE_LIMIT`: Maximum email size (default: 50mb) + +## API Endpoints + +### POST /mta-hook +Receives incoming emails from mail transfer agents. + +**Headers:** +- `x-hook-token`: Authentication token (if configured) + +**Body:** MTA message format with envelope and message data + +**Response:** Always returns `200 OK` with `{ action: 'accept' }` to prevent email bounces + +## Dependencies + +- **Kafka**: Message queue for asynchronous processing +- **KVS**: Key-value store for thread mapping persistence +- **Workspace API**: Huly workspace integration +- **Account Client**: User and workspace management + +## Development + +```bash +# Install dependencies +rush update + +# Build +rushx build + +# Run tests +rushx test + +# Start development server +rushx run-local + +``` + +## Deployment + +The service is designed to run as a containerized application with the following requirements: + +- Network access to Huly workspace APIs +- Connection to Kafka message queues +- Access to key-value store for persistence +- Ability to receive HTTP requests from mail servers diff --git a/services/mail/pod-inbound-mail/build.sh b/services/mail/pod-mail-worker/build.sh similarity index 100% rename from services/mail/pod-inbound-mail/build.sh rename to services/mail/pod-mail-worker/build.sh diff --git a/services/mail/pod-inbound-mail/config/rig.json b/services/mail/pod-mail-worker/config/rig.json similarity index 100% rename from services/mail/pod-inbound-mail/config/rig.json rename to services/mail/pod-mail-worker/config/rig.json diff --git a/services/mail/pod-inbound-mail/jest.config.js b/services/mail/pod-mail-worker/jest.config.js similarity index 100% rename from services/mail/pod-inbound-mail/jest.config.js rename to services/mail/pod-mail-worker/jest.config.js diff --git a/services/mail/pod-inbound-mail/package.json b/services/mail/pod-mail-worker/package.json similarity index 85% rename from services/mail/pod-inbound-mail/package.json rename to services/mail/pod-mail-worker/package.json index bb524183373..f373acee1ff 100644 --- a/services/mail/pod-inbound-mail/package.json +++ b/services/mail/pod-mail-worker/package.json @@ -1,5 +1,5 @@ { - "name": "@hcengineering/pod-inbound-mail", + "name": "@hcengineering/pod-mail-worker", "version": "0.6.0", "main": "lib/index.js", "svelte": "src/index.ts", @@ -18,11 +18,11 @@ "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", "bundle": "node ../../../common/scripts/esbuild.js --external=ws", - "docker:build": "../../../common/scripts/docker_build.sh hardcoreeng/inbound-mail", - "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/inbound-mail staging", - "docker:abuild": "docker build -t hardcoreeng/inbound-mail . --platform=linux/arm64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/inbound-mail", - "docker:push": "../../../common/scripts/docker_tag.sh hardcoreeng/inbound-mail", - "run-bundle": "rushx bundle --to @hcengineering/pod-inbound-mail && cross-env NODE_ENV=production node --inspect bundle/bundle.js", + "docker:build": "../../../common/scripts/docker_build.sh hardcoreeng/mail-worker", + "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/mail-worker staging", + "docker:abuild": "docker build -t hardcoreeng/mail-worker . --platform=linux/arm64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/mail-worker", + "docker:push": "../../../common/scripts/docker_tag.sh hardcoreeng/mail-worker", + "run-bundle": "rushx bundle --to @hcengineering/pod-mail-worker && cross-env NODE_ENV=production node --inspect bundle/bundle.js", "run-local": "ts-node src/index.ts", "format": "format src", "_phase:build": "compile transpile src", @@ -61,11 +61,13 @@ "@hcengineering/analytics-service": "^0.6.0", "@hcengineering/api-client": "^0.6.0", "@hcengineering/card": "^0.6.0", + "@hcengineering/chat": "^0.6.0", "@hcengineering/communication-rest-client": "^0.1.0", "@hcengineering/communication-sdk-types": "^0.1.0", "@hcengineering/communication-types": "^0.1.0", "@hcengineering/contact": "^0.6.24", "@hcengineering/core": "^0.6.32", + "@hcengineering/kafka": "^0.6.0", "@hcengineering/kvs-client": "^0.6.0", "@hcengineering/mail": "^0.6.0", "@hcengineering/mail-common": "^0.6.0", diff --git a/services/mail/pod-inbound-mail/src/__tests__/__mocks__/2attachments.txt b/services/mail/pod-mail-worker/src/__tests__/__mocks__/2attachments.txt similarity index 100% rename from services/mail/pod-inbound-mail/src/__tests__/__mocks__/2attachments.txt rename to services/mail/pod-mail-worker/src/__tests__/__mocks__/2attachments.txt diff --git a/services/mail/pod-inbound-mail/src/__tests__/__mocks__/attachment.txt b/services/mail/pod-mail-worker/src/__tests__/__mocks__/attachment.txt similarity index 100% rename from services/mail/pod-inbound-mail/src/__tests__/__mocks__/attachment.txt rename to services/mail/pod-mail-worker/src/__tests__/__mocks__/attachment.txt diff --git a/services/mail/pod-inbound-mail/src/__tests__/__mocks__/base64Message.json b/services/mail/pod-mail-worker/src/__tests__/__mocks__/base64Message.json similarity index 100% rename from services/mail/pod-inbound-mail/src/__tests__/__mocks__/base64Message.json rename to services/mail/pod-mail-worker/src/__tests__/__mocks__/base64Message.json diff --git a/services/mail/pod-inbound-mail/src/__tests__/decode.test.ts b/services/mail/pod-mail-worker/src/__tests__/decode.test.ts similarity index 100% rename from services/mail/pod-inbound-mail/src/__tests__/decode.test.ts rename to services/mail/pod-mail-worker/src/__tests__/decode.test.ts diff --git a/services/mail/pod-inbound-mail/src/__tests__/handlerMta.test.ts b/services/mail/pod-mail-worker/src/__tests__/handlerMta.test.ts similarity index 99% rename from services/mail/pod-inbound-mail/src/__tests__/handlerMta.test.ts rename to services/mail/pod-mail-worker/src/__tests__/handlerMta.test.ts index b39b052ab18..5c8fe9a1f5e 100644 --- a/services/mail/pod-inbound-mail/src/__tests__/handlerMta.test.ts +++ b/services/mail/pod-mail-worker/src/__tests__/handlerMta.test.ts @@ -27,7 +27,9 @@ import { type MtaMessage } from '../types' // Mock dependencies jest.mock('@hcengineering/mail-common', () => ({ createMessages: jest.fn(), - getProducer: jest.fn().mockReturnValue({}) + getProducer: jest.fn().mockReturnValue({}), + getMessageExtra: jest.fn().mockReturnValue({}), + isHulyMessage: jest.fn().mockReturnValue(false) })) jest.mock('../client', () => ({ diff --git a/services/mail/pod-inbound-mail/src/__tests__/parseMail.test.ts b/services/mail/pod-mail-worker/src/__tests__/parseMail.test.ts similarity index 100% rename from services/mail/pod-inbound-mail/src/__tests__/parseMail.test.ts rename to services/mail/pod-mail-worker/src/__tests__/parseMail.test.ts diff --git a/services/mail/pod-inbound-mail/src/__tests__/utils.test.ts b/services/mail/pod-mail-worker/src/__tests__/utils.test.ts similarity index 100% rename from services/mail/pod-inbound-mail/src/__tests__/utils.test.ts rename to services/mail/pod-mail-worker/src/__tests__/utils.test.ts diff --git a/services/mail/pod-inbound-mail/src/client.ts b/services/mail/pod-mail-worker/src/client.ts similarity index 64% rename from services/mail/pod-inbound-mail/src/client.ts rename to services/mail/pod-mail-worker/src/client.ts index 7745f6d6ef3..1ac85caaac0 100644 --- a/services/mail/pod-inbound-mail/src/client.ts +++ b/services/mail/pod-mail-worker/src/client.ts @@ -13,15 +13,21 @@ // limitations under the License. // -import { systemAccountUuid } from '@hcengineering/core' +import { systemAccountUuid, WorkspaceUuid } from '@hcengineering/core' import { BaseConfig } from '@hcengineering/mail-common' import { generateToken } from '@hcengineering/server-token' import { getClient } from '@hcengineering/kvs-client' +import { AccountClient, getClient as getAccountClientRaw } from '@hcengineering/account-client' import config from './config' // TODO: Find account UUID from mailboxes and use personal workspace -export const mailServiceToken = generateToken(systemAccountUuid, undefined, { service: 'inbound-mail' }, config.secret) +export const mailServiceToken = generateToken( + systemAccountUuid, + undefined, + { service: config.serviceId }, + config.secret +) export const baseConfig: BaseConfig = { AccountsURL: config.accountsUrl, KvsUrl: config.kvsUrl, @@ -30,4 +36,9 @@ export const baseConfig: BaseConfig = { QueueRegion: config.queueRegion ?? '', CommunicationTopic: config.communicationTopic } -export const kvsClient = getClient('inbound-mail', baseConfig.KvsUrl, mailServiceToken) +export const kvsClient = getClient(config.serviceId, baseConfig.KvsUrl, mailServiceToken) + +export function getAccountClient (workspaceUuid?: WorkspaceUuid): AccountClient { + const token = generateToken(systemAccountUuid, workspaceUuid, { service: config.serviceId }) + return getAccountClientRaw(config.accountsUrl, token) +} diff --git a/services/mail/pod-inbound-mail/src/config.ts b/services/mail/pod-mail-worker/src/config.ts similarity index 86% rename from services/mail/pod-inbound-mail/src/config.ts rename to services/mail/pod-mail-worker/src/config.ts index fa4a695ae00..4e52827c3b4 100644 --- a/services/mail/pod-inbound-mail/src/config.ts +++ b/services/mail/pod-mail-worker/src/config.ts @@ -29,6 +29,9 @@ interface Config { queueConfig: string queueRegion: string communicationTopic: string + serviceId: string + mailUrl: string + mailAuth: string } const config: Config = { @@ -58,7 +61,15 @@ const config: Config = { throw Error('QUEUE_CONFIG env var is not set') })(), queueRegion: process.env.QUEUE_REGION ?? '', - communicationTopic: process.env.COMMUNICATION_TOPIC ?? 'hulygun' + communicationTopic: process.env.COMMUNICATION_TOPIC ?? 'hulygun', + serviceId: process.env.SERVICE_ID ?? 'huly-mail', + mailUrl: (() => { + if (process.env.MAIL_URL !== undefined) { + return process.env.MAIL_URL + } + throw Error('MAIL_URL env var is not set') + })(), + mailAuth: process.env.MAIL_AUTH ?? '' } export default config diff --git a/services/mail/pod-inbound-mail/src/decode.ts b/services/mail/pod-mail-worker/src/decode.ts similarity index 100% rename from services/mail/pod-inbound-mail/src/decode.ts rename to services/mail/pod-mail-worker/src/decode.ts diff --git a/services/mail/pod-inbound-mail/src/handlerMta.ts b/services/mail/pod-mail-worker/src/handlerMta.ts similarity index 92% rename from services/mail/pod-inbound-mail/src/handlerMta.ts rename to services/mail/pod-mail-worker/src/handlerMta.ts index cf3fc4a9acd..7c0d935eb7e 100644 --- a/services/mail/pod-inbound-mail/src/handlerMta.ts +++ b/services/mail/pod-mail-worker/src/handlerMta.ts @@ -15,13 +15,20 @@ import { createHash } from 'crypto' import { Request, Response } from 'express' import { MeasureContext } from '@hcengineering/core' -import { type EmailContact, type EmailMessage, createMessages, getProducer } from '@hcengineering/mail-common' +import { + type EmailContact, + type EmailMessage, + createMessages, + getProducer, + getMessageExtra, + isHulyMessage +} from '@hcengineering/mail-common' import { getClient as getAccountClient } from '@hcengineering/account-client' import { createRestTxOperations } from '@hcengineering/api-client' import { mailServiceToken, baseConfig, kvsClient } from './client' import config from './config' -import { MtaMessage } from './types' +import { MtaMessage, HulyMessageType } from './types' import { getHeader, parseContent } from './utils' import { decodeEncodedWords } from './decode' @@ -36,6 +43,11 @@ export async function handleMtaHook (req: Request, res: Response, ctx: MeasureCo const mta: MtaMessage = req.body + const headers: string[] = mta.message.headers.map((header) => header[0].trim()) ?? [] + if (isHulyMessage(headers)) { + return + } + const from: EmailContact = getEmailContact(mta.envelope.from.address) if (config.ignoredAddresses.includes(from.email)) { return @@ -89,7 +101,8 @@ export async function handleMtaHook (req: Request, res: Response, ctx: MeasureCo replyTo: inReplyTo, incoming: true, modifiedOn: date, - sendOn: date + sendOn: date, + extra: getMessageExtra(HulyMessageType, true) } const accountClient = getAccountClient(config.accountsUrl, mailServiceToken) diff --git a/services/mail/pod-inbound-mail/src/index.ts b/services/mail/pod-mail-worker/src/index.ts similarity index 85% rename from services/mail/pod-inbound-mail/src/index.ts rename to services/mail/pod-mail-worker/src/index.ts index 33b723cdb91..ea766967e5f 100644 --- a/services/mail/pod-inbound-mail/src/index.ts +++ b/services/mail/pod-mail-worker/src/index.ts @@ -26,18 +26,19 @@ import { join } from 'path' import { baseConfig } from './client' import config from './config' import { handleMtaHook } from './handlerMta' +import { MailWorker } from './mailWorker' type RequestHandler = (req: Request, res: Response, ctx: MeasureContext, next?: NextFunction) => Promise async function main (): Promise { - const ctx = initStatisticsContext('inbound-mail', { + const ctx = initStatisticsContext(config.serviceId, { factory: () => createOpenTelemetryMetricsContext( - 'inbound-mail', + config.serviceId, {}, {}, newMetrics(), - new SplitLogger('inbound-mail', { + new SplitLogger(config.serviceId, { root: join(process.cwd(), 'logs'), enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' }) @@ -45,9 +46,9 @@ async function main (): Promise { }) setMetadata(serverToken.metadata.Secret, config.secret) - setMetadata(serverToken.metadata.Service, 'inbound-mail') + setMetadata(serverToken.metadata.Service, config.serviceId) - initQueue(ctx, 'inbound-mail', baseConfig) + initQueue(ctx, config.serviceId, baseConfig) const app = express() @@ -94,8 +95,18 @@ async function main (): Promise { storageConfig: config.storageConfig !== undefined ? '(stripped)' : undefined }) }) + await MailWorker.create(ctx) const shutdown = (): void => { + try { + MailWorker.getMailWorker() + .close() + .catch((err) => { + ctx.error('Failed to close MailWorker', { error: err.message }) + }) + } catch (error) { + ctx.error('Failed to get MailWorker for shutdown', { error: (error as Error).message }) + } server.close(() => { void closeQueue().then(() => process.exit()) }) diff --git a/services/mail/pod-mail-worker/src/mailWorker.ts b/services/mail/pod-mail-worker/src/mailWorker.ts new file mode 100644 index 00000000000..fdb8db40427 --- /dev/null +++ b/services/mail/pod-mail-worker/src/mailWorker.ts @@ -0,0 +1,282 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { MeasureContext, WorkspaceUuid, Doc, TxCUD, Tx } from '@hcengineering/core' +import { + toMessageEvent, + isNewChannelTx, + markdownToHtml, + getRecipients, + getReplySubject, + markdownToText, + isSyncedMessage, + getMailHeaders +} from '@hcengineering/mail-common' +import { ConsumerHandle, PlatformQueue, QueueTopic } from '@hcengineering/server-core' +import { getPlatformQueue } from '@hcengineering/kafka' +import { CreateMessageEvent } from '@hcengineering/communication-sdk-types' +import chat from '@hcengineering/chat' +import { Card } from '@hcengineering/card' + +import config from './config' +import { AccountClient, MailboxOptions } from '@hcengineering/account-client' +import { getAccountClient } from './client' +import { getClient as getWorkspaceClient, releaseClient } from './workspaceClient' +import { sendEmail } from './send' +import { HulyMessageType } from './types' + +export class MailWorker { + private queue: PlatformQueue | undefined + private txConsumer: ConsumerHandle | undefined + private mailboxOptions: MailboxOptions | undefined + private loadingPromise: Promise | undefined + + protected static _instance: MailWorker + + private constructor ( + private readonly ctx: MeasureContext, + private readonly accountClient: AccountClient + ) {} + + static async create (ctx: MeasureContext): Promise { + if (MailWorker._instance !== undefined) { + throw new Error('MailWorker already exists') + } + // Create account client for system operations + const accountClient = getAccountClient() + const worker = new MailWorker(ctx, accountClient) + MailWorker._instance = worker + + // Request mailbox options after creation + await worker.loadMailboxes() + await worker.startQueue() + + return worker + } + + static getMailWorker (): MailWorker { + if (MailWorker._instance !== undefined) { + return MailWorker._instance + } + throw new Error('MailWorker not exist') + } + + private async loadMailboxes (): Promise { + // If already loading, wait for the existing promise + if (this.loadingPromise !== undefined) { + await this.loadingPromise + return + } + this.loadingPromise = this.performLoad() + + try { + await this.loadingPromise + } finally { + this.loadingPromise = undefined + } + } + + private async performLoad (): Promise { + try { + this.ctx.info('Loading mailbox options...') + this.mailboxOptions = await this.accountClient.getMailboxOptions() + this.ctx.info('Mailbox options loaded', { + maxMailboxCount: this.mailboxOptions.maxMailboxCount, + availableDomainsCount: this.mailboxOptions.availableDomains.length + }) + } catch (err: any) { + this.ctx.error('Failed to load mailbox options', { error: err.message }) + throw err + } + } + + async startQueue (): Promise { + try { + if (config.queueRegion === undefined) { + this.ctx.info('Queue region not configured, skipping queue consumer setup') + return + } + + this.ctx.info('Starting queue consumer for mail worker', { + region: config.queueRegion + }) + + this.queue = getPlatformQueue(config.serviceId, config.queueRegion) + if (this.queue === undefined) { + this.ctx.error('Queue not found') + return + } + + this.txConsumer = this.queue.createConsumer>( + this.ctx, + QueueTopic.Tx, + this.queue.getClientId(), + async (msgs) => { + for (const msg of msgs) { + const workspaceUuid = msg.workspace + for (const tx of msg.value) { + // Check for new channel creation + if (isNewChannelTx(tx)) { + await this.handleNewChannelTx(workspaceUuid, tx) + continue + } + // Check for message events + const messageEvent = toMessageEvent(tx) + if (messageEvent !== undefined) { + await this.handleNewMessage(workspaceUuid, messageEvent) + } + } + } + }, + { + fromBegining: false + } + ) + + this.ctx.info('Queue consumer started successfully', { region: config.queueRegion }) + } catch (err: any) { + this.ctx.error('Failed to start queue consumer', { error: err.message }) + } + } + + /** + * Handle new channel creation transaction + */ + private async handleNewChannelTx (workspaceUuid: WorkspaceUuid, tx: Tx): Promise { + try { + this.ctx.info('New channel created, updating mailbox options', { + workspaceUuid, + txId: tx._id + }) + + // Reload mailbox options when new channels are created + await this.loadMailboxes() + } catch (err: any) { + this.ctx.error('Failed to handle new channel transaction', { + workspaceUuid, + txId: tx._id, + error: err.message + }) + } + } + + async handleNewMessage (workspaceUuid: WorkspaceUuid, message: CreateMessageEvent): Promise { + try { + if (isSyncedMessage(message)) { + return + } + + // Await any active load/reload promise before processing CreateMessageEvent + if (this.loadingPromise !== undefined) { + await this.loadingPromise + } + + try { + const workspaceClient = await getWorkspaceClient(workspaceUuid) + const thread = await workspaceClient.findOne(chat.masterTag.Thread, { _id: message.cardId }) + if (thread?.parent == null) { + return + } + const channel = await workspaceClient.findOne(chat.masterTag.Channel, { _id: thread.parent }) + if (channel === undefined || !this.isHulyMailChannel(channel)) { + return + } + + // Convert the platform message to email format and send + await this.sendMessageAsEmail(message, thread, channel, workspaceUuid) + } finally { + await releaseClient(this.ctx, workspaceUuid) + } + } catch (err: any) { + this.ctx.error('Failed to handle new message', { + workspaceUuid, + messageId: message.messageId, + error: err.message + }) + } + } + + private async sendMessageAsEmail ( + message: CreateMessageEvent, + thread: Card, + channel: Card, + workspaceUuid: WorkspaceUuid + ): Promise { + try { + this.ctx.info('Sending email message', { + workspaceUuid, + messageId: message.messageId, + socialId: message.socialId + }) + + const personUuid = await this.accountClient.findPersonBySocialId(message.socialId) + if (personUuid === undefined) { + this.ctx.error('Person not found for social ID', { socialId: message.socialId }) + return + } + const socialIds = await this.accountClient.getPersonInfo(personUuid) + const emailSocialId = socialIds.socialIds.find((id) => id.value.toLowerCase() === channel.title.toLowerCase()) + if (emailSocialId === undefined) { + this.ctx.error('Email social ID not found for channel', { + channelTitle: channel.title, + personUuid, + socialIds: socialIds.socialIds.map((id) => id.value) + }) + return + } + const recipients = await getRecipients(this.ctx, this.accountClient, thread, emailSocialId._id) + const html = markdownToHtml(message.content) + const text = markdownToText(message.content) + const subject = getReplySubject(thread.title) ?? '' + const to = [recipients?.to, ...(recipients?.copy ?? [])].filter( + (address) => address != null && address !== '' + ) as string[] + + await sendEmail(this.ctx, { + from: emailSocialId.value, + to, + subject, + html, + text, + headers: getMailHeaders(HulyMessageType, message._id) + }) + } catch (err: any) { + this.ctx.error('Failed to send message as email', { + messageId: message.messageId, + error: err.message + }) + throw err + } + } + + async close (): Promise { + if (this.txConsumer !== undefined) { + await this.txConsumer.close() + } + if (this.queue !== undefined) { + await this.queue.shutdown() + } + this.ctx.info('Mail worker closed') + } + + isHulyMailChannel (channel: Card): boolean { + const title = channel.title.toLowerCase() + const domains = this.mailboxOptions?.availableDomains + if (domains === undefined || domains.length === 0) { + return false + } + return domains.some((domain) => title.includes(domain.toLowerCase())) + } +} diff --git a/services/mail/pod-mail-worker/src/send.ts b/services/mail/pod-mail-worker/src/send.ts new file mode 100644 index 00000000000..d53b23c3c5b --- /dev/null +++ b/services/mail/pod-mail-worker/src/send.ts @@ -0,0 +1,32 @@ +import { concatLink, MeasureContext } from '@hcengineering/core' + +import config from './config' +import { MailMessage } from './types' + +export async function sendEmail (ctx: MeasureContext, message: MailMessage): Promise { + const mailURL = config.mailUrl + if (mailURL === undefined || mailURL === '') { + ctx.error('Please provide email service url to enable email sending') + return + } + const mailAuth = config.mailAuth + + const response = await fetch(concatLink(mailURL, '/send'), { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(mailAuth != null ? { Authorization: `Bearer ${mailAuth}` } : {}) + }, + body: JSON.stringify({ + ...message, + apiKey: mailAuth + }) + }) + if (!response.ok) { + const responseBody = await response.text() + ctx.error(`Failed to send email: ${response.status} ${response.statusText}. Response body: ${responseBody}`, { + to: message.to, + from: message.from + }) + } +} diff --git a/services/mail/pod-inbound-mail/src/types.ts b/services/mail/pod-mail-worker/src/types.ts similarity index 73% rename from services/mail/pod-inbound-mail/src/types.ts rename to services/mail/pod-mail-worker/src/types.ts index 12a07d5504c..b73c7f83cc7 100644 --- a/services/mail/pod-inbound-mail/src/types.ts +++ b/services/mail/pod-mail-worker/src/types.ts @@ -27,3 +27,19 @@ export interface MtaMessage { contents: string } } + +export interface MailMessage { + from: string + to: string[] + subject: string + text?: string + html?: string + headers?: Record | string[] + attachments?: { + name: string + contentType: string + data: Buffer + }[] +} + +export const HulyMessageType = 'huly-mail' diff --git a/services/mail/pod-inbound-mail/src/utils.ts b/services/mail/pod-mail-worker/src/utils.ts similarity index 100% rename from services/mail/pod-inbound-mail/src/utils.ts rename to services/mail/pod-mail-worker/src/utils.ts diff --git a/services/mail/pod-mail-worker/src/workspaceClient.ts b/services/mail/pod-mail-worker/src/workspaceClient.ts new file mode 100644 index 00000000000..e4b1bf02456 --- /dev/null +++ b/services/mail/pod-mail-worker/src/workspaceClient.ts @@ -0,0 +1,104 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { getClient as getAccountClient } from '@hcengineering/account-client' +import { createRestTxOperations } from '@hcengineering/api-client' +import core, { MeasureContext, PersonId, systemAccountUuid, TxOperations, WorkspaceUuid } from '@hcengineering/core' +import { generateToken } from '@hcengineering/server-token' +import config from './config' + +export const SERVICE_NAME = 'mail-worker' + +const clients = new Map>() +const clientUsage = new Map() + +export async function getClient (workspaceUuid: WorkspaceUuid, socialId?: PersonId): Promise { + const key = `${workspaceUuid}-${socialId ?? ''}` + let usage = clientUsage.get(key) ?? 0 + usage++ + clientUsage.set(key, usage) + const current = clients.get(key) + if (current !== undefined) { + if (current instanceof Promise) { + return await current + } + return current + } + + const client = createClient(workspaceUuid, socialId) + clients.set(key, client) + const cl = await client + clients.set(key, cl) + return cl +} + +export async function releaseClient ( + ctx: MeasureContext, + workspaceUuid: WorkspaceUuid, + socialId?: PersonId +): Promise { + const key = `${workspaceUuid}-${socialId ?? ''}` + let usage = clientUsage.get(key) + if (usage === undefined) { + ctx.warn(`Client for ${key} not found`) + return + } + usage-- + clientUsage.set(key, usage) + if (usage <= 0) { + setTimeout(() => { + close(key).catch((err) => { + ctx.error(`Failed to close client for ${key}`, err) + }) + }, 60000) + } +} + +async function close (key: string): Promise { + const usage = clientUsage.get(key) ?? 0 + if (usage > 0) return + const current = clients.get(key) + if (current !== undefined) { + clients.delete(key) + if (current instanceof Promise) { + const resolvedClient = await current + await resolvedClient.close() + } else { + await current.close() + } + } +} + +async function createClient (workspaceUuid: WorkspaceUuid, socialId?: PersonId): Promise { + const token = generateToken(systemAccountUuid, workspaceUuid, { service: SERVICE_NAME }) + let accountClient = getAccountClient(config.accountsUrl, token) + + if (socialId !== undefined && socialId !== core.account.System) { + const personUuid = await accountClient.findPersonBySocialId(socialId, true) + if (personUuid === undefined) { + throw new Error('Global person not found') + } + const token = generateToken(personUuid, workspaceUuid, { service: SERVICE_NAME }) + accountClient = getAccountClient(config.accountsUrl, token) + } + + const wsInfo = await accountClient.getLoginInfoByToken() + if (!('endpoint' in wsInfo)) { + throw new Error('Invalid login info') + } + const transactorUrl = wsInfo.endpoint.replace('ws://', 'http://').replace('wss://', 'https://') + const client = await createRestTxOperations(transactorUrl, wsInfo.workspace, wsInfo.token, true) + return client +} diff --git a/services/mail/pod-inbound-mail/tsconfig.json b/services/mail/pod-mail-worker/tsconfig.json similarity index 100% rename from services/mail/pod-inbound-mail/tsconfig.json rename to services/mail/pod-mail-worker/tsconfig.json diff --git a/services/mail/pod-mail/package.json b/services/mail/pod-mail/package.json index 41fcb253b9e..5e344f55f0a 100644 --- a/services/mail/pod-mail/package.json +++ b/services/mail/pod-mail/package.json @@ -43,14 +43,14 @@ "esbuild": "^0.24.2", "prettier": "^3.1.0", "ts-node": "^10.8.0", - "typescript": "^5.8.3", "jest": "^29.7.0", "ts-jest": "^29.1.1", "@types/jest": "^29.5.5", "@tsconfig/node16": "^1.0.4", "@types/cors": "^2.8.12", "@types/express": "^4.17.13", - "eslint-plugin-node": "^11.1.0" + "eslint-plugin-node": "^11.1.0", + "typescript": "^5.8.3" }, "dependencies": { "@aws-sdk/client-ses": "^3.738.0", diff --git a/services/mail/pod-mail/src/__tests__/main.test.ts b/services/mail/pod-mail/src/__tests__/main.test.ts index a5524f73aeb..560fe74c2a6 100644 --- a/services/mail/pod-mail/src/__tests__/main.test.ts +++ b/services/mail/pod-mail/src/__tests__/main.test.ts @@ -53,6 +53,7 @@ describe('handleSendMail', () => { sendMailMock = (mailClient.sendMessage as jest.Mock).mockResolvedValue({}) mockCtx = { info: jest.fn(), + warn: jest.fn(), error: jest.fn() } as unknown as MeasureContext }) @@ -64,7 +65,7 @@ describe('handleSendMail', () => { // eslint-disable-next-line @typescript-eslint/unbound-method expect(res.status).toHaveBeenCalledWith(400) - expect(res.send).toHaveBeenCalledWith({ err: "'text' is missing" }) + expect(res.send).toHaveBeenCalledWith({ err: "'text' and 'html' are missing" }) }) it('should return 400 if subject is missing', async () => { diff --git a/services/mail/pod-mail/src/main.ts b/services/mail/pod-mail/src/main.ts index 1b70994c060..cad46e24b78 100644 --- a/services/mail/pod-mail/src/main.ts +++ b/services/mail/pod-mail/src/main.ts @@ -83,23 +83,31 @@ export async function handleSendMail ( ): Promise { const { from, to, subject, text, html, attachments, headers, apiKey } = req.body if (process.env.API_KEY !== undefined && process.env.API_KEY !== apiKey) { + ctx.warn('Unauthorized access attempt to send email', { + from, + to + }) res.status(401).send({ err: 'Unauthorized' }) return } const fromAddress = from ?? config.source - if (text === undefined) { - res.status(400).send({ err: "'text' is missing" }) + if (text === undefined && html === undefined) { + ctx.warn('Text and html are missing in email request', { from, to }) + res.status(400).send({ err: "'text' and 'html' are missing" }) return } if (subject === undefined) { + ctx.warn('Subject is missing in email request', { from, to }) res.status(400).send({ err: "'subject' is missing" }) return } if (to === undefined) { + ctx.warn('To address is missing in email request', { from }) res.status(400).send({ err: "'to' is missing" }) return } if (fromAddress === undefined) { + ctx.warn('From address is missing in email request', { to }) res.status(400).send({ err: "'from' is missing" }) return }