Skip to content

Commit 0161035

Browse files
UBERF-13123: Fix mail message duplicates (#9684)
* UBERF-13123: Fix mail message duplicates Signed-off-by: Artem Savchenko <[email protected]> * UBERF-13123: Fix failed tests Signed-off-by: Artem Savchenko <[email protected]> * UBERF-13123: Refactor message headers Signed-off-by: Artem Savchenko <[email protected]> * UBERF-13123: Clean up Signed-off-by: Artem Savchenko <[email protected]> * UBERF-13123: Skip old messages Signed-off-by: Artem Savchenko <[email protected]> * UBERF-13123: decrease cache size Signed-off-by: Artem Savchenko <[email protected]> --------- Signed-off-by: Artem Savchenko <[email protected]>
1 parent 5f600e2 commit 0161035

File tree

16 files changed

+359
-82
lines changed

16 files changed

+359
-82
lines changed

common/config/rush/pnpm-lock.yaml

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

services/gmail/pod-gmail/src/config.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ interface Config extends BaseConfig {
2626
Credentials: string
2727
WATCH_TOPIC_NAME: string
2828
FooterMessage: string
29+
OutgoingSyncStartDate: Date // ISO date string - messages from mail chanel before this date will not attempt to be sent to Gmail
2930
InitLimit: number
3031
Version: IntegrationVersion
3132
QueueConfig: string
@@ -43,6 +44,7 @@ const envMap: { [key in keyof Config]: string } = {
4344
Credentials: 'Credentials',
4445
WATCH_TOPIC_NAME: 'WATCH_TOPIC_NAME',
4546
FooterMessage: 'FOOTER_MESSAGE',
47+
OutgoingSyncStartDate: 'OUTGOING_SYNC_START_DATE',
4648
InitLimit: 'INIT_LIMIT',
4749
KvsUrl: 'KVS_URL',
4850
StorageConfig: 'STORAGE_CONFIG',
@@ -72,6 +74,7 @@ const config: Config = (() => {
7274
WATCH_TOPIC_NAME: process.env[envMap.WATCH_TOPIC_NAME],
7375
InitLimit: parseNumber(process.env[envMap.InitLimit]) ?? 50,
7476
FooterMessage: process.env[envMap.FooterMessage] ?? '<br><br><p>Sent via <a href="https://huly.io">Huly</a></p>',
77+
OutgoingSyncStartDate: new Date(process.env[envMap.OutgoingSyncStartDate] ?? '2025-08-20T00:00:00.000Z'),
7578
KvsUrl: process.env[envMap.KvsUrl],
7679
StorageConfig: process.env[envMap.StorageConfig],
7780
Version: version,

services/gmail/pod-gmail/src/gmail.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,12 @@ export class GmailClient {
367367
if (personId !== this.socialId._id && !this.allSocialIds.has(personId)) {
368368
return
369369
}
370+
if (message.date !== undefined) {
371+
const messageDate = message.date instanceof Date ? message.date : new Date(message.date)
372+
if (messageDate < config.OutgoingSyncStartDate) {
373+
return
374+
}
375+
}
370376
const email = await this.getEmail()
371377
const thread = await this.client.findOne<Card>(chat.masterTag.Thread, { _id: message.cardId })
372378
const mailChannel = await this.getMailChannel()

services/gmail/pod-gmail/src/gmailController.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,6 @@ export class GmailController {
161161
async handleNewMessage (workspaceUuid: WorkspaceUuid, message: CreateMessageEvent): Promise<void> {
162162
const client = this.workspaces.get(workspaceUuid)
163163
if (client === undefined) {
164-
this.ctx.warn('No workspace client found', { socialId: message.socialId, workspaceUuid })
165164
return
166165
}
167166
await client.handleNewMessage(message)

services/gmail/pod-gmail/src/message/v1/send.ts

Whitespace-only changes.

services/gmail/pod-gmail/src/message/v2/message.ts

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ import {
2525
getProducer,
2626
MailRecipient,
2727
getMessageExtra,
28-
HulyMailHeader,
29-
HulyMessageIdHeader,
28+
MailHeader,
3029
SyncOptions
3130
} from '@hcengineering/mail-common'
3231
import { type KeyValueClient } from '@hcengineering/kvs-client'
@@ -95,11 +94,11 @@ function getHeaderValue (payload: gmail_v1.Schema$MessagePart | undefined, name:
9594
}
9695

9796
export function isHulyMessage (payload: gmail_v1.Schema$MessagePart | undefined): boolean {
98-
const hulyHeader = getHeaderValue(payload, HulyMailHeader)
97+
const hulyHeader = getHeaderValue(payload, MailHeader.HulySent)
9998
if (hulyHeader !== undefined) {
10099
return true
101100
}
102-
const hulyMessage = getHeaderValue(payload, HulyMessageIdHeader)
101+
const hulyMessage = getHeaderValue(payload, MailHeader.HulyMessageType)
103102
return hulyMessage !== undefined
104103
}
105104

services/gmail/pod-gmail/src/message/v2/send.ts

Lines changed: 6 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
import { CreateMessageEvent } from '@hcengineering/communication-sdk-types'
2-
import { type GaxiosResponse } from 'gaxios'
32
import { gmail_v1 } from 'googleapis'
43
import {
54
markdownToHtml,
65
getReplySubject,
76
getRecipients,
87
getMailHeaders,
9-
HulyMailHeader,
10-
HulyMessageIdHeader
8+
MailHeader,
9+
getEmailMessageIdFromHulyId
1110
} from '@hcengineering/mail-common'
1211
import { Card } from '@hcengineering/card'
1312
import { MeasureContext, PersonId } from '@hcengineering/core'
@@ -36,6 +35,7 @@ export async function makeHTMLBodyV2 (
3635
'Content-Transfer-Encoding: 7bit\n',
3736
`To: ${to} \n`,
3837
`From: ${from} \n`,
38+
`${MailHeader.Id}: ${getEmailMessageIdFromHulyId(message._id, from)}\n`,
3939
...getMailHeaders(GmailMessageType, message._id)
4040
]
4141

@@ -64,34 +64,14 @@ export function isPlatformSentMessage (message: gmail_v1.Schema$Message): boolea
6464

6565
// Check for custom platform headers
6666
const headers = message.payload.headers
67-
const platformSentHeader = headers.find((h) => h.name === HulyMailHeader)
67+
const platformSentHeader = headers.find((h) => h.name === MailHeader.HulySent)
6868
if (platformSentHeader?.value === 'true') {
6969
return true
7070
}
71-
72-
// Check for platform message ID header
73-
const platformMessageIdHeader = headers.find((h) => h.name === HulyMessageIdHeader)
74-
if (platformMessageIdHeader?.value != null) {
71+
const hulyMessageType = headers.find((h) => h.name === MailHeader.HulyMessageType)
72+
if (hulyMessageType?.value != null) {
7573
return true
7674
}
7775

7876
return false
7977
}
80-
81-
/**
82-
* Check if message has platform footer signature
83-
*/
84-
export function hasPlatformFooter (messageBody: string): boolean {
85-
const platformFooterPattern = /Sent via Huly/i
86-
return platformFooterPattern.test(messageBody)
87-
}
88-
89-
/**
90-
* Extract platform message ID from headers if available
91-
*/
92-
export function getPlatformMessageId (message: GaxiosResponse<gmail_v1.Schema$Message>): string | undefined {
93-
const headers = message.data?.payload?.headers
94-
if (headers == null) return undefined
95-
const platformMessageIdHeader = headers.find((h) => h.name === HulyMessageIdHeader)
96-
return platformMessageIdHeader?.value ?? undefined
97-
}
Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
//
2+
// Copyright © 2025 Hardcore Engineering Inc.
3+
//
4+
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License. You may
6+
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
//
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
16+
import { getEmailMessageIdFromHulyId, getHulyIdFromEmailMessageId, isHulyEmailMessageId } from '../utils'
17+
18+
describe('Email Message ID Conversion', () => {
19+
describe('getEmailMessageIdFromHulyId', () => {
20+
it('should convert Huly ID to email Message-ID format', () => {
21+
const hulyId = 'msg_123456789abcdef'
22+
const email = '[email protected]'
23+
const result = getEmailMessageIdFromHulyId(hulyId, email)
24+
expect(result).toBe('<[email protected]>')
25+
})
26+
27+
it('should handle different domains', () => {
28+
const hulyId = 'huly_message_001'
29+
const email = '[email protected]'
30+
const result = getEmailMessageIdFromHulyId(hulyId, email)
31+
expect(result).toBe('<[email protected]>')
32+
})
33+
34+
it('should handle subdomain emails', () => {
35+
const hulyId = 'test_msg'
36+
const email = '[email protected]'
37+
const result = getEmailMessageIdFromHulyId(hulyId, email)
38+
expect(result).toBe('<[email protected]>')
39+
})
40+
41+
it('should handle complex Huly IDs', () => {
42+
const hulyId = 'channel_123_thread_456_msg_789'
43+
const email = '[email protected]'
44+
const result = getEmailMessageIdFromHulyId(hulyId, email)
45+
expect(result).toBe('<[email protected]>')
46+
})
47+
48+
it('should throw error for invalid email', () => {
49+
const hulyId = 'msg_123'
50+
const invalidEmail = 'not-an-email'
51+
expect(() => getEmailMessageIdFromHulyId(hulyId, invalidEmail)).toThrow('Invalid email address')
52+
})
53+
})
54+
55+
describe('getHulyIdFromEmailMessageId', () => {
56+
it('should extract Huly ID from email Message-ID', () => {
57+
const messageId = '<[email protected]>'
58+
const email = '[email protected]'
59+
const result = getHulyIdFromEmailMessageId(messageId, email)
60+
expect(result).toBe('msg_123456789abcdef')
61+
})
62+
63+
it('should handle Message-ID without angle brackets', () => {
64+
const messageId = '[email protected]'
65+
const email = '[email protected]'
66+
const result = getHulyIdFromEmailMessageId(messageId, email)
67+
expect(result).toBe('msg_123456789abcdef')
68+
})
69+
70+
it('should return undefined for non-matching domain', () => {
71+
const messageId = '<[email protected]>'
72+
const email = '[email protected]'
73+
const result = getHulyIdFromEmailMessageId(messageId, email)
74+
expect(result).toBeUndefined()
75+
})
76+
77+
it('should handle complex domains', () => {
78+
const messageId = '<[email protected]>'
79+
const email = '[email protected]'
80+
const result = getHulyIdFromEmailMessageId(messageId, email)
81+
expect(result).toBe('channel_123_thread_456')
82+
})
83+
84+
it('should handle empty Huly ID part', () => {
85+
const messageId = '<@example.com>'
86+
const email = '[email protected]'
87+
const result = getHulyIdFromEmailMessageId(messageId, email)
88+
expect(result).toBe('')
89+
})
90+
91+
it('should return undefined for standard email Message-IDs', () => {
92+
const messageId = '<[email protected]>'
93+
const email = '[email protected]'
94+
const result = getHulyIdFromEmailMessageId(messageId, email)
95+
expect(result).toBeUndefined()
96+
})
97+
98+
it('should handle multiple @ symbols in Message-ID', () => {
99+
const messageId = '<msg@[email protected]>'
100+
const email = '[email protected]'
101+
const result = getHulyIdFromEmailMessageId(messageId, email)
102+
expect(result).toBe('msg@test')
103+
})
104+
105+
it('should throw error for invalid email', () => {
106+
const messageId = '<[email protected]>'
107+
const invalidEmail = 'not-an-email'
108+
expect(() => getHulyIdFromEmailMessageId(messageId, invalidEmail)).toThrow('Invalid email address')
109+
})
110+
})
111+
112+
describe('isHulyEmailMessageId', () => {
113+
it('should return true for valid Huly Message-ID', () => {
114+
const messageId = '<[email protected]>'
115+
const email = '[email protected]'
116+
const result = isHulyEmailMessageId(messageId, email)
117+
expect(result).toBe(true)
118+
})
119+
120+
it('should return false for non-matching domain', () => {
121+
const messageId = '<[email protected]>'
122+
const email = '[email protected]'
123+
const result = isHulyEmailMessageId(messageId, email)
124+
expect(result).toBe(false)
125+
})
126+
127+
it('should return false for standard email Message-IDs', () => {
128+
const messageId = '<[email protected]>'
129+
const email = '[email protected]'
130+
const result = isHulyEmailMessageId(messageId, email)
131+
expect(result).toBe(false)
132+
})
133+
134+
it('should return true for Message-ID without angle brackets', () => {
135+
const messageId = '[email protected]'
136+
const email = '[email protected]'
137+
const result = isHulyEmailMessageId(messageId, email)
138+
expect(result).toBe(true)
139+
})
140+
})
141+
142+
describe('Round-trip conversion', () => {
143+
it('should preserve Huly ID through round-trip conversion', () => {
144+
const originalHulyId = 'msg_123456789abcdef'
145+
const email = '[email protected]'
146+
147+
const messageId = getEmailMessageIdFromHulyId(originalHulyId, email)
148+
const extractedHulyId = getHulyIdFromEmailMessageId(messageId, email)
149+
150+
expect(extractedHulyId).toBe(originalHulyId)
151+
})
152+
153+
it('should work with complex Huly IDs', () => {
154+
const originalHulyId = 'channel_abc123_thread_def456_msg_789xyz'
155+
const email = '[email protected]'
156+
157+
const messageId = getEmailMessageIdFromHulyId(originalHulyId, email)
158+
const extractedHulyId = getHulyIdFromEmailMessageId(messageId, email)
159+
160+
expect(extractedHulyId).toBe(originalHulyId)
161+
})
162+
163+
it('should work with subdomain emails', () => {
164+
const originalHulyId = 'notification_001'
165+
const email = '[email protected]'
166+
167+
const messageId = getEmailMessageIdFromHulyId(originalHulyId, email)
168+
const extractedHulyId = getHulyIdFromEmailMessageId(messageId, email)
169+
170+
expect(extractedHulyId).toBe(originalHulyId)
171+
})
172+
})
173+
174+
describe('Edge cases', () => {
175+
it('should handle Huly ID with special characters', () => {
176+
const hulyId = 'msg-123_test.001'
177+
const email = '[email protected]'
178+
179+
const messageId = getEmailMessageIdFromHulyId(hulyId, email)
180+
expect(messageId).toBe('<[email protected]>')
181+
182+
const extractedHulyId = getHulyIdFromEmailMessageId(messageId, email)
183+
expect(extractedHulyId).toBe(hulyId)
184+
})
185+
186+
it('should handle very long Huly IDs', () => {
187+
const hulyId = 'very_long_huly_id_with_many_segments_and_characters_123456789abcdef'
188+
const email = '[email protected]'
189+
190+
const messageId = getEmailMessageIdFromHulyId(hulyId, email)
191+
const extractedHulyId = getHulyIdFromEmailMessageId(messageId, email)
192+
193+
expect(extractedHulyId).toBe(hulyId)
194+
})
195+
196+
it('should handle empty Huly ID', () => {
197+
const hulyId = ''
198+
const email = '[email protected]'
199+
200+
const messageId = getEmailMessageIdFromHulyId(hulyId, email)
201+
expect(messageId).toBe('<@example.com>')
202+
203+
const extractedHulyId = getHulyIdFromEmailMessageId(messageId, email)
204+
expect(extractedHulyId).toBe('')
205+
})
206+
})
207+
})

0 commit comments

Comments
 (0)