Skip to content

Commit 0fc0f68

Browse files
authored
feat(gmail): added gmail polling service to trigger workflow on incoming emails (#344)
* setup gmail polling service, not tested * general improvements to gmail polling and error handling, receive message but triggers wrong wrokflow * finished gmail polling service, works when I send multiple emails in a single polling period (triggers the workflow for each new email) * remove unread messages * remove unread messages * modified to process all incoming emails as individual workflow executions, enhance dedupe, general improvements * replaced desc w tooltips * added cron job for polling gmail * remove unused props, simplify naming convention * renoved extraneous comments, removed unused processIncomingEmails * fixed build issues * acknowledged PR comments
1 parent d79cad4 commit 0fc0f68

File tree

38 files changed

+1686
-160
lines changed

38 files changed

+1686
-160
lines changed

.husky/pre-commit

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
cd sim && npx lint-staged
1+
cd apps/sim && npx lint-staged

apps/sim/app/api/auth/oauth/gmail/labels/route.ts

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,14 @@ import { refreshAccessTokenIfNeeded } from '../../utils'
88

99
const logger = createLogger('GmailLabelsAPI')
1010

11+
interface GmailLabel {
12+
id: string
13+
name: string
14+
type: 'system' | 'user'
15+
messagesTotal?: number
16+
messagesUnread?: number
17+
}
18+
1119
export async function GET(request: NextRequest) {
1220
const requestId = crypto.randomUUID().slice(0, 8)
1321

@@ -57,7 +65,6 @@ export async function GET(request: NextRequest) {
5765
}
5866

5967
// Fetch labels from Gmail API
60-
logger.info(`[${requestId}] Fetching labels from Gmail API`)
6168
const response = await fetch('https://gmail.googleapis.com/gmail/v1/users/me/labels', {
6269
headers: {
6370
Authorization: `Bearer ${accessToken}`,
@@ -80,12 +87,13 @@ export async function GET(request: NextRequest) {
8087
}
8188

8289
const data = await response.json()
83-
84-
// Log the number of labels received
85-
logger.info(`[${requestId}] Received ${data.labels?.length || 0} labels from Gmail API`)
90+
if (!Array.isArray(data.labels)) {
91+
logger.error(`[${requestId}] Unexpected labels response structure:`, data)
92+
return NextResponse.json({ error: 'Invalid labels response' }, { status: 500 })
93+
}
8694

8795
// Transform the labels to a more usable format
88-
const labels = data.labels.map((label: any) => {
96+
const labels = data.labels.map((label: GmailLabel) => {
8997
// Format the label name with proper capitalization
9098
let formattedName = label.name
9199

@@ -106,7 +114,7 @@ export async function GET(request: NextRequest) {
106114

107115
// Filter labels if a query is provided
108116
const filteredLabels = query
109-
? labels.filter((label: any) =>
117+
? labels.filter((label: GmailLabel) =>
110118
label.name.toLowerCase().includes((query as string).toLowerCase())
111119
)
112120
: labels

apps/sim/app/api/help/route.ts

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,18 @@ export async function POST(req: NextRequest) {
6666
const images: { filename: string; content: Buffer; contentType: string }[] = []
6767

6868
for (const [key, value] of formData.entries()) {
69-
if (key.startsWith('image_') && value instanceof Blob) {
70-
const file = value as File
71-
const buffer = Buffer.from(await file.arrayBuffer())
72-
73-
images.push({
74-
filename: file.name,
75-
content: buffer,
76-
contentType: file.type,
77-
})
69+
if (key.startsWith('image_') && typeof value !== 'string') {
70+
if (value && 'arrayBuffer' in value) {
71+
const blob = value as unknown as Blob
72+
const buffer = Buffer.from(await blob.arrayBuffer())
73+
const filename = 'name' in value ? (value as any).name : `image_${key.split('_')[1]}`
74+
75+
images.push({
76+
filename,
77+
content: buffer,
78+
contentType: 'type' in value ? (value as any).type : 'application/octet-stream',
79+
})
80+
}
7881
}
7982
}
8083

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import { NextRequest, NextResponse } from 'next/server'
2+
import { nanoid } from 'nanoid'
3+
import { Logger } from '@/lib/logs/console-logger'
4+
import { pollGmailWebhooks } from '@/lib/webhooks/gmail-polling-service'
5+
6+
const logger = new Logger('GmailPollingAPI')
7+
8+
export const dynamic = 'force-dynamic'
9+
export const maxDuration = 300 // Allow up to 5 minutes for polling to complete
10+
11+
interface PollingTask {
12+
promise: Promise<any>
13+
startedAt: number
14+
}
15+
16+
const activePollingTasks = new Map<string, PollingTask>()
17+
const STALE_TASK_THRESHOLD_MS = 10 * 60 * 1000 // 10 minutes
18+
19+
function cleanupStaleTasks() {
20+
const now = Date.now()
21+
let removedCount = 0
22+
23+
for (const [requestId, task] of activePollingTasks.entries()) {
24+
if (now - task.startedAt > STALE_TASK_THRESHOLD_MS) {
25+
activePollingTasks.delete(requestId)
26+
removedCount++
27+
}
28+
}
29+
30+
if (removedCount > 0) {
31+
logger.info(`Cleaned up ${removedCount} stale polling tasks`)
32+
}
33+
34+
return removedCount
35+
}
36+
37+
export async function GET(request: NextRequest) {
38+
const requestId = nanoid()
39+
logger.info(`Gmail webhook polling triggered (${requestId})`)
40+
41+
try {
42+
const authHeader = request.headers.get('authorization')
43+
const webhookSecret = process.env.WEBHOOK_POLLING_SECRET
44+
45+
if (!webhookSecret) {
46+
logger.warn(`WEBHOOK_POLLING_SECRET is not set`)
47+
return new NextResponse('Configuration error: Webhook secret is not set', { status: 500 })
48+
}
49+
50+
if (!authHeader || authHeader !== `Bearer ${webhookSecret}`) {
51+
logger.warn(`Unauthorized access attempt to Gmail polling endpoint (${requestId})`)
52+
return new NextResponse('Unauthorized', { status: 401 })
53+
}
54+
55+
cleanupStaleTasks()
56+
57+
const pollingTask: PollingTask = {
58+
promise: null as any,
59+
startedAt: Date.now(),
60+
}
61+
62+
pollingTask.promise = pollGmailWebhooks()
63+
.then((results) => {
64+
logger.info(`Gmail polling completed successfully (${requestId})`, {
65+
userCount: results?.total || 0,
66+
successful: results?.successful || 0,
67+
failed: results?.failed || 0,
68+
})
69+
activePollingTasks.delete(requestId)
70+
return results
71+
})
72+
.catch((error) => {
73+
logger.error(`Error in background Gmail polling task (${requestId}):`, error)
74+
activePollingTasks.delete(requestId)
75+
throw error
76+
})
77+
78+
activePollingTasks.set(requestId, pollingTask)
79+
80+
return NextResponse.json({
81+
success: true,
82+
message: 'Gmail webhook polling started successfully',
83+
requestId,
84+
status: 'polling_started',
85+
activeTasksCount: activePollingTasks.size,
86+
})
87+
} catch (error) {
88+
logger.error(`Error initiating Gmail webhook polling (${requestId}):`, error)
89+
90+
return NextResponse.json(
91+
{
92+
success: false,
93+
message: 'Failed to start Gmail webhook polling',
94+
error: error instanceof Error ? error.message : 'Unknown error',
95+
requestId,
96+
},
97+
{ status: 500 }
98+
)
99+
}
100+
}

apps/sim/app/api/webhooks/route.ts

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,40 @@ export async function POST(request: NextRequest) {
182182
}
183183
// --- End Telegram specific logic ---
184184

185+
// --- Gmail webhook setup ---
186+
if (savedWebhook && provider === 'gmail') {
187+
logger.info(
188+
`[${requestId}] Gmail provider detected. Setting up Gmail webhook configuration.`
189+
)
190+
try {
191+
const { configureGmailPolling } = await import('@/lib/webhooks/utils')
192+
const success = await configureGmailPolling(userId, savedWebhook, requestId)
193+
194+
if (!success) {
195+
logger.error(`[${requestId}] Failed to configure Gmail polling`)
196+
return NextResponse.json(
197+
{
198+
error: 'Failed to configure Gmail polling',
199+
details: 'Please check your Gmail account permissions and try again',
200+
},
201+
{ status: 500 }
202+
)
203+
}
204+
205+
logger.info(`[${requestId}] Successfully configured Gmail polling`)
206+
} catch (err) {
207+
logger.error(`[${requestId}] Error setting up Gmail webhook configuration`, err)
208+
return NextResponse.json(
209+
{
210+
error: 'Failed to configure Gmail webhook',
211+
details: err instanceof Error ? err.message : 'Unknown error',
212+
},
213+
{ status: 500 }
214+
)
215+
}
216+
}
217+
// --- End Gmail specific logic ---
218+
185219
const status = existingWebhooks.length > 0 ? 200 : 201
186220
return NextResponse.json({ webhook: savedWebhook }, { status })
187221
} catch (error: any) {

apps/sim/app/api/webhooks/trigger/[path]/route.ts

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ export async function POST(
191191

192192
// Detect provider type
193193
const isAirtableWebhook = foundWebhook.provider === 'airtable'
194+
const isGmailWebhook = foundWebhook.provider === 'gmail'
194195

195196
// Handle Slack challenge verification (must be done before timeout)
196197
const slackChallengeResponse =
@@ -283,6 +284,49 @@ export async function POST(
283284
}
284285
}
285286

287+
// For Gmail: Process with specific email handling
288+
if (isGmailWebhook) {
289+
try {
290+
logger.info(`[${requestId}] Gmail webhook request received for webhook: ${foundWebhook.id}`)
291+
292+
const webhookSecret = foundWebhook.secret
293+
if (webhookSecret) {
294+
const secretHeader = request.headers.get('X-Webhook-Secret')
295+
if (secretHeader !== webhookSecret) {
296+
logger.warn(`[${requestId}] Invalid webhook secret`)
297+
return new NextResponse('Unauthorized', { status: 401 })
298+
}
299+
}
300+
301+
if (body.email) {
302+
logger.info(`[${requestId}] Processing Gmail email`, {
303+
emailId: body.email.id,
304+
subject:
305+
body.email?.payload?.headers?.find((h: any) => h.name === 'Subject')?.value ||
306+
'No subject',
307+
})
308+
309+
const executionId = uuidv4()
310+
logger.info(`[${requestId}] Executing workflow ${foundWorkflow.id} for Gmail email`)
311+
312+
return await processWebhook(
313+
foundWebhook,
314+
foundWorkflow,
315+
body,
316+
request,
317+
executionId,
318+
requestId
319+
)
320+
} else {
321+
logger.warn(`[${requestId}] Invalid Gmail webhook payload format`)
322+
return new NextResponse('Invalid payload format', { status: 400 })
323+
}
324+
} catch (error: any) {
325+
logger.error(`[${requestId}] Error processing Gmail webhook`, error)
326+
return new NextResponse(`Internal server error: ${error.message}`, { status: 500 })
327+
}
328+
}
329+
286330
// --- For all other webhook types: Use async processing with timeout ---
287331

288332
// Create timeout promise for fast initial response (2.5 seconds)

apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/credential-selector/components/oauth-required-modal.tsx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export interface OAuthRequiredModalProps {
3636
const SCOPE_DESCRIPTIONS: Record<string, string> = {
3737
'https://www.googleapis.com/auth/gmail.send': 'Send emails on your behalf',
3838
'https://www.googleapis.com/auth/gmail.labels': 'View and manage your email labels',
39+
'https://www.googleapis.com/auth/gmail.modify': 'View and manage your email messages',
3940
// 'https://www.googleapis.com/auth/gmail.readonly': 'View and read your email messages',
4041
// 'https://www.googleapis.com/auth/drive': 'View and manage your Google Drive files',
4142
'https://www.googleapis.com/auth/drive.file': 'View and manage your Google Drive files',

apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/webhook/components/providers/airtable-config.tsx renamed to apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/webhook/components/providers/airtable.tsx

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
import React from 'react'
2+
import { Info } from 'lucide-react'
3+
import { Button } from '@/components/ui/button'
24
import { Input } from '@/components/ui/input'
35
import { Label } from '@/components/ui/label'
46
import { Skeleton } from '@/components/ui/skeleton'
57
import { Switch } from '@/components/ui/switch'
8+
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
69
import { ConfigField } from '../ui/config-field'
710
import { ConfigSection } from '../ui/config-section'
811
import { InstructionsSection } from '../ui/instructions-section'
@@ -92,13 +95,32 @@ export function AirtableConfig({
9295
</ConfigField>
9396

9497
<div className="flex items-center justify-between rounded-lg border border-border p-3 shadow-sm bg-background">
95-
<div className="space-y-0.5 pr-4">
98+
<div className="flex items-center gap-2">
9699
<Label htmlFor="include-cell-values" className="font-normal">
97100
Include Full Record Data
98101
</Label>
99-
<p className="text-xs text-muted-foreground">
100-
Enable to receive the complete record data in the payload, not just changes.
101-
</p>
102+
<Tooltip>
103+
<TooltipTrigger asChild>
104+
<Button
105+
variant="ghost"
106+
size="sm"
107+
className="text-gray-500 p-1 h-6 w-6"
108+
aria-label="Learn more about including full record data"
109+
>
110+
<Info className="h-4 w-4" />
111+
</Button>
112+
</TooltipTrigger>
113+
<TooltipContent
114+
side="right"
115+
align="center"
116+
className="max-w-[300px] p-3 z-[100]"
117+
role="tooltip"
118+
>
119+
<p className="text-sm">
120+
Enable to receive the complete record data in the payload, not just changes.
121+
</p>
122+
</TooltipContent>
123+
</Tooltip>
102124
</div>
103125
{isLoadingToken ? (
104126
<Skeleton className="h-5 w-9" />

apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/webhook/components/providers/discord-config.tsx renamed to apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/webhook/components/providers/discord.tsx

File renamed without changes.

apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/webhook/components/providers/generic-config.tsx renamed to apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/webhook/components/providers/generic.tsx

File renamed without changes.

0 commit comments

Comments
 (0)