diff --git a/components/aspsms/aspsms.app.mjs b/components/aspsms/aspsms.app.mjs index 4c3ac04226cf8..eecb5a063d852 100644 --- a/components/aspsms/aspsms.app.mjs +++ b/components/aspsms/aspsms.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/chattermill/chattermill.app.mjs b/components/chattermill/chattermill.app.mjs index ac3ba7ba404ef..098c30fbce1ff 100644 --- a/components/chattermill/chattermill.app.mjs +++ b/components/chattermill/chattermill.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/gmail/actions/add-label-to-email/add-label-to-email.mjs b/components/gmail/actions/add-label-to-email/add-label-to-email.mjs index 4bc532aba38ef..a8c4f2e7a20e1 100644 --- a/components/gmail/actions/add-label-to-email/add-label-to-email.mjs +++ b/components/gmail/actions/add-label-to-email/add-label-to-email.mjs @@ -4,7 +4,7 @@ export default { key: "gmail-add-label-to-email", name: "Add Label to Email", description: "Add label(s) to an email message. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/modify)", - version: "0.0.11", + version: "0.0.12", type: "action", props: { gmail, diff --git a/components/gmail/actions/approve-workflow/approve-workflow.mjs b/components/gmail/actions/approve-workflow/approve-workflow.mjs index 9723d1939f2a1..090b448f773b5 100644 --- a/components/gmail/actions/approve-workflow/approve-workflow.mjs +++ b/components/gmail/actions/approve-workflow/approve-workflow.mjs @@ -4,7 +4,7 @@ export default { key: "gmail-approve-workflow", name: "Approve Workflow", description: "Suspend the workflow until approved by email. [See the documentation](https://pipedream.com/docs/code/nodejs/rerun#flowsuspend)", - version: "0.0.8", + version: "0.0.9", type: "action", props: { gmail, diff --git a/components/gmail/actions/archive-email/archive-email.mjs b/components/gmail/actions/archive-email/archive-email.mjs index a27952a6216d7..e1fb10456ae7a 100644 --- a/components/gmail/actions/archive-email/archive-email.mjs +++ b/components/gmail/actions/archive-email/archive-email.mjs @@ -5,7 +5,7 @@ export default { key: "gmail-archive-email", name: "Archive Email", description: "Archive an email message. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/modify)", - version: "0.0.6", + version: "0.0.7", type: "action", props: { gmail, diff --git a/components/gmail/actions/create-draft/create-draft.mjs b/components/gmail/actions/create-draft/create-draft.mjs index 64f1dd5955c0b..4aeaaa0293449 100644 --- a/components/gmail/actions/create-draft/create-draft.mjs +++ b/components/gmail/actions/create-draft/create-draft.mjs @@ -6,7 +6,7 @@ export default { key: "gmail-create-draft", name: "Create Draft", description: "Create a draft from your Google Workspace email account. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.drafts/create)", - version: "0.1.6", + version: "0.1.7", type: "action", props: { gmail, diff --git a/components/gmail/actions/download-attachment/download-attachment.mjs b/components/gmail/actions/download-attachment/download-attachment.mjs index acbf7ed03c071..80ac808d3adc8 100644 --- a/components/gmail/actions/download-attachment/download-attachment.mjs +++ b/components/gmail/actions/download-attachment/download-attachment.mjs @@ -7,7 +7,7 @@ export default { key: "gmail-download-attachment", name: "Download Attachment", description: "Download an attachment by attachmentId to the /tmp directory. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.messages.attachments/get)", - version: "0.0.10", + version: "0.0.11", type: "action", props: { gmail, diff --git a/components/gmail/actions/find-email/find-email.mjs b/components/gmail/actions/find-email/find-email.mjs index 14a015f6d9e68..322bd1806a3f9 100644 --- a/components/gmail/actions/find-email/find-email.mjs +++ b/components/gmail/actions/find-email/find-email.mjs @@ -1,12 +1,11 @@ -import { convert } from "html-to-text"; -import gmail from "../../gmail.app.mjs"; import utils from "../../common/utils.mjs"; +import gmail from "../../gmail.app.mjs"; export default { key: "gmail-find-email", name: "Find Email", description: "Find an email using Google's Search Engine. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/list)", - version: "0.1.4", + version: "0.1.5", type: "action", props: { gmail, @@ -56,11 +55,9 @@ export default { }); const messageIds = messages.map(({ id }) => id); const messagesToEmit = []; - for await (const message of this.gmail.getAllMessages(messageIds)) { + for await (let message of this.gmail.getAllMessages(messageIds)) { messagesToEmit.push(message); - let newPayload = ""; - const messageIdHeader = message.payload?.headers?.find( (h) => h.name.toLowerCase() === "message-id", ); @@ -93,17 +90,9 @@ export default { message.subject = subjectHeader.value; } - if (this.withTextPayload) { - if (message.payload?.body?.data && !Array.isArray(message.payload.parts)) { - const decodedBody = utils.decodeBase64Url(message.payload.body.data); - newPayload += convert(decodedBody); - } - - if (Array.isArray(message.payload?.parts)) { - newPayload += utils.extractTextFromParts(message.payload.parts); - } - - message.payload = newPayload; + const parsedMessage = utils.validateTextPayload(message, this.withTextPayload); + if (parsedMessage) { + message = parsedMessage; } else { if (message.payload?.body?.data && !Array.isArray(message.payload.parts)) { message.payload.body.text = utils.decodeBase64Url(message.payload.body.data); diff --git a/components/gmail/actions/list-labels/list-labels.mjs b/components/gmail/actions/list-labels/list-labels.mjs index f1c41cc067f07..0b7f2b64d8383 100644 --- a/components/gmail/actions/list-labels/list-labels.mjs +++ b/components/gmail/actions/list-labels/list-labels.mjs @@ -4,7 +4,7 @@ export default { key: "gmail-list-labels", name: "List Labels", description: "List all the existing labels in the connected account. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.labels/list)", - version: "0.0.7", + version: "0.0.8", type: "action", props: { gmail, diff --git a/components/gmail/actions/remove-label-from-email/remove-label-from-email.mjs b/components/gmail/actions/remove-label-from-email/remove-label-from-email.mjs index f0e29995f6f49..9f189e1a359ce 100644 --- a/components/gmail/actions/remove-label-from-email/remove-label-from-email.mjs +++ b/components/gmail/actions/remove-label-from-email/remove-label-from-email.mjs @@ -4,7 +4,7 @@ export default { key: "gmail-remove-label-from-email", name: "Remove Label from Email", description: "Remove label(s) from an email message. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/modify)", - version: "0.0.9", + version: "0.0.10", type: "action", props: { gmail, diff --git a/components/gmail/actions/send-email/send-email.mjs b/components/gmail/actions/send-email/send-email.mjs index 339c32c41a237..cbe5d77fb3d4e 100644 --- a/components/gmail/actions/send-email/send-email.mjs +++ b/components/gmail/actions/send-email/send-email.mjs @@ -6,7 +6,7 @@ export default { key: "gmail-send-email", name: "Send Email", description: "Send an email from your Google Workspace email account. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/send)", - version: "0.1.14", + version: "0.1.15", type: "action", props: { gmail, diff --git a/components/gmail/actions/update-org-signature/update-org-signature.mjs b/components/gmail/actions/update-org-signature/update-org-signature.mjs index 702275591f87f..48be0c6cd06a0 100644 --- a/components/gmail/actions/update-org-signature/update-org-signature.mjs +++ b/components/gmail/actions/update-org-signature/update-org-signature.mjs @@ -8,7 +8,7 @@ export default { name: "Update Signature for Email in Organization", description: `Update the signature for a specific email address in an organization. A Google Cloud service account with delegated domain-wide authority is required for this action. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.settings.sendAs/update)`, - version: "0.0.10", + version: "0.0.11", type: "action", props: { gmail, diff --git a/components/gmail/actions/update-primary-signature/update-primary-signature.mjs b/components/gmail/actions/update-primary-signature/update-primary-signature.mjs index 78b10f30a0947..aae983a0659d0 100644 --- a/components/gmail/actions/update-primary-signature/update-primary-signature.mjs +++ b/components/gmail/actions/update-primary-signature/update-primary-signature.mjs @@ -4,7 +4,7 @@ export default { key: "gmail-update-primary-signature", name: "Update Signature for Primary Email Address", description: "Update the signature for the primary email address. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.settings.sendAs/update)", - version: "0.0.10", + version: "0.0.11", type: "action", props: { gmail, diff --git a/components/gmail/common/utils.mjs b/components/gmail/common/utils.mjs index 28b93b83ee5b5..fba8a26facb03 100644 --- a/components/gmail/common/utils.mjs +++ b/components/gmail/common/utils.mjs @@ -46,9 +46,25 @@ function attachTextToParts(parts) { } }; +function validateTextPayload(message, withTextPayload) { + if (withTextPayload) { + let newPayload = ""; + if (message.payload?.body?.data && !Array.isArray(message.payload.parts)) { + const decodedBody = decodeBase64Url(message.payload.body.data); + newPayload = convert(decodedBody); + } else if (Array.isArray(message.payload?.parts)) { + newPayload = extractTextFromParts(message.payload.parts); + } + message.payload = newPayload; + return message; + } + return false; +}; + export default { parseArray, decodeBase64Url, extractTextFromParts, attachTextToParts, + validateTextPayload, }; diff --git a/components/gmail/package.json b/components/gmail/package.json index e38b8c0fcab44..c0067fa19131e 100644 --- a/components/gmail/package.json +++ b/components/gmail/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/gmail", - "version": "1.1.0", + "version": "1.1.1", "description": "Pipedream Gmail Components", "main": "gmail.app.mjs", "keywords": [ @@ -16,7 +16,7 @@ "@google-cloud/local-auth": "^2.1.0", "@google-cloud/pubsub": "^4.7.0", "@googleapis/gmail": "^13.0.1", - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "google-auth-library": "^8.7.0", "html-to-text": "^9.0.5", "mime": "^3.0.0", diff --git a/components/gmail/sources/common/base.mjs b/components/gmail/sources/common/base.mjs index 6054c37c7d3dd..7c8ce3188020e 100644 --- a/components/gmail/sources/common/base.mjs +++ b/components/gmail/sources/common/base.mjs @@ -1,4 +1,5 @@ import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import utils from "../../common/utils.mjs"; export default { props: { @@ -17,32 +18,34 @@ export default { || ArrayBuffer.isView(data); }, decodeContent(message) { - const MULTIPART_MIME_TYPE = "multipart"; - const { - payload: { - mimeType, body, parts, - }, - } = message; + let parsedMessage = utils.validateTextPayload(message, this.withTextPayload); + if (!parsedMessage) { + const MULTIPART_MIME_TYPE = "multipart"; + const { + payload: { + mimeType, body, parts, + }, + } = message; - if (!mimeType.startsWith(MULTIPART_MIME_TYPE)) { - return !this.isValidType(body?.data) - ? message - : { - ...message, - decodedContent: Buffer.from(body.data, "base64").toString(), - }; - } + if (!mimeType.startsWith(MULTIPART_MIME_TYPE)) { + return !this.isValidType(body?.data) + ? message + : { + ...message, + decodedContent: Buffer.from(body.data, "base64").toString(), + }; + } - const [ - firstPart, - ] = parts; + const [ + firstPart, + ] = parts; - return !this.isValidType(firstPart?.body?.data) - ? message - : { - ...message, - decodedContent: Buffer.from(firstPart.body.data, "base64").toString(), - }; + if (this.isValidType(firstPart?.body?.data)) { + message.decodedContent = Buffer.from(firstPart.body.data, "base64").toString(); + } + parsedMessage = message; + } + return parsedMessage; }, parseEmail(emailStr) { if (!emailStr) { diff --git a/components/gmail/sources/new-attachment-received/new-attachment-received.mjs b/components/gmail/sources/new-attachment-received/new-attachment-received.mjs index 0eeb82361ff3b..8f5891425618d 100644 --- a/components/gmail/sources/new-attachment-received/new-attachment-received.mjs +++ b/components/gmail/sources/new-attachment-received/new-attachment-received.mjs @@ -1,3 +1,4 @@ +import utils from "../../common/utils.mjs"; import gmail from "../../gmail.app.mjs"; import common from "../common/polling-messages.mjs"; @@ -6,7 +7,7 @@ export default { key: "gmail-new-attachment-received", name: "New Attachment Received", description: "Emit new event for each attachment in a message received. This source is capped at 100 max new messages per run.", - version: "0.0.15", + version: "0.1.0", type: "source", dedupe: "unique", props: { @@ -27,6 +28,12 @@ export default { label: "Labels", optional: true, }, + withTextPayload: { + type: "boolean", + label: "Return payload as plaintext", + description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**", + default: false, + }, }, methods: { ...common.methods, @@ -51,11 +58,12 @@ export default { }, emitEvent(message) { if (message) { - const { parts: attachments } = message.payload; + const { parts: attachments = [] } = message.payload; + const parsedMessage = utils.validateTextPayload(message, this.withTextPayload); attachments.filter((attachment) => attachment.body.attachmentId).forEach((attachment) => { this.$emit({ - message, + message: parsedMessage || message, attachment, }, this.generateMeta(attachment, message)); }); diff --git a/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs b/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs index ad35bf219f9a8..515a022acc7a0 100644 --- a/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs +++ b/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs @@ -6,7 +6,7 @@ export default { key: "gmail-new-email-matching-search", name: "New Email Matching Search", description: "Emit new event when an email matching the search criteria is received. This source is capped at 100 max new messages per run.", - version: "0.0.13", + version: "0.1.0", type: "source", dedupe: "unique", props: { @@ -27,6 +27,12 @@ export default { label: "Labels", optional: true, }, + withTextPayload: { + type: "boolean", + label: "Return payload as plaintext", + description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**", + default: false, + }, }, methods: { ...common.methods, diff --git a/components/gmail/sources/new-email-received/new-email-received.mjs b/components/gmail/sources/new-email-received/new-email-received.mjs index f9a44f3454369..998ce93ecf709 100644 --- a/components/gmail/sources/new-email-received/new-email-received.mjs +++ b/components/gmail/sources/new-email-received/new-email-received.mjs @@ -15,7 +15,7 @@ export default { name: "New Email Received", description: "Emit new event when a new email is received.", type: "source", - version: "0.2.8", + version: "0.3.0", dedupe: "unique", props: { gmail, @@ -41,6 +41,12 @@ export default { intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, }, }, + withTextPayload: { + type: "boolean", + label: "Return payload as plaintext", + description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**", + default: false, + }, serviceAccountKeyJson: { type: "string", label: "Service Account Key JSON", diff --git a/components/gmail/sources/new-labeled-email/new-labeled-email.mjs b/components/gmail/sources/new-labeled-email/new-labeled-email.mjs index 0d79813eef32b..37b6240056f43 100644 --- a/components/gmail/sources/new-labeled-email/new-labeled-email.mjs +++ b/components/gmail/sources/new-labeled-email/new-labeled-email.mjs @@ -8,7 +8,7 @@ export default { name: "New Labeled Email", description: "Emit new event when a new email is labeled.", type: "source", - version: "0.0.14", + version: "0.1.0", dedupe: "unique", props: { gmail, @@ -21,6 +21,12 @@ export default { type: "string[]", label: "Labels", }, + withTextPayload: { + type: "boolean", + label: "Return payload as plaintext", + description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**", + default: false, + }, }, methods: { ...common.methods, diff --git a/components/gmail/sources/new-sent-email/new-sent-email.mjs b/components/gmail/sources/new-sent-email/new-sent-email.mjs index 96bfb35c2c33f..0ece945e4f955 100644 --- a/components/gmail/sources/new-sent-email/new-sent-email.mjs +++ b/components/gmail/sources/new-sent-email/new-sent-email.mjs @@ -6,7 +6,7 @@ export default { key: "gmail-new-sent-email", name: "New Sent Email", description: "Emit new event for each new email sent. (Maximum of 100 events emited per execution)", - version: "0.0.14", + version: "0.1.0", type: "source", dedupe: "unique", props: { @@ -18,6 +18,12 @@ export default { "q", ], }, + withTextPayload: { + type: "boolean", + label: "Return payload as plaintext", + description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**", + default: false, + }, }, methods: { ...common.methods, diff --git a/components/goformz/goformz.app.mjs b/components/goformz/goformz.app.mjs index 911dc2b015ab0..e3faeb0b29de4 100644 --- a/components/goformz/goformz.app.mjs +++ b/components/goformz/goformz.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/lark/lark.app.mjs b/components/lark/lark.app.mjs index 8d0d2381dc5df..a434364213871 100644 --- a/components/lark/lark.app.mjs +++ b/components/lark/lark.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/momentum_ams/momentum_ams.app.mjs b/components/momentum_ams/momentum_ams.app.mjs index a1c48314f7273..a9a13189e1115 100644 --- a/components/momentum_ams/momentum_ams.app.mjs +++ b/components/momentum_ams/momentum_ams.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/opinion_stage/opinion_stage.app.mjs b/components/opinion_stage/opinion_stage.app.mjs index 7369ca1c75a32..3b0013c92319c 100644 --- a/components/opinion_stage/opinion_stage.app.mjs +++ b/components/opinion_stage/opinion_stage.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/orderspace/orderspace.app.mjs b/components/orderspace/orderspace.app.mjs index 7f5269fe4e0dd..2015e23db53c9 100644 --- a/components/orderspace/orderspace.app.mjs +++ b/components/orderspace/orderspace.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/sage_accounting/sage_accounting.app.mjs b/components/sage_accounting/sage_accounting.app.mjs index 4276f468f51d4..a8c3c00fc54b1 100644 --- a/components/sage_accounting/sage_accounting.app.mjs +++ b/components/sage_accounting/sage_accounting.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/smashsend/smashsend.app.mjs b/components/smashsend/smashsend.app.mjs index a11faf475d6fc..014613ec92db9 100644 --- a/components/smashsend/smashsend.app.mjs +++ b/components/smashsend/smashsend.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/stiply/stiply.app.mjs b/components/stiply/stiply.app.mjs index 710c34d71d1ac..1f8ca8b1c715b 100644 --- a/components/stiply/stiply.app.mjs +++ b/components/stiply/stiply.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 52568e0f32258..b5563a99152e7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2355,8 +2355,7 @@ importers: components/chatsonic: {} - components/chattermill: - specifiers: {} + components/chattermill: {} components/chatwork: dependencies: @@ -5353,8 +5352,8 @@ importers: specifier: ^13.0.1 version: 13.0.1 '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 google-auth-library: specifier: ^8.7.0 version: 8.9.0 @@ -5399,8 +5398,7 @@ importers: components/godial: {} - components/goformz: - specifiers: {} + components/goformz: {} components/gohighlevel: dependencies: @@ -7291,8 +7289,7 @@ importers: specifier: ^3.0.0 version: 3.0.3 - components/lark: - specifiers: {} + components/lark: {} components/lastpass: dependencies: @@ -8497,8 +8494,7 @@ importers: components/mollie: {} - components/momentum_ams: - specifiers: {} + components/momentum_ams: {} components/monday: dependencies: @@ -9412,8 +9408,7 @@ importers: specifier: ^0.9.0 version: 0.9.0 - components/opinion_stage: - specifiers: {} + components/opinion_stage: {} components/opnform: dependencies: @@ -9484,8 +9479,7 @@ importers: components/order_sender: {} - components/orderspace: - specifiers: {} + components/orderspace: {} components/originality_ai: dependencies: @@ -11562,8 +11556,7 @@ importers: components/ryver: {} - components/sage_accounting: - specifiers: {} + components/sage_accounting: {} components/sage_intacct: {} @@ -12980,8 +12973,7 @@ importers: components/stealthseminar: {} - components/stiply: - specifiers: {} + components/stiply: {} components/storeganise: dependencies: