17235 source gmail sources return too much text

components/gmail/actions/add-label-to-email/add-label-to-email.mjs

@@ -4,7 +4,7 @@ export default {
   key: "gmail-add-label-to-email",
   name: "Add Label to Email",
   description: "Add label(s) to an email message. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/modify)",
-  version: "0.0.11",
+  version: "0.0.12",
   type: "action",
   props: {
     gmail,

components/gmail/actions/approve-workflow/approve-workflow.mjs

@@ -4,7 +4,7 @@ export default {
   key: "gmail-approve-workflow",
   name: "Approve Workflow",
   description: "Suspend the workflow until approved by email. [See the documentation](https://pipedream.com/docs/code/nodejs/rerun#flowsuspend)",
-  version: "0.0.8",
+  version: "0.0.9",
   type: "action",
   props: {
     gmail,

components/gmail/actions/archive-email/archive-email.mjs

@@ -5,7 +5,7 @@ export default {
   key: "gmail-archive-email",
   name: "Archive Email",
   description: "Archive an email message. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/modify)",
-  version: "0.0.6",
+  version: "0.0.7",
   type: "action",
   props: {
     gmail,

components/gmail/actions/create-draft/create-draft.mjs

@@ -6,7 +6,7 @@ export default {
   key: "gmail-create-draft",
   name: "Create Draft",
   description: "Create a draft from your Google Workspace email account. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.drafts/create)",
-  version: "0.1.6",
+  version: "0.1.7",
   type: "action",
   props: {
     gmail,

components/gmail/actions/download-attachment/download-attachment.mjs

@@ -7,7 +7,7 @@ export default {
   key: "gmail-download-attachment",
   name: "Download Attachment",
   description: "Download an attachment by attachmentId to the /tmp directory. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.messages.attachments/get)",
-  version: "0.0.10",
+  version: "0.0.11",
   type: "action",
   props: {
     gmail,

components/gmail/actions/find-email/find-email.mjs

@@ -1,12 +1,11 @@
-import { convert } from "html-to-text";
-import gmail from "../../gmail.app.mjs";
 import utils from "../../common/utils.mjs";
+import gmail from "../../gmail.app.mjs";
 
 export default {
   key: "gmail-find-email",
   name: "Find Email",
   description: "Find an email using Google's Search Engine. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/list)",
-  version: "0.1.4",
+  version: "0.1.5",
   type: "action",
   props: {
     gmail,
@@ -56,11 +55,9 @@ export default {
     });
     const messageIds = messages.map(({ id }) => id);
     const messagesToEmit = [];
-    for await (const message of this.gmail.getAllMessages(messageIds)) {
+    for await (let message of this.gmail.getAllMessages(messageIds)) {
       messagesToEmit.push(message);
 
-      let newPayload = "";
-
       const messageIdHeader = message.payload?.headers?.find(
         (h) => h.name.toLowerCase() === "message-id",
       );
@@ -93,17 +90,9 @@ export default {
         message.subject = subjectHeader.value;
       }
 
-      if (this.withTextPayload) {
-        if (message.payload?.body?.data && !Array.isArray(message.payload.parts)) {
-          const decodedBody = utils.decodeBase64Url(message.payload.body.data);
-          newPayload += convert(decodedBody);
-        }
-
-        if (Array.isArray(message.payload?.parts)) {
-          newPayload += utils.extractTextFromParts(message.payload.parts);
-        }
-
-        message.payload = newPayload;
+      const parsedMessage = utils.validateTextPayload(message, this.withTextPayload);
+      if (parsedMessage) {
+        message = parsedMessage;
       } else {
         if (message.payload?.body?.data && !Array.isArray(message.payload.parts)) {
           message.payload.body.text = utils.decodeBase64Url(message.payload.body.data);

components/gmail/actions/list-labels/list-labels.mjs

@@ -4,7 +4,7 @@ export default {
   key: "gmail-list-labels",
   name: "List Labels",
   description: "List all the existing labels in the connected account. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.labels/list)",
-  version: "0.0.7",
+  version: "0.0.8",
   type: "action",
   props: {
     gmail,

components/gmail/actions/remove-label-from-email/remove-label-from-email.mjs

@@ -4,7 +4,7 @@ export default {
   key: "gmail-remove-label-from-email",
   name: "Remove Label from Email",
   description: "Remove label(s) from an email message. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/modify)",
-  version: "0.0.9",
+  version: "0.0.10",
   type: "action",
   props: {
     gmail,

components/gmail/actions/send-email/send-email.mjs

@@ -6,7 +6,7 @@ export default {
   key: "gmail-send-email",
   name: "Send Email",
   description: "Send an email from your Google Workspace email account. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/send)",
-  version: "0.1.14",
+  version: "0.1.15",
   type: "action",
   props: {
     gmail,

components/gmail/actions/update-org-signature/update-org-signature.mjs

@@ -8,7 +8,7 @@ export default {
   name: "Update Signature for Email in Organization",
   description: `Update the signature for a specific email address in an organization.
     A Google Cloud service account with delegated domain-wide authority is required for this action. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.settings.sendAs/update)`,
-  version: "0.0.10",
+  version: "0.0.11",
   type: "action",
   props: {
     gmail,

components/gmail/actions/update-primary-signature/update-primary-signature.mjs

@@ -4,7 +4,7 @@ export default {
   key: "gmail-update-primary-signature",
   name: "Update Signature for Primary Email Address",
   description: "Update the signature for the primary email address. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.settings.sendAs/update)",
-  version: "0.0.10",
+  version: "0.0.11",
   type: "action",
   props: {
     gmail,

components/gmail/common/utils.mjs

@@ -46,9 +46,27 @@ function attachTextToParts(parts) {
   }
 };
 
+function validateTextPayload(message, withTextPayload) {
+  if (withTextPayload) {
+    let newPayload = "";
+    if (message.payload?.body?.data && !Array.isArray(message.payload.parts)) {
+      const decodedBody = decodeBase64Url(message.payload.body.data);
+      newPayload += convert(decodedBody);
+    }
+
+    if (Array.isArray(message.payload?.parts)) {
+      newPayload += extractTextFromParts(message.payload.parts);
+    }
+    message.payload = newPayload;
+    return message;
+  }
+  return false;
+};
+
 export default {
   parseArray,
   decodeBase64Url,
   extractTextFromParts,
   attachTextToParts,
+  validateTextPayload,
 };

components/gmail/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/gmail",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Pipedream Gmail Components",
   "main": "gmail.app.mjs",
   "keywords": [
@@ -16,7 +16,7 @@
     "@google-cloud/local-auth": "^2.1.0",
     "@google-cloud/pubsub": "^4.7.0",
     "@googleapis/gmail": "^13.0.1",
-    "@pipedream/platform": "^3.0.3",
+    "@pipedream/platform": "^3.1.0",
     "google-auth-library": "^8.7.0",
     "html-to-text": "^9.0.5",
     "mime": "^3.0.0",

components/gmail/sources/common/base.mjs

@@ -1,4 +1,5 @@
 import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import utils from "../../common/utils.mjs";
 
 export default {
   props: {
@@ -17,32 +18,34 @@ export default {
         || ArrayBuffer.isView(data);
     },
     decodeContent(message) {
-      const MULTIPART_MIME_TYPE = "multipart";
-      const {
-        payload: {
-          mimeType, body, parts,
-        },
-      } = message;
+      let parsedMessage = utils.validateTextPayload(message, this.withTextPayload);
+      if (!parsedMessage) {
+        const MULTIPART_MIME_TYPE = "multipart";
+        const {
+          payload: {
+            mimeType, body, parts,
+          },
+        } = message;
 
-      if (!mimeType.startsWith(MULTIPART_MIME_TYPE)) {
-        return !this.isValidType(body?.data)
-          ? message
-          : {
-            ...message,
-            decodedContent: Buffer.from(body.data, "base64").toString(),
-          };
-      }
+        if (!mimeType.startsWith(MULTIPART_MIME_TYPE)) {
+          return !this.isValidType(body?.data)
+            ? message
+            : {
+              ...message,
+              decodedContent: Buffer.from(body.data, "base64").toString(),
+            };
+        }
 
-      const [
-        firstPart,
-      ] = parts;
+        const [
+          firstPart,
+        ] = parts;
 
-      return !this.isValidType(firstPart?.body?.data)
-        ? message
-        : {
-          ...message,
-          decodedContent: Buffer.from(firstPart.body.data, "base64").toString(),
-        };
+        if (this.isValidType(firstPart?.body?.data)) {
+          message.decodedContent = Buffer.from(firstPart.body.data, "base64").toString();
+        }
+        parsedMessage = message;
+      }
+      return parsedMessage;
     },
     parseEmail(emailStr) {
       if (!emailStr) {

components/gmail/sources/new-attachment-received/new-attachment-received.mjs

@@ -1,3 +1,4 @@
+import utils from "../../common/utils.mjs";
 import gmail from "../../gmail.app.mjs";
 import common from "../common/polling-messages.mjs";
 
@@ -6,7 +7,7 @@ export default {
   key: "gmail-new-attachment-received",
   name: "New Attachment Received",
   description: "Emit new event for each attachment in a message received. This source is capped at 100 max new messages per run.",
-  version: "0.0.15",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -27,6 +28,12 @@ export default {
       label: "Labels",
       optional: true,
     },
+    withTextPayload: {
+      type: "boolean",
+      label: "Return payload as plaintext",
+      description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**",
+      default: false,
+    },
   },
   methods: {
     ...common.methods,
@@ -51,11 +58,12 @@ export default {
     },
     emitEvent(message) {
       if (message) {
-        const { parts: attachments } = message.payload;
+        const { parts: attachments = [] } = message.payload;
+        const parsedMessage = utils.validateTextPayload(message, this.withTextPayload);
 
         attachments.filter((attachment) => attachment.body.attachmentId).forEach((attachment) => {
           this.$emit({
-            message,
+            message: parsedMessage || message,
             attachment,
           }, this.generateMeta(attachment, message));
         });

components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs

@@ -6,7 +6,7 @@ export default {
   key: "gmail-new-email-matching-search",
   name: "New Email Matching Search",
   description: "Emit new event when an email matching the search criteria is received. This source is capped at 100 max new messages per run.",
-  version: "0.0.13",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -27,6 +27,12 @@ export default {
       label: "Labels",
       optional: true,
     },
+    withTextPayload: {
+      type: "boolean",
+      label: "Return payload as plaintext",
+      description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**",
+      default: false,
+    },
   },
   methods: {
     ...common.methods,

components/gmail/sources/new-email-received/new-email-received.mjs

@@ -15,7 +15,7 @@
   name: "New Email Received",
   description: "Emit new event when a new email is received.",
   type: "source",
-  version: "0.2.8",
+  version: "0.3.0",
   dedupe: "unique",
   props: {
     gmail,
@@ -41,14 +41,20 @@
         intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
       },
     },
+    withTextPayload: {
+      type: "boolean",
+      label: "Return payload as plaintext",
+      description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**",
+      default: false,
+    },
     serviceAccountKeyJson: {
       type: "string",
       label: "Service Account Key JSON",
       optional: true,
       hidden: true,
       reloadProps: true,
     },
     serviceAccountKeyJsonInstructions: {
       type: "alert",
       alertType: "info",
       content: `1) [Create a service account in GCP](https://cloud.google.com/iam/docs/creating-managing-service-accounts) and set the following permission: **Pub/Sub Admin**
@@ -104,7 +110,7 @@
       description: "Emails with the specified labels will be excluded from results",
       optional: true,
     },
     permissionAlert: {
       type: "alert",
       alertType: "error",
       content: `Unable to grant publish permission to Gmail API service account.
@@ -117,7 +123,7 @@
       `,
       hidden: true,
     },
     latencyWarningAlert: {
       type: "alert",
       alertType: "warning",
       content:

components/gmail/sources/new-labeled-email/new-labeled-email.mjs

@@ -8,7 +8,7 @@ export default {
   name: "New Labeled Email",
   description: "Emit new event when a new email is labeled.",
   type: "source",
-  version: "0.0.14",
+  version: "0.1.0",
   dedupe: "unique",
   props: {
     gmail,
@@ -21,6 +21,12 @@ export default {
       type: "string[]",
       label: "Labels",
     },
+    withTextPayload: {
+      type: "boolean",
+      label: "Return payload as plaintext",
+      description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**",
+      default: false,
+    },
   },
   methods: {
     ...common.methods,

components/gmail/sources/new-sent-email/new-sent-email.mjs

@@ -6,7 +6,7 @@ export default {
   key: "gmail-new-sent-email",
   name: "New Sent Email",
   description: "Emit new event for each new email sent. (Maximum of 100 events emited per execution)",
-  version: "0.0.14",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -18,6 +18,12 @@ export default {
         "q",
       ],
     },
+    withTextPayload: {
+      type: "boolean",
+      label: "Return payload as plaintext",
+      description: "Convert the payload response into a single text field. **This reduces the size of the payload and makes it easier for LLMs work with.**",
+      default: false,
+    },
   },
   methods: {
     ...common.methods,