Gmail - increase fetch limit (PipedreamHQ#17167)

andrewjschuang · joy-chanboop · commit 99d536951c06 · 2025-06-21T11:28:43.000+08:00
* remove get messages function that uses promise all

* increase max limit

* bump versions
diff --git a/components/gmail/actions/add-label-to-email/add-label-to-email.mjs b/components/gmail/actions/add-label-to-email/add-label-to-email.mjs
@@ -4,7 +4,7 @@ export default {
   key: "gmail-add-label-to-email",
   name: "Add Label to Email",
   description: "Add label(s) to an email message. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/modify)",
-  version: "0.0.10",
+  version: "0.0.11",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/actions/approve-workflow/approve-workflow.mjs b/components/gmail/actions/approve-workflow/approve-workflow.mjs
@@ -4,7 +4,7 @@ export default {
   key: "gmail-approve-workflow",
   name: "Approve Workflow",
   description: "Suspend the workflow until approved by email. [See the documentation](https://pipedream.com/docs/code/nodejs/rerun#flowsuspend)",
-  version: "0.0.7",
+  version: "0.0.8",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/actions/archive-email/archive-email.mjs b/components/gmail/actions/archive-email/archive-email.mjs
@@ -5,7 +5,7 @@ export default {
   key: "gmail-archive-email",
   name: "Archive Email",
   description: "Archive an email message. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/modify)",
-  version: "0.0.5",
+  version: "0.0.6",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/actions/create-draft/create-draft.mjs b/components/gmail/actions/create-draft/create-draft.mjs
@@ -6,7 +6,7 @@ export default {
   key: "gmail-create-draft",
   name: "Create Draft",
   description: "Create a draft from your Google Workspace email account. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.drafts/create)",
-  version: "0.1.5",
+  version: "0.1.6",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/actions/download-attachment/download-attachment.mjs b/components/gmail/actions/download-attachment/download-attachment.mjs
@@ -7,7 +7,7 @@ export default {
   key: "gmail-download-attachment",
   name: "Download Attachment",
   description: "Download an attachment by attachmentId to the /tmp directory. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.messages.attachments/get)",
-  version: "0.0.9",
+  version: "0.0.10",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/actions/find-email/find-email.mjs b/components/gmail/actions/find-email/find-email.mjs
@@ -6,7 +6,7 @@ export default {
   key: "gmail-find-email",
   name: "Find Email",
   description: "Find an email using Google's Search Engine. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/list)",
-  version: "0.1.3",
+  version: "0.1.4",
   type: "action",
   props: {
     gmail,
@@ -55,9 +55,10 @@ export default {
       maxResults: this.maxResults,
     });
     const messageIds = messages.map(({ id }) => id);
-    let messagesToEmit = await this.gmail.getMessages(messageIds);
+    const messagesToEmit = [];
+    for await (const message of this.gmail.getAllMessages(messageIds)) {
+      messagesToEmit.push(message);
 
-    for await (const message of messagesToEmit) {
       let newPayload = "";
 
       const messageIdHeader = message.payload?.headers?.find(
diff --git a/components/gmail/actions/list-labels/list-labels.mjs b/components/gmail/actions/list-labels/list-labels.mjs
@@ -4,7 +4,7 @@ export default {
   key: "gmail-list-labels",
   name: "List Labels",
   description: "List all the existing labels in the connected account. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.labels/list)",
-  version: "0.0.6",
+  version: "0.0.7",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/actions/remove-label-from-email/remove-label-from-email.mjs b/components/gmail/actions/remove-label-from-email/remove-label-from-email.mjs
@@ -4,7 +4,7 @@ export default {
   key: "gmail-remove-label-from-email",
   name: "Remove Label from Email",
   description: "Remove label(s) from an email message. [See the docs](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/modify)",
-  version: "0.0.8",
+  version: "0.0.9",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/actions/send-email/send-email.mjs b/components/gmail/actions/send-email/send-email.mjs
@@ -6,7 +6,7 @@ export default {
   key: "gmail-send-email",
   name: "Send Email",
   description: "Send an email from your Google Workspace email account. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.messages/send)",
-  version: "0.1.13",
+  version: "0.1.14",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/actions/update-org-signature/update-org-signature.mjs b/components/gmail/actions/update-org-signature/update-org-signature.mjs
@@ -8,7 +8,7 @@ export default {
   name: "Update Signature for Email in Organization",
   description: `Update the signature for a specific email address in an organization.
     A Google Cloud service account with delegated domain-wide authority is required for this action. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.settings.sendAs/update)`,
-  version: "0.0.9",
+  version: "0.0.10",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/actions/update-primary-signature/update-primary-signature.mjs b/components/gmail/actions/update-primary-signature/update-primary-signature.mjs
@@ -4,7 +4,7 @@ export default {
   key: "gmail-update-primary-signature",
   name: "Update Signature for Primary Email Address",
   description: "Update the signature for the primary email address. [See the documentation](https://developers.google.com/gmail/api/reference/rest/v1/users.settings.sendAs/update)",
-  version: "0.0.9",
+  version: "0.0.10",
   type: "action",
   props: {
     gmail,
diff --git a/components/gmail/common/constants.mjs b/components/gmail/common/constants.mjs
@@ -5,12 +5,14 @@ const BODY_TYPES = {
 };
 const HISTORICAL_EVENTS = 10;
 const DEFAULT_LIMIT = 100;
+const MAX_LIMIT = 500;
 const INBOX_LABEL_ID = "INBOX";
 
 export default {
   USER_ID,
   BODY_TYPES,
   HISTORICAL_EVENTS,
   DEFAULT_LIMIT,
+  MAX_LIMIT,
   INBOX_LABEL_ID,
 };
diff --git a/components/gmail/gmail.app.mjs b/components/gmail/gmail.app.mjs
@@ -410,12 +410,6 @@ export default {
       const { value: subject } = message.payload.headers.find(({ name }) => name === "Subject");
       return subject;
     },
-    async getMessages(ids = []) {
-      const promises = ids.map((id) => this.getMessage({
-        id,
-      }));
-      return Promise.all(promises);
-    },
     async *getAllMessages(ids = []) {
       for (const id of ids) {
         const message = await this.getMessage({
diff --git a/components/gmail/package.json b/components/gmail/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/gmail",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Pipedream Gmail Components",
   "main": "gmail.app.mjs",
   "keywords": [
diff --git a/components/gmail/sources/common/polling-history.mjs b/components/gmail/sources/common/polling-history.mjs
@@ -60,9 +60,13 @@ export default {
         index === self.findIndex((m) => m.id === message.id));
 
       const messageIds = uniqueMessages.map(({ id }) => id);
-      const messagesWithHistoryId = await this.gmail.getMessages(messageIds);
+      const messages = [];
+      for await (const message of this.gmail.getAllMessages(messageIds)) {
+        messages.push(message);
+      }
+
       const sortedMessages =
-        Array.from(messagesWithHistoryId)
+        Array.from(messages)
           .sort((a, b) => (Number(b.historyId) - Number(a.historyId)));
 
       const { historyId } = await this.gmail.getMessage({
@@ -74,6 +78,7 @@ export default {
       const opts = {
         startHistoryId: String(startHistoryId),
         historyTypes: this.getHistoryTypes(),
+        maxResults: constants.MAX_LIMIT,
       };
 
       const length = this.labels?.length > 0
diff --git a/components/gmail/sources/common/polling-messages.mjs b/components/gmail/sources/common/polling-messages.mjs
@@ -57,7 +57,10 @@ export default {
       }
     },
     async processHistoricalEvents(messageIds) {
-      let messages = await this.gmail.getMessages(messageIds);
+      let messages = [];
+      for await (const message of this.gmail.getAllMessages(messageIds)) {
+        messages.push(message);
+      }
       messages = messages.sort((a, b) => (a.internalDate - b.internalDate));
       this.setLastDate(messages[messages.length - 1].internalDate);
       messages.forEach((message) => this.emitEvent(message));
diff --git a/components/gmail/sources/new-attachment-received/new-attachment-received.mjs b/components/gmail/sources/new-attachment-received/new-attachment-received.mjs
@@ -6,7 +6,7 @@ export default {
   key: "gmail-new-attachment-received",
   name: "New Attachment Received",
   description: "Emit new event for each attachment in a message received. This source is capped at 100 max new messages per run.",
-  version: "0.0.14",
+  version: "0.0.15",
   type: "source",
   dedupe: "unique",
   props: {
diff --git a/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs b/components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs
@@ -6,7 +6,7 @@ export default {
   key: "gmail-new-email-matching-search",
   name: "New Email Matching Search",
   description: "Emit new event when an email matching the search criteria is received. This source is capped at 100 max new messages per run.",
-  version: "0.0.12",
+  version: "0.0.13",
   type: "source",
   dedupe: "unique",
   props: {
diff --git a/components/gmail/sources/new-email-received/new-email-received.mjs b/components/gmail/sources/new-email-received/new-email-received.mjs
@@ -15,7 +15,7 @@ export default {
   name: "New Email Received",
   description: "Emit new event when a new email is received.",
   type: "source",
-  version: "0.2.7",
+  version: "0.2.8",
   dedupe: "unique",
   props: {
     gmail,
@@ -112,7 +112,7 @@ export default {
       \n2. Select "View Permissions" for the topic you intend to use for this source.
       \n3. Click "ADD PRINCIPAL"
       \n4. Select "Pub/Sub Publisher" for the Role.
-      \n5. Enter \`serviceAccount:gmail-api-push@system.gserviceaccount.com\` as the principal. 
+      \n5. Enter \`serviceAccount:gmail-api-push@system.gserviceaccount.com\` as the principal.
       \n6. Click "Save"
       `,
       hidden: true,
@@ -311,7 +311,7 @@ export default {
                 \n2. Select "View Permissions" for the topic you intend to use for this source.
                 \n3. Click "ADD PRINCIPAL"
                 \n4. Select "Pub/Sub Publisher" for the Role.
-                \n5. Enter \`serviceAccount:gmail-api-push@system.gserviceaccount.com\` as the principal. 
+                \n5. Enter \`serviceAccount:gmail-api-push@system.gserviceaccount.com\` as the principal.
                 \n6. Click "Save"
                 `,
                 hidden: false,
diff --git a/components/gmail/sources/new-labeled-email/new-labeled-email.mjs b/components/gmail/sources/new-labeled-email/new-labeled-email.mjs
@@ -8,7 +8,7 @@ export default {
   name: "New Labeled Email",
   description: "Emit new event when a new email is labeled.",
   type: "source",
-  version: "0.0.13",
+  version: "0.0.14",
   dedupe: "unique",
   props: {
     gmail,
diff --git a/components/gmail/sources/new-sent-email/new-sent-email.mjs b/components/gmail/sources/new-sent-email/new-sent-email.mjs
@@ -6,7 +6,7 @@ export default {
   key: "gmail-new-sent-email",
   name: "New Sent Email",
   description: "Emit new event for each new email sent. (Maximum of 100 events emited per execution)",
-  version: "0.0.13",
+  version: "0.0.14",
   type: "source",
   dedupe: "unique",
   props: {
