[Gmail] fix last date (#16205)

* truncate timestamp in query

* additional improvements

* bump versions

* pnpm

Author: andrewjschuang

components/gmail/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/gmail",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "description": "Pipedream Gmail Components",
   "main": "gmail.app.mjs",
   "keywords": [

components/gmail/sources/common/polling-messages.mjs

@@ -7,10 +7,9 @@ export default {
     ...common.hooks,
     async deploy() {
       const messageIds = await this.getMessageIds(constants.HISTORICAL_EVENTS);
-      if (!messageIds?.length) {
-        return;
+      if (messageIds.length) {
+        await this.processHistoricalEvents(messageIds);
       }
-      await this.processHistoricalEvents(messageIds);
     },
     async activate() {
       console.log(`Previous lastDate: ${this.getLastDate()}`);
@@ -24,37 +23,38 @@ export default {
       return this.db.get("lastDate");
     },
     setLastDate(lastDate) {
-      this.db.set("lastDate", lastDate);
+      this.db.set("lastDate", parseInt(lastDate));
     },
     constructQuery(lastDate) {
       const { q: query } = this;
       const after = !query?.includes("after:") && lastDate
-        ? `after:${lastDate / 1000}`
+        ? `after:${Math.trunc(lastDate / 1000)}`
         : "";
-      return [
+      const q = [
         after,
         query,
       ].join(" ").trim();
+      console.log(`Polling for new messages with query: ${q}`);
+      return q;
     },
     async getMessageIds(max, lastDate = 0) {
-      console.log("Polling for new messages...");
       const { messages } = await this.gmail.listMessages({
         q: this.constructQuery(lastDate),
         labelIds: this.getLabels(),
         maxResults: max,
       });
-      return messages?.map((message) => message.id);
+      return messages?.map((message) => message.id) ?? [];
     },
     async processMessageIds(messageIds, lastDate) {
       let maxDate = lastDate;
       const messages = this.gmail.getAllMessages(messageIds);
       for await (const message of messages) {
-        if (message.internalDate >= lastDate) {
-          this.emitEvent(message);
-          maxDate = Math.max(maxDate, message.internalDate);
-        }
+        this.emitEvent(message);
+        maxDate = Math.max(maxDate, message.internalDate);
+      }
+      if (maxDate !== lastDate) {
+        this.setLastDate(maxDate);
       }
-      if (maxDate) this.setLastDate(maxDate);
     },
     async processHistoricalEvents(messageIds) {
       let messages = await this.gmail.getMessages(messageIds);
@@ -66,10 +66,11 @@ export default {
   async run() {
     const lastDate = this.getLastDate();
     const messageIds = await this.getMessageIds(constants.DEFAULT_LIMIT, lastDate);
-    if (!messageIds?.length) {
+    if (messageIds.length) {
+      console.log(`Processing ${messageIds.length} message(s)...`);
+      await this.processMessageIds(messageIds.reverse(), lastDate);
+    } else {
       console.log("There are no new messages. Exiting...");
-      return;
     }
-    await this.processMessageIds(messageIds.reverse(), lastDate);
   },
 };

components/gmail/sources/new-attachment-received/new-attachment-received.mjs

@@ -6,7 +6,7 @@ export default {
   key: "gmail-new-attachment-received",
   name: "New Attachment Received",
   description: "Emit new event for each attachment in a message received. This source is capped at 100 max new messages per run.",
-  version: "0.0.9",
+  version: "0.0.10",
   type: "source",
   dedupe: "unique",
   props: {
@@ -31,18 +31,13 @@ export default {
   methods: {
     ...common.methods,
     constructQuery(lastDate) {
-      const { q: query } = this;
-      const hasAttachment = query?.includes("has:attachment")
-        ? ""
-        : "has:attachment";
-      const after = !query?.includes("after:") && lastDate
-        ? `after:${lastDate / 1000}`
-        : "";
-      return [
-        hasAttachment,
-        after,
-        query,
-      ].join(" ").trim();
+      const hasAttachment = "has:attachment";
+      if (!this.q) {
+        this.q = hasAttachment;
+      } else if (!this.q.includes(hasAttachment)) {
+        this.q = `${this.q} ${hasAttachment}`;
+      }
+      return common.methods.constructQuery.call(this, lastDate);
     },
     getLabels() {
       return this.labels;

components/gmail/sources/new-email-matching-search/new-email-matching-search.mjs

@@ -6,7 +6,7 @@ export default {
   key: "gmail-new-email-matching-search",
   name: "New Email Matching Search",
   description: "Emit new event when an email matching the search criteria is received. This source is capped at 100 max new messages per run.",
-  version: "0.0.7",
+  version: "0.0.8",
   type: "source",
   dedupe: "unique",
   props: {

components/gmail/sources/new-sent-email/new-sent-email.mjs

@@ -6,7 +6,7 @@ export default {
   key: "gmail-new-sent-email",
   name: "New Sent Email",
   description: "Emit new event for each new email sent. (Maximum of 100 events emited per execution)",
-  version: "0.0.8",
+  version: "0.0.9",
   type: "source",
   dedupe: "unique",
   props: {