Google Drive - new source changes-to-files-in-drive using Changes API instead of File Subscriptions API

components/google_drive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/google_drive",
-  "version": "1.0.7",
+  "version": "1.1.0",
   "description": "Pipedream Google_drive Components",
   "main": "google_drive.app.mjs",
   "keywords": [

components/google_drive/sources/changes-to-files-in-drive/changes-to-files-in-drive.mjs

@@ -0,0 +1,147 @@
+import common from "../common-webhook.mjs";
+import {
+  MY_DRIVE_VALUE,
+  GOOGLE_DRIVE_NOTIFICATION_ADD,
+  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+  GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+} from "../../common/constants.mjs";
+import commonDedupeChanges from "../common-dedupe-changes.mjs";
+import { stashFile } from "../../common/utils.mjs";
+
+export default {
+  ...common,
+  key: "google_drive-changes-to-files-in-drive",
+  name: "Changes to Files in Drive",
+  description: "Emit new event when a change is made to one of the specified files. [See the documentation](https://developers.google.com/drive/api/v3/reference/changes/watch)",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    infoAlert: {
+      type: "alert",
+      alertType: "info",
+      content: "This source uses `changes.watch` and supports watching 10+ files. To watch for changes to fewer than 10 files, you may want to use the **Changes to Specific Files** source instead (uses `files.watch`).",
+    },
+    ...common.props,
+    drive: {
+      type: "string",
+      label: "Drive",
+      description: "Defaults to `My Drive`. To use a [Shared Drive](https://support.google.com/a/users/answer/9310351), use the **Changes to Specific Files (Shared Drive)** source instead.",
+      optional: true,
+      default: MY_DRIVE_VALUE,
+    },
+    files: {
+      type: "string[]",
+      label: "Files",
+      description: "The files you want to watch for changes.",
+      options({ prevContext }) {
+        const { nextPageToken } = prevContext;
+        return this.googleDrive.listFilesOptions(nextPageToken, this.getListFilesOpts());
+      },
+    },
+    includeLink: {
+      label: "Include Link",
+      type: "boolean",
+      description: "Upload file to your File Stash and emit temporary download link to the file. Google Workspace documents will be converted to PDF. See [the docs](https://pipedream.com/docs/connect/components/files) to learn more about working with files in Pipedream.",
+      default: false,
+      optional: true,
+    },
+    dir: {
+      type: "dir",
+      accessMode: "write",
+      optional: true,
+    },
+    ...commonDedupeChanges.props,
+  },
+  hooks: {
+    async deploy() {
+      const daysAgo = new Date();
+      daysAgo.setDate(daysAgo.getDate() - 30);
+      const timeString = daysAgo.toISOString();
+
+      const args = this.getListFilesOpts({
+        q: `mimeType != "application/vnd.google-apps.folder" and modifiedTime > "${timeString}" and trashed = false`,
+        fields: "files",
+        pageSize: 5,
+      });
+
+      const { files } = await this.googleDrive.listFilesInPage(null, args);
+
+      await this.processChanges(files);
+    },
+    ...common.hooks,
+  },
+  methods: {
+    ...common.methods,
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_ADD,
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+        GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+      ];
+    },
+    generateMeta(data, headers) {
+      const {
+        id: fileId,
+        name: fileName,
+        modifiedTime: tsString,
+      } = data;
+      const ts = Date.parse(tsString);
+      const resourceState = headers && headers["x-goog-resource-state"];
+
+      const summary = resourceState
+        ? `${resourceState.toUpperCase()} - ${fileName || "Untitled"}`
+        : fileName || "Untitled";
+
+      return {
+        id: `${fileId}-${ts}`,
+        summary,
+        ts,
+      };
+    },
+    isFileRelevant(file) {
+      return this.files.includes(file.id);
+    },
+    getChanges(headers) {
+      if (!headers) {
+        return {
+          change: { },
+        };
+      }
+      return {
+        change: {
+          state: headers["x-goog-resource-state"],
+          resourceURI: headers["x-goog-resource-uri"],
+          changed: headers["x-goog-changed"], // "Additional details about the changes. Possible values: content, parents, children, permissions"
+        },
+      };
+    },
+    async processChange(file, headers) {
+      const changes = this.getChanges(headers);
+      const fileInfo = await this.googleDrive.getFile(file.id);
+      if (this.includeLink) {
+        fileInfo.file = await stashFile(file, this.googleDrive, this.dir);
+      }
+      const eventToEmit = {
+        file: fileInfo,
+        ...changes,
+      };
+      const meta = this.generateMeta(fileInfo, headers);
+      this.$emit(eventToEmit, meta);
+    },
+    async processChanges(changedFiles, headers) {
+      console.log(`Processing ${changedFiles.length} changed files`);
+      console.log(`Changed files: ${JSON.stringify(changedFiles, null, 2)}!!!`);
+      console.log(`Files: ${this.files}!!!`);
+
+      const filteredFiles = this.checkMinimumInterval(changedFiles);
+      for (const file of filteredFiles) {
+        if (!this.isFileRelevant(file)) {
+          console.log(`Skipping event for irrelevant file ${file.id}`);
+          continue;
+        }
+        await this.processChange(file, headers);
+      }
+    },
+  },
+};

components/google_drive/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs

@@ -26,9 +26,9 @@
 export default {
   ...common,
   key: "google_drive-changes-to-specific-files-shared-drive",
   name: "Changes to Specific Files (Shared Drive)",
   description: "Watches for changes to specific files in a shared drive, emitting an event when a change is made to one of those files",
-  version: "0.3.0",
+  version: "0.3.1",
   type: "source",
   // Dedupe events based on the "x-goog-message-number" header for the target channel:
   // https://developers.google.com/drive/api/v3/push#making-watch-requests
@@ -92,15 +92,14 @@
         modifiedTime: tsString,
       } = data;
       const ts = Date.parse(tsString);
-      const eventId = headers && headers["x-goog-message-number"];
       const resourceState = headers && headers["x-goog-resource-state"];
 
       const summary = resourceState
         ? `${resourceState.toUpperCase()} - ${fileName || "Untitled"}`
         : fileName || "Untitled";
 
       return {
-        id: `${fileId}-${eventId || ts}`,
+        id: `${fileId}-${ts}`,
         summary,
         ts,
       };
@@ -124,14 +123,15 @@
     },
     async processChange(file, headers) {
       const changes = this.getChanges(headers);
+      const fileInfo = await this.googleDrive.getFile(file.id);
       if (this.includeLink) {
-        file.file = await stashFile(file, this.googleDrive, this.dir);
+        fileInfo.file = await stashFile(file, this.googleDrive, this.dir);
       }
       const eventToEmit = {
-        file,
+        file: fileInfo,
         ...changes,
       };
-      const meta = this.generateMeta(file, headers);
+      const meta = this.generateMeta(fileInfo, headers);
       this.$emit(eventToEmit, meta);
     },
     async processChanges(changedFiles, headers) {

components/google_drive/sources/changes-to-specific-files/changes-to-specific-files.mjs

@@ -3,6 +3,7 @@
 import { v4 as uuid } from "uuid";
 import { MY_DRIVE_VALUE } from "../../common/constants.mjs";
 import changesToSpecificFiles from "../changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs";
+import { ConfigurationError } from "@pipedream/platform";
 import sampleEmit from "./test-event.mjs";
 
 /**
@@ -13,14 +14,19 @@
 export default {
   ...changesToSpecificFiles,
   key: "google_drive-changes-to-specific-files",
   name: "Changes to Specific Files",
   description: "Watches for changes to specific files, emitting an event when a change is made to one of those files. To watch for changes to [shared drive](https://support.google.com/a/users/answer/9310351) files, use the **Changes to Specific Files (Shared Drive)** source instead.",
-  version: "0.3.0",
+  version: "0.3.1",
   type: "source",
   // Dedupe events based on the "x-goog-message-number" header for the target channel:
   // https://developers.google.com/drive/api/v3/push#making-watch-requests
   dedupe: "unique",
   props: {
+    infoAlert: {
+      type: "alert",
+      alertType: "info",
+      content: "This source uses `files.watch` and supports up to 10 file subscriptions. To watch for changes to more than 10 files, use the **Changes to Files in Drive** source instead (uses `changes.watch`).",
+    },
     ...changesToSpecificFiles.props,
     drive: {
       type: "string",
@@ -50,6 +56,12 @@
   },
   hooks: {
     ...changesToSpecificFiles.hooks,
+    async deploy() {
+      if (this.files.length > 10) {
+        throw new ConfigurationError("This source only supports up to 10 files");
+      }
+      await changesToSpecificFiles.hooks.deploy.bind(this)();
+    },
     async activate() {
       // Called when a component is created or updated. Handles all the logic
       // for starting and stopping watch notifications tied to the desired

components/google_drive/sources/common-dedupe-changes.mjs

@@ -1,6 +1,6 @@
 export default {
   props: {
     intervalAlert: {
       type: "alert",
       alertType: "info",
       content: `This source can emit many events in quick succession while a file is being edited. By default, it will not emit another event for the same file for at least 1 minute.
@@ -13,7 +13,7 @@
       description: "How many minutes to wait until the same file can emit another event.\n\nIf set to `0`, this interval is disabled and all events will be emitted.",
       min: 0,
       max: 60,
-      default: 1,
+      default: 3,
       optional: true,
     },
   },

components/google_drive/sources/common-webhook.mjs

@@ -10,7 +10,10 @@ export default {
   props: {
     googleDrive,
     db: "$.service.db",
-    http: "$.interface.http",
+    http: {
+      type: "$.interface.http",
+      customResponse: true,
+    },
     drive: {
       propDefinition: [
         googleDrive,
@@ -157,6 +160,10 @@ export default {
       this._setChannelID(newChannelID);
       this._setPageToken(newPageToken);
       return;
+    } else {
+      this.http.respond({
+        status: 200,
+      });
     }
 
     const { headers } = event;

components/google_drive/sources/new-files-instant/new-files-instant.mjs

@@ -11,7 +11,7 @@ export default {
   key: "google_drive-new-files-instant",
   name: "New Files (Instant)",
   description: "Emit new event when a new file is added in your linked Google Drive",
-  version: "0.2.0",
+  version: "0.2.1",
   type: "source",
   dedupe: "unique",
   props: {

components/google_drive/sources/new-files-shared-drive/new-files-shared-drive.mjs

@@ -7,7 +7,7 @@ export default {
   key: "google_drive-new-files-shared-drive",
   name: "New Files (Shared Drive)",
   description: "Emit new event when a new file is added in your shared Google Drive",
-  version: "0.1.0",
+  version: "0.1.1",
   type: "source",
   dedupe: "unique",
   props: {

components/google_drive/sources/new-or-modified-comments/new-or-modified-comments.mjs

@@ -17,7 +17,7 @@ export default {
   name: "New or Modified Comments (Instant)",
   description:
     "Emit new event when a comment is created or modified in the selected file",
-  version: "1.0.9",
+  version: "1.0.10",
   type: "source",
   // Dedupe events based on the "x-goog-message-number" header for the target channel:
   // https://developers.google.com/drive/api/v3/push#making-watch-requests

components/google_drive/sources/new-or-modified-files/new-or-modified-files.mjs

@@ -25,7 +25,7 @@ export default {
   key: "google_drive-new-or-modified-files",
   name: "New or Modified Files (Instant)",
   description: "Emit new event when a file in the selected Drive is created, modified or trashed.",
-  version: "0.4.0",
+  version: "0.4.1",
   type: "source",
   // Dedupe events based on the "x-goog-message-number" header for the target channel:
   // https://developers.google.com/drive/api/v3/push#making-watch-requests

components/google_drive/sources/new-or-modified-folders/new-or-modified-folders.mjs

@@ -20,7 +20,7 @@ export default {
   key: "google_drive-new-or-modified-folders",
   name: "New or Modified Folders (Instant)",
   description: "Emit new event when a folder is created or modified in the selected Drive",
-  version: "0.2.2",
+  version: "0.2.3",
   type: "source",
   // Dedupe events based on the "x-goog-message-number" header for the target channel:
   // https://developers.google.com/drive/api/v3/push#making-watch-requests

components/google_drive/sources/new-spreadsheet/new-spreadsheet.mjs

@@ -6,7 +6,7 @@ export default {
   type: "source",
   name: "New Spreadsheet (Instant)",
   description: "Emit new event when a new spreadsheet is created in a drive.",
-  version: "0.1.15",
+  version: "0.1.16",
   props: {
     googleDrive: newFilesInstant.props.googleDrive,
     db: newFilesInstant.props.db,