diff --git a/components/google_drive/package.json b/components/google_drive/package.json index 7a751e83c3876..7e9f715042b25 100644 --- a/components/google_drive/package.json +++ b/components/google_drive/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/google_drive", - "version": "1.0.7", + "version": "1.1.0", "description": "Pipedream Google_drive Components", "main": "google_drive.app.mjs", "keywords": [ diff --git a/components/google_drive/sources/changes-to-files-in-drive/changes-to-files-in-drive.mjs b/components/google_drive/sources/changes-to-files-in-drive/changes-to-files-in-drive.mjs new file mode 100644 index 0000000000000..7b6d61082bb2e --- /dev/null +++ b/components/google_drive/sources/changes-to-files-in-drive/changes-to-files-in-drive.mjs @@ -0,0 +1,147 @@ +import common from "../common-webhook.mjs"; +import { + MY_DRIVE_VALUE, + GOOGLE_DRIVE_NOTIFICATION_ADD, + GOOGLE_DRIVE_NOTIFICATION_CHANGE, + GOOGLE_DRIVE_NOTIFICATION_UPDATE, +} from "../../common/constants.mjs"; +import commonDedupeChanges from "../common-dedupe-changes.mjs"; +import { stashFile } from "../../common/utils.mjs"; + +export default { + ...common, + key: "google_drive-changes-to-files-in-drive", + name: "Changes to Files in Drive", + description: "Emit new event when a change is made to one of the specified files. [See the documentation](https://developers.google.com/drive/api/v3/reference/changes/watch)", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + infoAlert: { + type: "alert", + alertType: "info", + content: "This source uses `changes.watch` and supports watching 10+ files. To watch for changes to fewer than 10 files, you may want to use the **Changes to Specific Files** source instead (uses `files.watch`).", + }, + ...common.props, + drive: { + type: "string", + label: "Drive", + description: "Defaults to `My Drive`. To use a [Shared Drive](https://support.google.com/a/users/answer/9310351), use the **Changes to Specific Files (Shared Drive)** source instead.", + optional: true, + default: MY_DRIVE_VALUE, + }, + files: { + type: "string[]", + label: "Files", + description: "The files you want to watch for changes.", + options({ prevContext }) { + const { nextPageToken } = prevContext; + return this.googleDrive.listFilesOptions(nextPageToken, this.getListFilesOpts()); + }, + }, + includeLink: { + label: "Include Link", + type: "boolean", + description: "Upload file to your File Stash and emit temporary download link to the file. Google Workspace documents will be converted to PDF. See [the docs](https://pipedream.com/docs/connect/components/files) to learn more about working with files in Pipedream.", + default: false, + optional: true, + }, + dir: { + type: "dir", + accessMode: "write", + optional: true, + }, + ...commonDedupeChanges.props, + }, + hooks: { + async deploy() { + const daysAgo = new Date(); + daysAgo.setDate(daysAgo.getDate() - 30); + const timeString = daysAgo.toISOString(); + + const args = this.getListFilesOpts({ + q: `mimeType != "application/vnd.google-apps.folder" and modifiedTime > "${timeString}" and trashed = false`, + fields: "files", + pageSize: 5, + }); + + const { files } = await this.googleDrive.listFilesInPage(null, args); + + await this.processChanges(files); + }, + ...common.hooks, + }, + methods: { + ...common.methods, + getUpdateTypes() { + return [ + GOOGLE_DRIVE_NOTIFICATION_ADD, + GOOGLE_DRIVE_NOTIFICATION_CHANGE, + GOOGLE_DRIVE_NOTIFICATION_UPDATE, + ]; + }, + generateMeta(data, headers) { + const { + id: fileId, + name: fileName, + modifiedTime: tsString, + } = data; + const ts = Date.parse(tsString); + const resourceState = headers && headers["x-goog-resource-state"]; + + const summary = resourceState + ? `${resourceState.toUpperCase()} - ${fileName || "Untitled"}` + : fileName || "Untitled"; + + return { + id: `${fileId}-${ts}`, + summary, + ts, + }; + }, + isFileRelevant(file) { + return this.files.includes(file.id); + }, + getChanges(headers) { + if (!headers) { + return { + change: { }, + }; + } + return { + change: { + state: headers["x-goog-resource-state"], + resourceURI: headers["x-goog-resource-uri"], + changed: headers["x-goog-changed"], // "Additional details about the changes. Possible values: content, parents, children, permissions" + }, + }; + }, + async processChange(file, headers) { + const changes = this.getChanges(headers); + const fileInfo = await this.googleDrive.getFile(file.id); + if (this.includeLink) { + fileInfo.file = await stashFile(file, this.googleDrive, this.dir); + } + const eventToEmit = { + file: fileInfo, + ...changes, + }; + const meta = this.generateMeta(fileInfo, headers); + this.$emit(eventToEmit, meta); + }, + async processChanges(changedFiles, headers) { + console.log(`Processing ${changedFiles.length} changed files`); + console.log(`Changed files: ${JSON.stringify(changedFiles, null, 2)}!!!`); + console.log(`Files: ${this.files}!!!`); + + const filteredFiles = this.checkMinimumInterval(changedFiles); + for (const file of filteredFiles) { + if (!this.isFileRelevant(file)) { + console.log(`Skipping event for irrelevant file ${file.id}`); + continue; + } + await this.processChange(file, headers); + } + }, + }, +}; diff --git a/components/google_drive/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs b/components/google_drive/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs index deb811da157ef..28d095bfbb977 100644 --- a/components/google_drive/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs +++ b/components/google_drive/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs @@ -28,7 +28,7 @@ export default { key: "google_drive-changes-to-specific-files-shared-drive", name: "Changes to Specific Files (Shared Drive)", description: "Watches for changes to specific files in a shared drive, emitting an event when a change is made to one of those files", - version: "0.3.0", + version: "0.3.1", type: "source", // Dedupe events based on the "x-goog-message-number" header for the target channel: // https://developers.google.com/drive/api/v3/push#making-watch-requests @@ -92,7 +92,6 @@ export default { modifiedTime: tsString, } = data; const ts = Date.parse(tsString); - const eventId = headers && headers["x-goog-message-number"]; const resourceState = headers && headers["x-goog-resource-state"]; const summary = resourceState @@ -100,7 +99,7 @@ export default { : fileName || "Untitled"; return { - id: `${fileId}-${eventId || ts}`, + id: `${fileId}-${ts}`, summary, ts, }; @@ -124,14 +123,15 @@ export default { }, async processChange(file, headers) { const changes = this.getChanges(headers); + const fileInfo = await this.googleDrive.getFile(file.id); if (this.includeLink) { - file.file = await stashFile(file, this.googleDrive, this.dir); + fileInfo.file = await stashFile(file, this.googleDrive, this.dir); } const eventToEmit = { - file, + file: fileInfo, ...changes, }; - const meta = this.generateMeta(file, headers); + const meta = this.generateMeta(fileInfo, headers); this.$emit(eventToEmit, meta); }, async processChanges(changedFiles, headers) { diff --git a/components/google_drive/sources/changes-to-specific-files/changes-to-specific-files.mjs b/components/google_drive/sources/changes-to-specific-files/changes-to-specific-files.mjs index dc9393736716d..b647babbb53f3 100644 --- a/components/google_drive/sources/changes-to-specific-files/changes-to-specific-files.mjs +++ b/components/google_drive/sources/changes-to-specific-files/changes-to-specific-files.mjs @@ -3,6 +3,7 @@ import includes from "lodash/includes.js"; import { v4 as uuid } from "uuid"; import { MY_DRIVE_VALUE } from "../../common/constants.mjs"; import changesToSpecificFiles from "../changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs"; +import { ConfigurationError } from "@pipedream/platform"; import sampleEmit from "./test-event.mjs"; /** @@ -15,12 +16,17 @@ export default { key: "google_drive-changes-to-specific-files", name: "Changes to Specific Files", description: "Watches for changes to specific files, emitting an event when a change is made to one of those files. To watch for changes to [shared drive](https://support.google.com/a/users/answer/9310351) files, use the **Changes to Specific Files (Shared Drive)** source instead.", - version: "0.3.0", + version: "0.3.1", type: "source", // Dedupe events based on the "x-goog-message-number" header for the target channel: // https://developers.google.com/drive/api/v3/push#making-watch-requests dedupe: "unique", props: { + infoAlert: { + type: "alert", + alertType: "info", + content: "This source uses `files.watch` and supports up to 10 file subscriptions. To watch for changes to more than 10 files, use the **Changes to Files in Drive** source instead (uses `changes.watch`).", + }, ...changesToSpecificFiles.props, drive: { type: "string", @@ -50,6 +56,12 @@ export default { }, hooks: { ...changesToSpecificFiles.hooks, + async deploy() { + if (this.files.length > 10) { + throw new ConfigurationError("This source only supports up to 10 files"); + } + await changesToSpecificFiles.hooks.deploy.bind(this)(); + }, async activate() { // Called when a component is created or updated. Handles all the logic // for starting and stopping watch notifications tied to the desired diff --git a/components/google_drive/sources/common-dedupe-changes.mjs b/components/google_drive/sources/common-dedupe-changes.mjs index dea635457bda8..070589ccb6ade 100644 --- a/components/google_drive/sources/common-dedupe-changes.mjs +++ b/components/google_drive/sources/common-dedupe-changes.mjs @@ -13,7 +13,7 @@ You can change or disable this minimum interval using the prop \`Minimum Interva description: "How many minutes to wait until the same file can emit another event.\n\nIf set to `0`, this interval is disabled and all events will be emitted.", min: 0, max: 60, - default: 1, + default: 3, optional: true, }, }, diff --git a/components/google_drive/sources/common-webhook.mjs b/components/google_drive/sources/common-webhook.mjs index 0448ab485f448..e78544ea434f9 100644 --- a/components/google_drive/sources/common-webhook.mjs +++ b/components/google_drive/sources/common-webhook.mjs @@ -10,7 +10,10 @@ export default { props: { googleDrive, db: "$.service.db", - http: "$.interface.http", + http: { + type: "$.interface.http", + customResponse: true, + }, drive: { propDefinition: [ googleDrive, @@ -157,6 +160,10 @@ export default { this._setChannelID(newChannelID); this._setPageToken(newPageToken); return; + } else { + this.http.respond({ + status: 200, + }); } const { headers } = event; diff --git a/components/google_drive/sources/new-files-instant/new-files-instant.mjs b/components/google_drive/sources/new-files-instant/new-files-instant.mjs index 542f1d210cc9a..386715bf6812e 100644 --- a/components/google_drive/sources/new-files-instant/new-files-instant.mjs +++ b/components/google_drive/sources/new-files-instant/new-files-instant.mjs @@ -11,7 +11,7 @@ export default { key: "google_drive-new-files-instant", name: "New Files (Instant)", description: "Emit new event when a new file is added in your linked Google Drive", - version: "0.2.0", + version: "0.2.1", type: "source", dedupe: "unique", props: { diff --git a/components/google_drive/sources/new-files-shared-drive/new-files-shared-drive.mjs b/components/google_drive/sources/new-files-shared-drive/new-files-shared-drive.mjs index 726892a5590d4..7ba93278da3b0 100644 --- a/components/google_drive/sources/new-files-shared-drive/new-files-shared-drive.mjs +++ b/components/google_drive/sources/new-files-shared-drive/new-files-shared-drive.mjs @@ -7,7 +7,7 @@ export default { key: "google_drive-new-files-shared-drive", name: "New Files (Shared Drive)", description: "Emit new event when a new file is added in your shared Google Drive", - version: "0.1.0", + version: "0.1.1", type: "source", dedupe: "unique", props: { diff --git a/components/google_drive/sources/new-or-modified-comments/new-or-modified-comments.mjs b/components/google_drive/sources/new-or-modified-comments/new-or-modified-comments.mjs index 1a7b44748bf36..dd9de452e6367 100644 --- a/components/google_drive/sources/new-or-modified-comments/new-or-modified-comments.mjs +++ b/components/google_drive/sources/new-or-modified-comments/new-or-modified-comments.mjs @@ -17,7 +17,7 @@ export default { name: "New or Modified Comments (Instant)", description: "Emit new event when a comment is created or modified in the selected file", - version: "1.0.9", + version: "1.0.10", type: "source", // Dedupe events based on the "x-goog-message-number" header for the target channel: // https://developers.google.com/drive/api/v3/push#making-watch-requests diff --git a/components/google_drive/sources/new-or-modified-files/new-or-modified-files.mjs b/components/google_drive/sources/new-or-modified-files/new-or-modified-files.mjs index 2db84962a6505..63d597fe06109 100644 --- a/components/google_drive/sources/new-or-modified-files/new-or-modified-files.mjs +++ b/components/google_drive/sources/new-or-modified-files/new-or-modified-files.mjs @@ -25,7 +25,7 @@ export default { key: "google_drive-new-or-modified-files", name: "New or Modified Files (Instant)", description: "Emit new event when a file in the selected Drive is created, modified or trashed.", - version: "0.4.0", + version: "0.4.1", type: "source", // Dedupe events based on the "x-goog-message-number" header for the target channel: // https://developers.google.com/drive/api/v3/push#making-watch-requests diff --git a/components/google_drive/sources/new-or-modified-folders/new-or-modified-folders.mjs b/components/google_drive/sources/new-or-modified-folders/new-or-modified-folders.mjs index 7d11ba872f060..8e404e2143d7e 100644 --- a/components/google_drive/sources/new-or-modified-folders/new-or-modified-folders.mjs +++ b/components/google_drive/sources/new-or-modified-folders/new-or-modified-folders.mjs @@ -20,7 +20,7 @@ export default { key: "google_drive-new-or-modified-folders", name: "New or Modified Folders (Instant)", description: "Emit new event when a folder is created or modified in the selected Drive", - version: "0.2.2", + version: "0.2.3", type: "source", // Dedupe events based on the "x-goog-message-number" header for the target channel: // https://developers.google.com/drive/api/v3/push#making-watch-requests diff --git a/components/google_drive/sources/new-spreadsheet/new-spreadsheet.mjs b/components/google_drive/sources/new-spreadsheet/new-spreadsheet.mjs index 63781464ce77f..b5aaf4e42c19c 100644 --- a/components/google_drive/sources/new-spreadsheet/new-spreadsheet.mjs +++ b/components/google_drive/sources/new-spreadsheet/new-spreadsheet.mjs @@ -6,7 +6,7 @@ export default { type: "source", name: "New Spreadsheet (Instant)", description: "Emit new event when a new spreadsheet is created in a drive.", - version: "0.1.15", + version: "0.1.16", props: { googleDrive: newFilesInstant.props.googleDrive, db: newFilesInstant.props.db,