Add optional file download URL to file-related source emits

components/aws/.upm/store.json

@@ -1 +1 @@
-{"version":2,"languages":{"nodejs-npm":{"guessedImports":["aws-sdk","axios","shortid"],"guessedImportsHash":"08190b61ee0169fd35ea81dce6f4754d"}}}
+{"version":2, "languages":{"nodejs-npm":{"guessedImports":["aws-sdk", "axios", "shortid"], "guessedImportsHash":"08190b61ee0169fd35ea81dce6f4754d"}}}

components/aws/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/aws",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "Pipedream Aws Components",
   "main": "aws.app.mjs",
   "keywords": [
@@ -34,6 +34,7 @@
     "@pipedream/platform": "^3.1.0",
     "adm-zip": "^0.5.10",
     "dedent": "^1.5.1",
+    "file-type": "^21.0.0",
     "mailparser": "^3.6.6",
     "mailparser-mit": "^1.0.0",
     "nanoid": "^5.0.4",

components/aws/sources/common/include-link.mjs

@@ -0,0 +1,68 @@
+import { Readable } from "stream";
+import { fileTypeFromBuffer } from "file-type";
+
+export default {
+  props: {
+    includeLink: {
+      label: "Include Link",
+      type: "boolean",
+      description: "Upload file to your File Stash and emit temporary download link to the file. See [the docs](https://pipedream.com/docs/connect/components/files) to learn more about working with files in Pipedream.",
+      default: false,
+      optional: true,
+    },
+    dir: {
+      type: "dir",
+      accessMode: "write",
+      optional: true,
+    },
+  },
+  methods: {
+    async streamToBuffer(stream) {
+      const chunks = [];
+      for await (const chunk of stream) {
+        chunks.push(chunk);
+      }
+      return Buffer.concat(chunks);
+    },
+    async stashFile(item) {
+      const { Body } = await this.getObject({
+        Bucket: item.bucket.name,
+        Key: item.object.key,
+      });
+      const filepath = `${item.bucket.name}/${item.object.key}`;
+      const buffer = await this.streamToBuffer(Body);
+      const type = await fileTypeFromBuffer(buffer);
+      // Upload the attachment to the configured directory (File Stash) so it
+      // can be accessed later.
+      const file = await this.dir.open(filepath).fromReadableStream(
+        Readable.from(buffer),
+        type?.mime,
+        buffer.length,
+      );
+      // Return file details and temporary download link:
+      // { path, get_url, s3Key, type }
+      return await file.withoutPutUrl().withGetUrl();
+    },
+    async processEvent(event) {
+      const { Message: rawMessage } = event.body;
+      const {
+        Records: s3Events = [],
+        Event: eventType,
+      } = JSON.parse(rawMessage);
+
+      if (eventType === "s3:TestEvent") {
+        console.log("Received initial test event. Skipping...");
+        return;
+      }
+
+      for (const s3Event of s3Events) {
+        const meta = this.generateMeta(s3Event);
+        let { s3: item } = s3Event;
+        if (this.includeLink) {
+          item.file = await this.stashFile(item);
+        }
+        this.$emit(item, meta);
+      }
+    },
+  },
+};

components/aws/sources/s3-new-file/s3-new-file.mjs

@@ -1,15 +1,21 @@
 import base from "../common/s3.mjs";
+import includeLink from "../common/include-link.mjs";
 
 export default {
   ...base,
   type: "source",
   key: "aws-s3-new-file",
   name: "New S3 File",
   description: "Emit new event when a file is added to an S3 bucket",
-  version: "0.1.4",
+  version: "0.2.0",
   dedupe: "unique",
+  props: {
+    ...base.props,
+    ...includeLink.props,
+  },
   methods: {
     ...base.methods,
+    ...includeLink.methods,
     getEvents() {
       return [
         "s3:ObjectCreated:*",

components/aws/sources/s3-restored-file/s3-restored-file.mjs

@@ -1,12 +1,13 @@
 import base from "../common/s3.mjs";
+import includeLink from "../common/include-link.mjs";
 
 export default {
   ...base,
   type: "source",
   key: "aws-s3-restored-file",
   name: "New Restored S3 File",
-  description: "Emit new event when an file is restored into a S3 bucket",
-  version: "0.1.4",
+  description: "Emit new event when a file is restored into an S3 bucket",
+  version: "0.2.0",
   dedupe: "unique",
   props: {
     ...base.props,
@@ -16,9 +17,11 @@ export default {
       description: "When enabled, this event source will also emit events whenever a restore is initiated",
       default: false,
     },
+    ...includeLink.props,
   },
   methods: {
     ...base.methods,
+    ...includeLink.methods,
     getEvents() {
       return [
         this.detectRestoreInitiation

components/box/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/box",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Pipedream Box Components",
   "main": "box.app.mjs",
   "keywords": [
@@ -11,6 +11,7 @@
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "dependencies": {
     "@pipedream/platform": "^3.1.0",
+    "file-type": "^21.0.0",
     "form-data": "^4.0.0",
     "path": "^0.12.7",
     "stream": "^0.0.2",

components/box/sources/new-file/new-file.mjs

@@ -1,13 +1,30 @@
 import common from "../common/common.mjs";
+import { Readable } from "stream";
+import { fileTypeFromBuffer } from "file-type";
 
 export default {
   key: "box-new-file",
   name: "New File Event",
-  description: "Emit new event when a new file uploaded on a target. [See the documentation](https://developer.box.com/reference/post-webhooks)",
-  version: "0.0.4",
+  description: "Emit new event when a new file is uploaded to a target. [See the documentation](https://developer.box.com/reference/post-webhooks)",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   ...common,
+  props: {
+    ...common.props,
+    includeLink: {
+      label: "Include Link",
+      type: "boolean",
+      description: "Upload file to your File Stash and emit temporary download link to the file. See [the docs](https://pipedream.com/docs/connect/components/files) to learn more about working with files in Pipedream.",
+      default: false,
+      optional: true,
+    },
+    dir: {
+      type: "dir",
+      accessMode: "write",
+      optional: true,
+    },
+  },
   methods: {
     ...common.methods,
     getTriggers() {
@@ -18,5 +35,29 @@ export default {
     getSummary(event) {
       return  `New file uploaded event with ID(${event.id})`;
     },
+    async stashFile(event) {
+      const response = await this.app.downloadFile({
+        fileId: event.source.id,
+        responseType: "arraybuffer",
+      });
+      const buffer = Buffer.from(response);
+      const filepath = `${event.source.id}/${event.source.name}`;
+      const type = await fileTypeFromBuffer(buffer);
+      const file = await this.dir.open(filepath).fromReadableStream(
+        Readable.from(buffer),
+        type?.mime,
+        buffer.length,
+      );
+      return await file.withoutPutUrl().withGetUrl();
+    },
+  },
+  async run(event) {
+    if (this.includeLink) {
+      event.body.file = await this.stashFile(event.body);
+    }
+    this.$emit(
+      event.body,
+      this.getMetadata(event.body),
+    );
   },
 };

components/egnyte/actions/create-folder/create-folder.mjs

@@ -4,7 +4,7 @@ export default {
   key: "egnyte-create-folder",
   name: "Create Folder",
   description: "Creates a new folder in your Egnyte workspace. [See the documentation](https://developers.egnyte.com/docs/File_System_Management_API_Documentation#Create-a-Folder)",
-  version: "0.0.1",
+  version: "0.0.2",
   type: "action",
   props: {
     egnyte,

components/egnyte/actions/upload-file/upload-file.mjs

@@ -6,7 +6,7 @@ export default {
   key: "egnyte-upload-file",
   name: "Upload File",
   description: "Uploads a file to a specified folder in Egnyte. [See the documentation](https://developers.egnyte.com/docs/File_System_Management_API_Documentation#Upload-a-File)",
-  version: "0.1.1",
+  version: "0.1.2",
   type: "action",
   props: {
     egnyte,

components/egnyte/egnyte.app.mjs

@@ -55,5 +55,14 @@ export default {
         ...opts,
       });
     },
+    downloadFile({
+      folderPath, filename, ...opts
+    }) {
+      return this._makeRequest({
+        path: `/fs-content/${folderPath}/${filename}`,
+        responseType: "arraybuffer",
+        ...opts,
+      });
+    },
   },
 };

components/egnyte/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/egnyte",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "Pipedream Egnyte Components",
   "main": "egnyte.app.mjs",
   "keywords": [

components/egnyte/sources/common/base.mjs

@@ -30,6 +30,12 @@ export default {
     generateMeta() {
       throw new Error("generateMeta is not implemented");
     },
+    emitEvents(items) {
+      items.reverse().forEach((item) => {
+        const meta = this.generateMeta(item);
+        this.$emit(item, meta);
+      });
+    },
   },
   async run() {
     const lastTs = this._getLastTs();
@@ -67,10 +73,7 @@ export default {
         }
       }
 
-      newItems.reverse().forEach((item) => {
-        const meta = this.generateMeta(item);
-        this.$emit(item, meta);
-      });
+      await this.emitEvents(newItems);
     };
 
     await processFolder(this.folderPath);

components/egnyte/sources/new-file-in-folder/new-file-in-folder.mjs

@@ -1,13 +1,30 @@
 import common from "../common/base.mjs";
+import { Readable } from "stream";
+import { fileTypeFromBuffer } from "file-type";
 
 export default {
   ...common,
   key: "egnyte-new-file-in-folder",
   name: "New File in Folder",
   description: "Emit new event when a file is added within the specified folder in Egnyte. [See the documentation](https://developers.egnyte.com/docs/read/File_System_Management_API_Documentation#List-File-or-Folder)",
-  version: "0.0.1",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
+  props: {
+    ...common.props,
+    includeLink: {
+      label: "Include Link",
+      type: "boolean",
+      description: "Upload attachment to your File Stash and emit temporary download link to the file. See [the docs](https://pipedream.com/docs/connect/components/files) to learn more about working with files in Pipedream.",
+      default: false,
+      optional: true,
+    },
+    dir: {
+      type: "dir",
+      accessMode: "write",
+      optional: true,
+    },
+  },
   methods: {
     ...common.methods,
     getResourceType() {
@@ -20,5 +37,29 @@ export default {
         ts: file.uploaded,
       };
     },
+    async stashFile(item) {
+      const response = await this.egnyte.downloadFile({
+        folderPath: this.folderPath,
+        filename: item.name,
+      });
+      const buffer = Buffer.from(response);
+      const filepath = `${item.entry_id}/${item.name}`;
+      const type = await fileTypeFromBuffer(buffer);
+      const file = await this.dir.open(filepath).fromReadableStream(
+        Readable.from(buffer),
+        type?.mime,
+        buffer.length,
+      );
+      return await file.withoutPutUrl().withGetUrl();
+    },
+    async emitEvents(items) {
+      for (const item of items) {
+        if (this.includeLink) {
+          item.file = await this.stashFile(item);
+        }
+        const meta = this.generateMeta(item);
+        this.$emit(item, meta);
+      };
+    },
   },
 };

components/egnyte/sources/new-folder-added/new-folder-added.mjs

@@ -5,7 +5,7 @@ export default {
   key: "egnyte-new-folder-added",
   name: "New Folder",
   description: "Emit new event when a folder is added within the specified folder in Egnyte. [See the documentation](https://developers.egnyte.com/docs/read/File_System_Management_API_Documentation#List-File-or-Folder).",
-  version: "0.0.1",
+  version: "0.0.2",
   type: "source",
   dedupe: "unique",
   methods: {

components/gmail/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/gmail",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "Pipedream Gmail Components",
   "main": "gmail.app.mjs",
   "keywords": [
@@ -21,6 +21,7 @@
     "html-to-text": "^9.0.5",
     "mime": "^3.0.0",
     "nodemailer": "^6.7.8",
+    "stream": "^0.0.3",
     "uuid": "^10.0.0"
   }
 }

components/gmail/sources/common/polling-history.mjs

@@ -187,7 +187,7 @@ export default {
         console.log(`Message ${id} not found: ${error.message}`);
         return;
       }
-      this.emitEvent(message);
+      await this.emitEvent(message);
     },
   },
   async run() {

components/gmail/sources/common/polling-messages.mjs

@@ -49,7 +49,7 @@ export default {
       let maxDate = lastDate;
       const messages = this.gmail.getAllMessages(messageIds);
       for await (const message of messages) {
-        this.emitEvent(message);
+        await this.emitEvent(message);
         maxDate = Math.max(maxDate, message.internalDate);
       }
       if (maxDate !== lastDate) {
@@ -63,7 +63,9 @@ export default {
       }
       messages = messages.sort((a, b) => (a.internalDate - b.internalDate));
       this.setLastDate(messages[messages.length - 1].internalDate);
-      messages.forEach((message) => this.emitEvent(message));
+      for (const message of messages) {
+        await this.emitEvent(message);
+      }
     },
   },
   async run() {