Add optional file download URL to file-related source emits

components/aws/.upm/store.json

@@ -1 +1 @@
-{"version":2,"languages":{"nodejs-npm":{"guessedImports":["aws-sdk","axios","shortid"],"guessedImportsHash":"08190b61ee0169fd35ea81dce6f4754d"}}}
+{"version":2, "languages":{"nodejs-npm":{"guessedImports":["aws-sdk", "axios", "shortid"], "guessedImportsHash":"08190b61ee0169fd35ea81dce6f4754d"}}}

components/aws/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/aws",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "Pipedream Aws Components",
   "main": "aws.app.mjs",
   "keywords": [
@@ -34,6 +34,7 @@
     "@pipedream/platform": "^3.1.0",
     "adm-zip": "^0.5.10",
     "dedent": "^1.5.1",
+    "file-type": "^21.0.0",
     "mailparser": "^3.6.6",
     "mailparser-mit": "^1.0.0",
     "nanoid": "^5.0.4",

components/aws/sources/common/include-link.mjs

@@ -0,0 +1,68 @@
+import { Readable } from "stream";
+import { fileTypeFromBuffer } from "file-type";
+
+export default {
+  props: {
+    includeLink: {
+      label: "Include Link",
+      type: "boolean",
+      description: "Upload file to your File Stash and emit temporary download link to the file. See [the docs](https://pipedream.com/docs/connect/components/files) to learn more about working with files in Pipedream.",
+      default: false,
+      optional: true,
+    },
+    dir: {
+      type: "dir",
+      accessMode: "write",
+      optional: true,
+    },
+  },
+  methods: {
+    async streamToBuffer(stream) {
+      const chunks = [];
+      for await (const chunk of stream) {
+        chunks.push(chunk);
+      }
+      return Buffer.concat(chunks);
+    },
+    async stashFile(item) {
+      const { Body } = await this.getObject({
+        Bucket: item.bucket.name,
+        Key: item.object.key.replace(/\+/g, " "),
+      });
+      const filepath = `${item.bucket.name}/${item.object.key}`;
+      const buffer = await this.streamToBuffer(Body);
+      const type = await fileTypeFromBuffer(buffer);
+      // Upload the attachment to the configured directory (File Stash) so it
+      // can be accessed later.
+      const file = await this.dir.open(filepath).fromReadableStream(
+        Readable.from(buffer),
+        type?.mime,
+        buffer.length,
+      );
+      // Return file details and temporary download link:
+      // { path, get_url, s3Key, type }
+      return await file.withoutPutUrl().withGetUrl();
+    },
+    async processEvent(event) {
+      const { Message: rawMessage } = event.body;
+      const {
+        Records: s3Events = [],
+        Event: eventType,
+      } = JSON.parse(rawMessage);
+
+      if (eventType === "s3:TestEvent") {
+        console.log("Received initial test event. Skipping...");
+        return;
+      }
+
+      for (const s3Event of s3Events) {
+        const meta = this.generateMeta(s3Event);
+        let { s3: item } = s3Event;
+        if (this.includeLink) {
+          item.file = await this.stashFile(item);
+        }
+        this.$emit(item, meta);
+      }
+    },
+  },
+};

components/aws/sources/s3-new-file/s3-new-file.mjs

@@ -1,15 +1,21 @@
 import base from "../common/s3.mjs";
+import includeLink from "../common/include-link.mjs";
 
 export default {
   ...base,
   type: "source",
   key: "aws-s3-new-file",
   name: "New S3 File",
   description: "Emit new event when a file is added to an S3 bucket",
-  version: "0.1.4",
+  version: "0.2.0",
   dedupe: "unique",
+  props: {
+    ...base.props,
+    ...includeLink.props,
+  },
   methods: {
     ...base.methods,
+    ...includeLink.methods,
     getEvents() {
       return [
         "s3:ObjectCreated:*",

components/aws/sources/s3-restored-file/s3-restored-file.mjs

@@ -1,24 +1,32 @@
 import base from "../common/s3.mjs";
+import includeLink from "../common/include-link.mjs";
 
 export default {
   ...base,
   type: "source",
   key: "aws-s3-restored-file",
   name: "New Restored S3 File",
-  description: "Emit new event when an file is restored into a S3 bucket",
-  version: "0.1.4",
+  description: "Emit new event when a file is restored into an S3 bucket",
+  version: "0.2.0",
   dedupe: "unique",
   props: {
+    info: {
+      type: "alert",
+      alertType: "info",
+      content: "Allows receipt of notifications for event initiation and completion when restoring objects from the **S3 Glacier Flexible Retrieval** storage class, **S3 Glacier Deep Archive** storage class, **S3 Intelligent-Tiering Archive Access tier**, and **S3 Intelligent-Tiering Deep Archive Access tier**.",
+    },
     ...base.props,
     detectRestoreInitiation: {
       type: "boolean",
       label: "Detect Restore Initiation",
       description: "When enabled, this event source will also emit events whenever a restore is initiated",
       default: false,
     },
+    ...includeLink.props,
   },
   methods: {
     ...base.methods,
+    ...includeLink.methods,
     getEvents() {
       return [
         this.detectRestoreInitiation

components/box/actions/create-sign-request/create-sign-request.mjs

@@ -6,7 +6,7 @@ export default {
   name: "Create Box Sign Request",
   description: "Creates a signature request. This involves preparing a document for signing and sending the signature request to signers. [See the documentation](https://developer.box.com/reference/post-sign-requests/).",
   key: "box-create-sign-request",
-  version: "0.0.1",
+  version: "0.0.2",
   type: "action",
   props: {
     app,

components/box/actions/download-file/download-file.mjs

@@ -7,7 +7,7 @@ export default {
   name: "Download File",
   description: "Downloads a file from Box to your workflow's `/tmp` directory. [See the documentation](https://developer.box.com/reference/get-files-id-content/)",
   key: "box-download-file",
-  version: "0.0.4",
+  version: "0.0.5",
   type: "action",
   props: {
     app,

components/box/actions/get-comments/get-comments.mjs

@@ -5,7 +5,7 @@ export default {
   name: "Get Comments",
   description: "Fetches comments for a file. [See the documentation](https://developer.box.com/reference/get-files-id-comments/).",
   key: "box-get-comments",
-  version: "0.0.1",
+  version: "0.0.2",
   type: "action",
   props: {
     app,

components/box/actions/search-content/search-content.mjs

@@ -6,7 +6,7 @@ export default {
   name: "Search Content",
   description: "Searches for files, folders, web links, and shared files across the users content or across the entire enterprise. [See the documentation](https://developer.box.com/reference/get-search/).",
   key: "box-search-content",
-  version: "0.0.4",
+  version: "0.0.5",
   type: "action",
   props: {
     app,

components/box/actions/upload-file-version/upload-file-version.mjs

@@ -5,7 +5,7 @@ export default {
   name: "Upload File Version",
   description: "Update a file's content. [See the documentation](https://developer.box.com/reference/post-files-id-content/).",
   key: "box-upload-file-version",
-  version: "0.1.1",
+  version: "0.1.2",
   type: "action",
   props: {
     app,

components/box/actions/upload-file/upload-file.mjs

@@ -5,7 +5,7 @@ export default {
   name: "Upload a File",
   description: "Uploads a small file to Box. [See the documentation](https://developer.box.com/reference/post-files-content/).",
   key: "box-upload-file",
-  version: "0.1.1",
+  version: "0.1.2",
   type: "action",
   props: {
     app,

components/box/box.app.mjs

@@ -129,7 +129,9 @@ export default {
       type: "string",
       label: "Target",
       description: "The target item that will be used by webhooks.",
-      async options({ prevContext }) {
+      async options({
+        prevContext, type,
+      }) {
         const {
           marker,
           queue,
@@ -154,7 +156,7 @@ export default {
                 ...resp.entries.filter((e) => e.type == "folder").map((e) => e.id),
               ],
             },
-            options: resp.entries.map((e) => ({
+            options: resp.entries.filter((e) => !type || e.type == type).map((e) => ({
               value: JSON.stringify({
                 id: e.id,
                 type: e.type,
@@ -185,7 +187,7 @@ export default {
                   ...innerResp.entries.filter((e) => e.type == "folder").map((e) => e.id),
                 ],
               },
-              options: innerResp.entries.map((e) => ({
+              options: innerResp.entries.filter((e) => !type || e.type == type).map((e) => ({
                 value: JSON.stringify({
                   id: e.id,
                   type: e.type,

components/box/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/box",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Pipedream Box Components",
   "main": "box.app.mjs",
   "keywords": [
@@ -11,6 +11,7 @@
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "dependencies": {
     "@pipedream/platform": "^3.1.0",
+    "file-type": "^21.0.0",
     "form-data": "^4.0.0",
     "path": "^0.12.7",
     "stream": "^0.0.2",

components/box/sources/new-event/new-event.mjs

@@ -5,7 +5,7 @@ export default {
   key: "box-new-event",
   name: "New Event",
   description: "Emit new event when an event with subscribed event source triggered on a target. [See the documentation](https://developer.box.com/reference/post-webhooks)",
-  version: "0.0.4",
+  version: "0.0.5",
   type: "source",
   dedupe: "unique",
   ...common,

components/box/sources/new-file/new-file.mjs

@@ -1,13 +1,39 @@
 import common from "../common/common.mjs";
+import { Readable } from "stream";
+import { fileTypeFromBuffer } from "file-type";
 
 export default {
   key: "box-new-file",
   name: "New File Event",
-  description: "Emit new event when a new file uploaded on a target. [See the documentation](https://developer.box.com/reference/post-webhooks)",
-  version: "0.0.4",
+  description: "Emit new event when a new file is uploaded to a target. [See the documentation](https://developer.box.com/reference/post-webhooks)",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   ...common,
+  props: {
+    ...common.props,
+    webhookTarget: {
+      propDefinition: [
+        common.props.app,
+        "webhookTarget",
+        () => ({
+          type: "folder",
+        }),
+      ],
+    },
+    includeLink: {
+      label: "Include Link",
+      type: "boolean",
+      description: "Upload file to your File Stash and emit temporary download link to the file. See [the docs](https://pipedream.com/docs/connect/components/files) to learn more about working with files in Pipedream.",
+      default: false,
+      optional: true,
+    },
+    dir: {
+      type: "dir",
+      accessMode: "write",
+      optional: true,
+    },
+  },
   methods: {
     ...common.methods,
     getTriggers() {
@@ -18,5 +44,29 @@ export default {
     getSummary(event) {
       return  `New file uploaded event with ID(${event.id})`;
     },
+    async stashFile(event) {
+      const response = await this.app.downloadFile({
+        fileId: event.source.id,
+        responseType: "arraybuffer",
+      });
+      const buffer = Buffer.from(response);
+      const filepath = `${event.source.id}/${event.source.name}`;
+      const type = await fileTypeFromBuffer(buffer);
+      const file = await this.dir.open(filepath).fromReadableStream(
+        Readable.from(buffer),
+        type?.mime,
+        buffer.length,
+      );
+      return await file.withoutPutUrl().withGetUrl();
+    },
+  },
+  async run(event) {
+    if (this.includeLink) {
+      event.body.file = await this.stashFile(event.body);
+    }
+    this.$emit(
+      event.body,
+      this.getMetadata(event.body),
+    );
   },
 };

components/box/sources/new-folder/new-folder.mjs

@@ -4,7 +4,7 @@ export default {
   key: "box-new-folder",
   name: "New Folder Event",
   description: "Emit new event when a new folder created on a target. [See the documentation](https://developer.box.com/reference/post-webhooks)",
-  version: "0.0.4",
+  version: "0.0.5",
   type: "source",
   dedupe: "unique",
   ...common,

components/buddee/buddee.app.mjs

@@ -8,4 +8,4 @@ export default {
       console.log(Object.keys(this.$auth));
     },
   },
-};
+};

components/egnyte/actions/create-folder/create-folder.mjs

@@ -4,7 +4,7 @@ export default {
   key: "egnyte-create-folder",
   name: "Create Folder",
   description: "Creates a new folder in your Egnyte workspace. [See the documentation](https://developers.egnyte.com/docs/File_System_Management_API_Documentation#Create-a-Folder)",
-  version: "0.0.1",
+  version: "0.0.2",
   type: "action",
   props: {
     egnyte,

components/egnyte/actions/upload-file/upload-file.mjs

@@ -6,7 +6,7 @@ export default {
   key: "egnyte-upload-file",
   name: "Upload File",
   description: "Uploads a file to a specified folder in Egnyte. [See the documentation](https://developers.egnyte.com/docs/File_System_Management_API_Documentation#Upload-a-File)",
-  version: "0.1.1",
+  version: "0.1.2",
   type: "action",
   props: {
     egnyte,

components/egnyte/egnyte.app.mjs

@@ -55,5 +55,14 @@ export default {
         ...opts,
       });
     },
+    downloadFile({
+      folderPath, filename, ...opts
+    }) {
+      return this._makeRequest({
+        path: `/fs-content/${folderPath}/${filename}`,
+        responseType: "arraybuffer",
+        ...opts,
+      });
+    },
   },
 };

components/egnyte/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/egnyte",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "Pipedream Egnyte Components",
   "main": "egnyte.app.mjs",
   "keywords": [