Applying File URL or Path improvements

components/_2markdown/_2markdown.app.mjs

@@ -6,8 +6,8 @@ export default {
   propDefinitions: {
     filePath: {
       type: "string",
-      label: "File Path",
-      description: "The path to an HTML file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)",
+      label: "File Path or URL",
+      description: "An HTML file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.html`)",
     },
   },
   methods: {

components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs

@@ -1,12 +1,12 @@
 import _2markdown from "../../_2markdown.app.mjs";
-import fs from "fs";
+import { getFileStreamAndMetadata } from "@pipedream/platform";
 import FormData from "form-data";
 
 export default {
   key: "_2markdown-html-file-to-markdown",
   name: "HTML File to Markdown",
   description: "Convert an HTML file to Markdown format. [See the documentation](https://2markdown.com/docs#file2md)",
-  version: "0.0.1",
+  version: "0.1.0",
   type: "action",
   props: {
     _2markdown,
@@ -20,9 +20,14 @@ export default {
   async run({ $ }) {
     const form = new FormData();
 
-    form.append("document", fs.createReadStream(this.filePath.includes("tmp/")
-      ? this.filePath
-      : `/tmp/${this.filePath}`));
+    const {
+      stream, metadata,
+    } = await getFileStreamAndMetadata(this.filePath);
+    form.append("document", stream, {
+      contentType: metadata.contentType,
+      knownLength: metadata.size,
+      filename: metadata.name,
+    });
 
     const response = await this._2markdown.htmlFileToMarkdown({
       $,

components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs

@@ -1,12 +1,12 @@
 import _2markdown from "../../_2markdown.app.mjs";
-import fs from "fs";
+import { getFileStreamAndMetadata } from "@pipedream/platform";
 import FormData from "form-data";
 
 export default {
   key: "_2markdown-pdf-to-markdown",
   name: "PDF to Markdown",
   description: "Convert a PDF document to Markdown format. [See the documentation](https://2markdown.com/docs#pdf2md)",
-  version: "0.0.1",
+  version: "0.1.0",
   type: "action",
   props: {
     _2markdown,
@@ -15,7 +15,7 @@ export default {
         _2markdown,
         "filePath",
       ],
-      description: "The path to a PDF file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)",
+      description: "A PDF file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)",
     },
     waitForCompletion: {
       type: "boolean",
@@ -27,9 +27,14 @@ export default {
   async run({ $ }) {
     const form = new FormData();
 
-    form.append("document", fs.createReadStream(this.filePath.includes("tmp/")
-      ? this.filePath
-      : `/tmp/${this.filePath}`));
+    const {
+      stream, metadata,
+    } = await getFileStreamAndMetadata(this.filePath);
+    form.append("document", stream, {
+      contentType: metadata.contentType,
+      knownLength: metadata.size,
+      filename: metadata.name,
+    });
 
     let response = await this._2markdown.pdfToMarkdown({
       $,

components/_2markdown/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/_2markdown",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Pipedream 2markdown Components",
   "main": "_2markdown.app.mjs",
   "keywords": [
@@ -13,7 +13,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@pipedream/platform": "^3.0.3",
+    "@pipedream/platform": "^3.1.0",
     "form-data": "^4.0.1"
   }
 }

components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs

@@ -1,13 +1,13 @@
 import app from "../../akeneo.app.mjs";
-import utils from "../../common/utils.mjs";
-import { ConfigurationError } from "@pipedream/platform";
+import {
+  ConfigurationError, getFileStreamAndMetadata,
+} from "@pipedream/platform";
 import FormData from "form-data";
-import fs from "fs";
 
 export default {
   type: "action",
   key: "akeneo-create-a-new-product-media-file",
-  version: "0.0.1",
+  version: "0.1.0",
   name: "Create A New Product Media File",
   description: "Allows you to create a new media file and associate it to an attribute value of a given product or product model. [See the docs](https://api.akeneo.com/api-reference.html#post_media_files)",
   props: {
@@ -32,18 +32,15 @@ export default {
     },
     filename: {
       type: "string",
-      label: "File",
-      description: "The file to be uploaded, please provide a file from `/tmp`. To upload a file to `/tmp` folder, please follow the doc [here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)",
+      label: "File Path or URL",
+      description: "The file to be uploaded. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)",
     },
   },
   async run ({ $ }) {
     if (!this.productId && !this.productModelCode) {
       throw new ConfigurationError("Either `Product Identifier` or `Product Model Code` should be set!");
     }
-    const path = utils.checkTmp(this.filename);
-    if (!fs.existsSync(path)) {
-      throw new ConfigurationError("File does not exist!");
-    }
+
     const payload = {
       attribute: this.mediaFileAttributeCode,
       scope: null,
@@ -57,9 +54,15 @@ export default {
       payload.code = this.productModelCode;
       data.append("product_model", JSON.stringify(payload));
     }
-    const file = fs.readFileSync(path);
-    const fileParts = path.split("/");
-    data.append("file", file, fileParts[fileParts.length - 1]);
+
+    const {
+      stream, metadata,
+    } = await getFileStreamAndMetadata(this.filename);
+    data.append("file", stream, {
+      contentType: metadata.contentType,
+      knownLength: metadata.size,
+      filename: metadata.name,
+    });
     const contentLength = data.getLengthSync();
     await this.app.createProductMediaFile({
       $,

components/akeneo/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/akeneo",
-  "version": "0.0.2",
+  "version": "0.1.0",
   "description": "Pipedream Akeneo Components",
   "main": "akeneo.app.mjs",
   "keywords": [
@@ -10,7 +10,7 @@
   "homepage": "https://pipedream.com/apps/akeneo",
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "dependencies": {
-    "@pipedream/platform": "^1.3.0",
+    "@pipedream/platform": "^3.1.0",
     "form-data": "^4.0.0"
   },
   "publishConfig": {

components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs

@@ -6,13 +6,13 @@ export default {
   name: "Add Document Via File Upload",
   description: "Add a document via file upload. [See the documentation](https://docs.askyourpdf.com/askyourpdf-docs/#2.-adding-document-via-file-upload)",
   type: "action",
-  version: "0.0.2",
+  version: "0.1.0",
   props: {
     app,
     file: {
       type: "string",
-      label: "File Path",
-      description: "File path of a file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)",
+      label: "File Path or URL",
+      description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)",
     },
   },
   methods: {

components/askyourpdf/askyourpdf.app.mjs

@@ -27,12 +27,12 @@ export default {
         ...headers,
       };
     },
-    getConfig({
+    async getConfig({
       headers, data: preData, ...args
     } = {}) {
       const contentType = constants.CONTENT_TYPE_KEY_HEADER;
       const hasMultipartHeader = utils.hasMultipartHeader(headers);
-      const data = hasMultipartHeader && utils.getFormData(preData) || preData;
+      const data = hasMultipartHeader && await utils.getFormData(preData) || preData;
       const currentHeaders = this.getHeaders(headers);
 
       return {
@@ -49,7 +49,7 @@ export default {
     async makeRequest({
       step = this, path, headers, data, summary, ...args
     } = {}) {
-      const config = this.getConfig({
+      const config = await this.getConfig({
         url: this.getUrl(path),
         headers,
         data,

components/askyourpdf/common/utils.mjs

@@ -1,26 +1,30 @@
-import { createReadStream } from "fs";
+import { getFileStreamAndMetadata } from "@pipedream/platform";
 import FormData from "form-data";
 import constants from "./constants.mjs";
 
-function buildFormData(formData, data, parentKey) {
+async function buildFormData(formData, data, parentKey) {
   if (data && typeof(data) === "object") {
-    Object.keys(data)
-      .forEach(async (key) => {
-        buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key);
-      });
-
+    for (const key of Object.keys(data)) {
+      await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key);
+    }
   } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) {
-    formData.append(parentKey, createReadStream(data));
-
+    const {
+      stream, metadata,
+    } = await getFileStreamAndMetadata(data);
+    formData.append(parentKey, stream, {
+      contentType: metadata.contentType,
+      knownLength: metadata.size,
+      filename: metadata.name,
+    });
   } else if (data) {
     formData.append(parentKey, (data).toString());
   }
 }
 
-function getFormData(data) {
+async function getFormData(data) {
   try {
     const formData = new FormData();
-    buildFormData(formData, data);
+    await buildFormData(formData, data);
     return formData;
   } catch (error) {
     console.log("FormData Error", error);

components/askyourpdf/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/askyourpdf",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Pipedream AskYourPDF Components",
   "main": "askyourpdf.app.mjs",
   "keywords": [
@@ -10,9 +10,8 @@
   "homepage": "https://pipedream.com/apps/askyourpdf",
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "dependencies": {
-    "@pipedream/platform": "^1.6.2",
-    "form-data": "^4.0.0",
-    "fs": "^0.0.1-security"
+    "@pipedream/platform": "^3.1.0",
+    "form-data": "^4.0.0"
   },
   "publishConfig": {
     "access": "public"

components/box/actions/upload-file-version/upload-file-version.mjs

@@ -5,7 +5,7 @@ export default {
   name: "Upload File Version",
   description: "Update a file's content. [See the documentation](https://developer.box.com/reference/post-files-id-content/).",
   key: "box-upload-file-version",
-  version: "0.0.2",
+  version: "0.1.0",
   type: "action",
   props: {
     app,

components/box/actions/upload-file/upload-file.mjs

@@ -5,7 +5,7 @@ export default {
   name: "Upload a File",
   description: "Uploads a small file to Box. [See the documentation](https://developer.box.com/reference/post-files-content/).",
   key: "box-upload-file",
-  version: "0.0.4",
+  version: "0.1.0",
   type: "action",
   props: {
     app,

components/box/box.app.mjs

@@ -13,8 +13,8 @@ export default {
     },
     file: {
       type: "string",
-      label: "File",
-      description: "The file to upload to Box, please provide a file from `/tmp`. To upload a file to `/tmp` folder, please follow the doc [here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)",
+      label: "File Path or URL",
+      description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)",
     },
     createdAt: {
       type: "string",

components/box/common/common-file-upload.mjs

@@ -1,4 +1,3 @@
-import { ConfigurationError } from "@pipedream/platform";
 import FormData from "form-data";
 import utils from "./utils.mjs";
 
@@ -9,12 +8,9 @@ export function getFileUploadBody({
   fileName,
   parentId,
 }) {
-  const fileValidation = utils.isValidFile(file);
-  if (!fileValidation) {
-    throw new ConfigurationError("`file` must be a valid file path!");
-  }
-  const fileMeta = utils.getFileMeta(fileValidation);
-  const fileContent = utils.getFileStream(fileValidation);
+  const {
+    fileMeta, fileContent,
+  } = utils.getFileData(file);
   const attributes = fileMeta.attributes;
   if (createdAt && utils.checkRFC3339(createdAt)) {
     attributes.content_created_at = createdAt;

components/box/common/utils.mjs

@@ -1,34 +1,27 @@
-import fs from "fs";
 import path from "path";
 import constants from "./constants.mjs";
+import { getFileStreamAndMetadata } from "@pipedream/platform";
 
 export default {
-  isValidFile(filePath) {
-    const filePathWithTmp = `/tmp/${filePath}`;
-    if (fs.existsSync(filePathWithTmp)) {
-      return filePathWithTmp;
-    } else if (fs.existsSync(filePath)) {
-      return filePath;
-    }
-    return false;
-  },
-  getFileStream(filePath) {
-    return fs.createReadStream(filePath);
-  },
-  getFileMeta(filePath) {
-    const stats = fs.statSync(filePath);
+  async getFileData(filePath) {
+    const {
+      stream, metadata,
+    } = await getFileStreamAndMetadata(filePath);
     return {
-      attributes: {
-        content_created_at: new Date(stats.ctimeMs).toISOString()
-          .split(".")[0] + "Z",
-        content_modified_at: new Date(stats.mtimeMs).toISOString()
-          .split(".")[0] + "Z",
-        name: path.basename(filePath),
-        parent: {
-          id: 0,
+      fileContent: stream,
+      fileMeta: {
+        attributes: {
+          content_created_at: new Date(metadata.lastModified).toISOString()
+            .split(".")[0] + "Z",
+          content_modified_at: new Date(metadata.lastModified).toISOString()
+            .split(".")[0] + "Z",
+          name: path.basename(filePath),
+          parent: {
+            id: 0,
+          },
         },
+        size: metadata.size,
       },
-      size: stats.size,
     };
   },
   checkRFC3339(dateTimeStr) {