[FEATURE] Recursive directories upload for S3 - Fixed

Author: jcortes

components/aws/actions/s3-upload-file-tmp/s3-upload-file-tmp.mjs

@@ -7,12 +7,12 @@ import { ConfigurationError } from "@pipedream/platform";
 export default {
   ...common,
   key: "aws-s3-upload-file-tmp",
-  name: "S3 - Upload File - /tmp",
+  name: "S3 - Upload Files - /tmp",
   description: toSingleLineString(`
     Accepts a file path or folder path starting from /tmp, then uploads the contents to S3.
     [See the docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html)
   `),
-  version: "1.0.2",
+  version: "1.0.3",
   type: "action",
   props: {
     aws: common.props.aws,
@@ -25,15 +25,8 @@ export default {
     },
     path: {
       type: "string",
-      label: "File Path",
-      description: "A path starting from `/tmp`, i.e. `/tmp/some_text_file.txt`",
-      optional: true,
-    },
-    folderPath: {
-      type: "string",
-      label: "Folder Path",
-      description: "A path starting from `/tmp`, i.e. `/tmp/some_folder`. If provided, all the files inside this path will be uploaded. This will override the `Filename Key` and `File Path` props.",
-      optional: true,
+      label: "File Or Folder Path",
+      description: "Path starting from `/tmp`. If it's a directory, all files will be uploaded.",
     },
     customFilename: {
       type: common.props.key.type,
@@ -44,58 +37,83 @@ export default {
   },
   methods: {
     ...common.methods,
-    async uploadFolderFiles($) {
+    getFilesRecursive(dir) {
+      let results = [];
+      const items = fs.readdirSync(dir);
+      for (const item of items) {
+        const itemPath = join(dir, item);
+        const stat = fs.statSync(itemPath);
+        if (stat.isDirectory()) {
+          results = results.concat(this.getFilesRecursive(itemPath));
+        } else {
+          results.push(itemPath);
+        }
+      }
+      return results;
+    },
+    async uploadFolderFiles($, folderPath) {
       const {
         uploadFile,
         bucket,
-        folderPath,
         prefix,
       } = this;
-
-      const files = fs.readdirSync(folderPath);
-      const promises = [];
-      for (const filename of files) {
-        const fileContent = fs.readFileSync(join(folderPath, filename), {
+      const files = this.getFilesRecursive(folderPath);
+      const response = await Promise.all(files.map(async (filePath) => {
+        const fileContent = fs.readFileSync(filePath, {
           encoding: "base64",
         });
-        promises.push(uploadFile({
+        const relativePath = filePath.substring(folderPath.length + 1);
+        const s3Key = join(prefix, relativePath);
+
+        await uploadFile({
           Bucket: bucket,
-          Key: join(prefix, filename),
+          Key: s3Key,
           Body: Buffer.from(fileContent, "base64"),
-        }));
-      }
-      const response = await Promise.all(promises);
+        });
+        return {
+          filePath,
+          s3Key,
+          status: "uploaded",
+        };
+      }));
       $.export("$summary", `Uploaded all files from ${folderPath} to S3`);
       return response;
     },
-    async uploadSingleFile($) {
+    async uploadSingleFile($, filePath) {
       const {
         uploadFile,
         bucket,
-        path,
         prefix,
         customFilename,
       } = this;
 
-      if (!path) {
-        throw new ConfigurationError("File Path is required");
-      }
-      const file = fs.readFileSync(path, {
+      const file = fs.readFileSync(filePath, {
         encoding: "base64",
       });
-      const filename = customFilename || path.split("/").pop();
+      const filename = customFilename || filePath.split("/").pop();
+
       const response = await uploadFile({
         Bucket: bucket,
         Key: join(prefix, filename),
         Body: Buffer.from(file, "base64"),
       });
+
       $.export("$summary", `Uploaded file ${filename} to S3`);
       return response;
     },
   },
   async run({ $ }) {
-    return this.folderPath
-      ? await this.uploadFolderFiles($)
-      : await this.uploadSingleFile($);
+    const {
+      uploadSingleFile,
+      uploadFolderFiles,
+      path,
+    } = this;
+    if (!fs.existsSync(path)) {
+      throw new ConfigurationError(`The file or directory path \`${path}\` does not exist. Please verify the path and include the leading /tmp if needed.`);
+    }
+    const stat = fs.statSync(path);
+    return stat.isDirectory()
+      ? await uploadFolderFiles($, path)
+      : await uploadSingleFile($, path);
   },
 };

components/aws/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/aws",
-  "version": "0.7.4",
+  "version": "0.7.5",
   "description": "Pipedream Aws Components",
   "main": "aws.app.mjs",
   "keywords": [