diff --git a/components/aws/actions/s3-upload-file-tmp/s3-upload-file-tmp.mjs b/components/aws/actions/s3-upload-file-tmp/s3-upload-file-tmp.mjs index a5a1a314b305b..86468bf12f025 100644 --- a/components/aws/actions/s3-upload-file-tmp/s3-upload-file-tmp.mjs +++ b/components/aws/actions/s3-upload-file-tmp/s3-upload-file-tmp.mjs @@ -7,12 +7,12 @@ import { ConfigurationError } from "@pipedream/platform"; export default { ...common, key: "aws-s3-upload-file-tmp", - name: "S3 - Upload File - /tmp", + name: "S3 - Upload Files - /tmp", description: toSingleLineString(` Accepts a file path or folder path starting from /tmp, then uploads the contents to S3. [See the docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html) `), - version: "1.0.2", + version: "1.0.3", type: "action", props: { aws: common.props.aws, @@ -25,15 +25,8 @@ export default { }, path: { type: "string", - label: "File Path", - description: "A path starting from `/tmp`, i.e. `/tmp/some_text_file.txt`", - optional: true, - }, - folderPath: { - type: "string", - label: "Folder Path", - description: "A path starting from `/tmp`, i.e. `/tmp/some_folder`. If provided, all the files inside this path will be uploaded. This will override the `Filename Key` and `File Path` props.", - optional: true, + label: "File Or Folder Path", + description: "Path starting from `/tmp`. If it's a directory, all files will be uploaded.", }, customFilename: { type: common.props.key.type, @@ -44,58 +37,83 @@ export default { }, methods: { ...common.methods, - async uploadFolderFiles($) { + getFilesRecursive(dir) { + let results = []; + const items = fs.readdirSync(dir); + for (const item of items) { + const itemPath = join(dir, item); + const stat = fs.statSync(itemPath); + if (stat.isDirectory()) { + results = results.concat(this.getFilesRecursive(itemPath)); + } else { + results.push(itemPath); + } + } + return results; + }, + async uploadFolderFiles($, folderPath) { const { uploadFile, bucket, - folderPath, prefix, } = this; - - const files = fs.readdirSync(folderPath); - const promises = []; - for (const filename of files) { - const fileContent = fs.readFileSync(join(folderPath, filename), { + const files = this.getFilesRecursive(folderPath); + const response = await Promise.all(files.map(async (filePath) => { + const fileContent = fs.readFileSync(filePath, { encoding: "base64", }); - promises.push(uploadFile({ + const relativePath = filePath.substring(folderPath.length + 1); + const s3Key = join(prefix, relativePath); + + await uploadFile({ Bucket: bucket, - Key: join(prefix, filename), + Key: s3Key, Body: Buffer.from(fileContent, "base64"), - })); - } - const response = await Promise.all(promises); + }); + return { + filePath, + s3Key, + status: "uploaded", + }; + })); $.export("$summary", `Uploaded all files from ${folderPath} to S3`); return response; }, - async uploadSingleFile($) { + async uploadSingleFile($, filePath) { const { uploadFile, bucket, - path, prefix, customFilename, } = this; - if (!path) { - throw new ConfigurationError("File Path is required"); - } - const file = fs.readFileSync(path, { + const file = fs.readFileSync(filePath, { encoding: "base64", }); - const filename = customFilename || path.split("/").pop(); + const filename = customFilename || filePath.split("/").pop(); + const response = await uploadFile({ Bucket: bucket, Key: join(prefix, filename), Body: Buffer.from(file, "base64"), }); + $.export("$summary", `Uploaded file ${filename} to S3`); return response; }, }, async run({ $ }) { - return this.folderPath - ? await this.uploadFolderFiles($) - : await this.uploadSingleFile($); + const { + uploadSingleFile, + uploadFolderFiles, + path, + } = this; + if (!fs.existsSync(path)) { + throw new ConfigurationError(`The file or directory path \`${path}\` does not exist. Please verify the path and include the leading /tmp if needed.`); + } + const stat = fs.statSync(path); + return stat.isDirectory() + ? await uploadFolderFiles($, path) + : await uploadSingleFile($, path); }, }; diff --git a/components/aws/package.json b/components/aws/package.json index c5a5cee3e3468..2dee0cc912151 100644 --- a/components/aws/package.json +++ b/components/aws/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/aws", - "version": "0.7.4", + "version": "0.7.5", "description": "Pipedream Aws Components", "main": "aws.app.mjs", "keywords": [