diff --git a/components/_2markdown/_2markdown.app.mjs b/components/_2markdown/_2markdown.app.mjs index 5996f3464f0b9..4fa47f6e6f432 100644 --- a/components/_2markdown/_2markdown.app.mjs +++ b/components/_2markdown/_2markdown.app.mjs @@ -6,8 +6,8 @@ export default { propDefinitions: { filePath: { type: "string", - label: "File Path", - description: "The path to an HTML file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "An HTML file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.html`)", }, }, methods: { diff --git a/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs b/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs index c5e8510e1e639..c238c14ff71c0 100644 --- a/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs +++ b/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs @@ -1,12 +1,12 @@ import _2markdown from "../../_2markdown.app.mjs"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "_2markdown-html-file-to-markdown", name: "HTML File to Markdown", description: "Convert an HTML file to Markdown format. [See the documentation](https://2markdown.com/docs#file2md)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { _2markdown, @@ -20,9 +20,14 @@ export default { async run({ $ }) { const form = new FormData(); - form.append("document", fs.createReadStream(this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`)); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + form.append("document", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); const response = await this._2markdown.htmlFileToMarkdown({ $, diff --git a/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs b/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs index 597a80fdb6812..2a27deafd32cf 100644 --- a/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs +++ b/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs @@ -1,12 +1,12 @@ import _2markdown from "../../_2markdown.app.mjs"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "_2markdown-pdf-to-markdown", name: "PDF to Markdown", description: "Convert a PDF document to Markdown format. [See the documentation](https://2markdown.com/docs#pdf2md)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { _2markdown, @@ -15,7 +15,7 @@ export default { _2markdown, "filePath", ], - description: "The path to a PDF file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + description: "A PDF file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, waitForCompletion: { type: "boolean", @@ -27,9 +27,14 @@ export default { async run({ $ }) { const form = new FormData(); - form.append("document", fs.createReadStream(this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`)); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + form.append("document", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); let response = await this._2markdown.pdfToMarkdown({ $, diff --git a/components/_2markdown/package.json b/components/_2markdown/package.json index 9df443a7fe99d..7fcf281a32410 100644 --- a/components/_2markdown/package.json +++ b/components/_2markdown/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/_2markdown", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream 2markdown Components", "main": "_2markdown.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.1" } } diff --git a/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs b/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs index 7a2715c5ce0c2..f43f439b34223 100644 --- a/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs +++ b/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs @@ -1,13 +1,13 @@ import app from "../../akeneo.app.mjs"; -import utils from "../../common/utils.mjs"; -import { ConfigurationError } from "@pipedream/platform"; +import { + ConfigurationError, getFileStreamAndMetadata, +} from "@pipedream/platform"; import FormData from "form-data"; -import fs from "fs"; export default { type: "action", key: "akeneo-create-a-new-product-media-file", - version: "0.0.1", + version: "0.1.0", name: "Create A New Product Media File", description: "Allows you to create a new media file and associate it to an attribute value of a given product or product model. [See the docs](https://api.akeneo.com/api-reference.html#post_media_files)", props: { @@ -32,18 +32,15 @@ export default { }, filename: { type: "string", - label: "File", - description: "The file to be uploaded, please provide a file from `/tmp`. To upload a file to `/tmp` folder, please follow the doc [here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to be uploaded. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, async run ({ $ }) { if (!this.productId && !this.productModelCode) { throw new ConfigurationError("Either `Product Identifier` or `Product Model Code` should be set!"); } - const path = utils.checkTmp(this.filename); - if (!fs.existsSync(path)) { - throw new ConfigurationError("File does not exist!"); - } + const payload = { attribute: this.mediaFileAttributeCode, scope: null, @@ -57,9 +54,15 @@ export default { payload.code = this.productModelCode; data.append("product_model", JSON.stringify(payload)); } - const file = fs.readFileSync(path); - const fileParts = path.split("/"); - data.append("file", file, fileParts[fileParts.length - 1]); + + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filename); + data.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); const contentLength = data.getLengthSync(); await this.app.createProductMediaFile({ $, diff --git a/components/akeneo/package.json b/components/akeneo/package.json index b53068ab24a0b..7c2aa0121119f 100644 --- a/components/akeneo/package.json +++ b/components/akeneo/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/akeneo", - "version": "0.0.2", + "version": "0.1.0", "description": "Pipedream Akeneo Components", "main": "akeneo.app.mjs", "keywords": [ @@ -10,7 +10,7 @@ "homepage": "https://pipedream.com/apps/akeneo", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^1.3.0", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" }, "publishConfig": { diff --git a/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs b/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs index 4b4b50d7a0631..e93de416ceb2e 100644 --- a/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs +++ b/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs @@ -6,13 +6,13 @@ export default { name: "Add Document Via File Upload", description: "Add a document via file upload. [See the documentation](https://docs.askyourpdf.com/askyourpdf-docs/#2.-adding-document-via-file-upload)", type: "action", - version: "0.0.2", + version: "0.1.0", props: { app, file: { type: "string", - label: "File Path", - description: "File path of a file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, methods: { diff --git a/components/askyourpdf/askyourpdf.app.mjs b/components/askyourpdf/askyourpdf.app.mjs index 94a7e34bfb456..72244e8a3256f 100644 --- a/components/askyourpdf/askyourpdf.app.mjs +++ b/components/askyourpdf/askyourpdf.app.mjs @@ -27,12 +27,12 @@ export default { ...headers, }; }, - getConfig({ + async getConfig({ headers, data: preData, ...args } = {}) { const contentType = constants.CONTENT_TYPE_KEY_HEADER; const hasMultipartHeader = utils.hasMultipartHeader(headers); - const data = hasMultipartHeader && utils.getFormData(preData) || preData; + const data = hasMultipartHeader && await utils.getFormData(preData) || preData; const currentHeaders = this.getHeaders(headers); return { @@ -49,7 +49,7 @@ export default { async makeRequest({ step = this, path, headers, data, summary, ...args } = {}) { - const config = this.getConfig({ + const config = await this.getConfig({ url: this.getUrl(path), headers, data, diff --git a/components/askyourpdf/common/utils.mjs b/components/askyourpdf/common/utils.mjs index fb5909561429a..4b22a4e5dbae4 100644 --- a/components/askyourpdf/common/utils.mjs +++ b/components/askyourpdf/common/utils.mjs @@ -1,26 +1,30 @@ -import { createReadStream } from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; import constants from "./constants.mjs"; -function buildFormData(formData, data, parentKey) { +async function buildFormData(formData, data, parentKey) { if (data && typeof(data) === "object") { - Object.keys(data) - .forEach(async (key) => { - buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); - }); - + for (const key of Object.keys(data)) { + await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); + } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, createReadStream(data)); - + const { + stream, metadata, + } = await getFileStreamAndMetadata(data); + formData.append(parentKey, stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } else if (data) { formData.append(parentKey, (data).toString()); } } -function getFormData(data) { +async function getFormData(data) { try { const formData = new FormData(); - buildFormData(formData, data); + await buildFormData(formData, data); return formData; } catch (error) { console.log("FormData Error", error); diff --git a/components/askyourpdf/package.json b/components/askyourpdf/package.json index afdaf84cde945..7b8dbdcd072e9 100644 --- a/components/askyourpdf/package.json +++ b/components/askyourpdf/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/askyourpdf", - "version": "0.1.1", + "version": "0.2.0", "description": "Pipedream AskYourPDF Components", "main": "askyourpdf.app.mjs", "keywords": [ @@ -10,9 +10,8 @@ "homepage": "https://pipedream.com/apps/askyourpdf", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^1.6.2", - "form-data": "^4.0.0", - "fs": "^0.0.1-security" + "@pipedream/platform": "^3.1.0", + "form-data": "^4.0.0" }, "publishConfig": { "access": "public" diff --git a/components/box/actions/upload-file-version/upload-file-version.mjs b/components/box/actions/upload-file-version/upload-file-version.mjs index 48a9ed40212a6..9884061a1c8af 100644 --- a/components/box/actions/upload-file-version/upload-file-version.mjs +++ b/components/box/actions/upload-file-version/upload-file-version.mjs @@ -5,7 +5,7 @@ export default { name: "Upload File Version", description: "Update a file's content. [See the documentation](https://developer.box.com/reference/post-files-id-content/).", key: "box-upload-file-version", - version: "0.0.2", + version: "0.1.0", type: "action", props: { app, @@ -55,7 +55,7 @@ export default { const { file, fileId, createdAt, modifiedAt, fileName, parentId, } = this; - const data = this.getFileUploadBody({ + const data = await this.getFileUploadBody({ file, createdAt, modifiedAt, diff --git a/components/box/actions/upload-file/upload-file.mjs b/components/box/actions/upload-file/upload-file.mjs index 6e9f42e3ec6b8..476d98cf924bb 100644 --- a/components/box/actions/upload-file/upload-file.mjs +++ b/components/box/actions/upload-file/upload-file.mjs @@ -5,7 +5,7 @@ export default { name: "Upload a File", description: "Uploads a small file to Box. [See the documentation](https://developer.box.com/reference/post-files-content/).", key: "box-upload-file", - version: "0.0.4", + version: "0.1.0", type: "action", props: { app, @@ -47,7 +47,7 @@ export default { const { file, createdAt, modifiedAt, fileName, parentId, } = this; - const data = this.getFileUploadBody({ + const data = await this.getFileUploadBody({ file, createdAt, modifiedAt, diff --git a/components/box/box.app.mjs b/components/box/box.app.mjs index f7a2cbd47cb44..84302cb68efa6 100644 --- a/components/box/box.app.mjs +++ b/components/box/box.app.mjs @@ -13,8 +13,8 @@ export default { }, file: { type: "string", - label: "File", - description: "The file to upload to Box, please provide a file from `/tmp`. To upload a file to `/tmp` folder, please follow the doc [here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, createdAt: { type: "string", diff --git a/components/box/common/common-file-upload.mjs b/components/box/common/common-file-upload.mjs index 8bcc6f70af13e..a5e4434af54bc 100644 --- a/components/box/common/common-file-upload.mjs +++ b/components/box/common/common-file-upload.mjs @@ -1,20 +1,16 @@ -import { ConfigurationError } from "@pipedream/platform"; import FormData from "form-data"; import utils from "./utils.mjs"; -export function getFileUploadBody({ +export async function getFileUploadBody({ file, createdAt, modifiedAt, fileName, parentId, }) { - const fileValidation = utils.isValidFile(file); - if (!fileValidation) { - throw new ConfigurationError("`file` must be a valid file path!"); - } - const fileMeta = utils.getFileMeta(fileValidation); - const fileContent = utils.getFileStream(fileValidation); + const { + fileMeta, fileContent, + } = await utils.getFileData(file); const attributes = fileMeta.attributes; if (createdAt && utils.checkRFC3339(createdAt)) { attributes.content_created_at = createdAt; diff --git a/components/box/common/utils.mjs b/components/box/common/utils.mjs index 3bc4affe137d8..2897f8d946c48 100644 --- a/components/box/common/utils.mjs +++ b/components/box/common/utils.mjs @@ -1,34 +1,27 @@ -import fs from "fs"; import path from "path"; import constants from "./constants.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { - isValidFile(filePath) { - const filePathWithTmp = `/tmp/${filePath}`; - if (fs.existsSync(filePathWithTmp)) { - return filePathWithTmp; - } else if (fs.existsSync(filePath)) { - return filePath; - } - return false; - }, - getFileStream(filePath) { - return fs.createReadStream(filePath); - }, - getFileMeta(filePath) { - const stats = fs.statSync(filePath); + async getFileData(filePath) { + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); return { - attributes: { - content_created_at: new Date(stats.ctimeMs).toISOString() - .split(".")[0] + "Z", - content_modified_at: new Date(stats.mtimeMs).toISOString() - .split(".")[0] + "Z", - name: path.basename(filePath), - parent: { - id: 0, + fileContent: stream, + fileMeta: { + attributes: { + content_created_at: new Date(metadata.lastModified).toISOString() + .split(".")[0] + "Z", + content_modified_at: new Date(metadata.lastModified).toISOString() + .split(".")[0] + "Z", + name: path.basename(filePath), + parent: { + id: 0, + }, }, + size: metadata.size, }, - size: stats.size, }; }, checkRFC3339(dateTimeStr) { diff --git a/components/box/package.json b/components/box/package.json index 40752ed28373d..67da5b3df1053 100644 --- a/components/box/package.json +++ b/components/box/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/box", - "version": "0.3.0", + "version": "0.4.0", "description": "Pipedream Box Components", "main": "box.app.mjs", "keywords": [ @@ -10,7 +10,7 @@ "homepage": "https://pipedream.com/apps/box", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^1.5.1", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0", "path": "^0.12.7", "stream": "^0.0.2", diff --git a/components/fileforge/actions/generate-pdf/generate-pdf.mjs b/components/fileforge/actions/generate-pdf/generate-pdf.mjs index 9294fc2d2798a..83593ed0c9c5f 100644 --- a/components/fileforge/actions/generate-pdf/generate-pdf.mjs +++ b/components/fileforge/actions/generate-pdf/generate-pdf.mjs @@ -1,16 +1,13 @@ import FormData from "form-data"; -import fs from "fs"; -import { - checkTmp, - parseObject, -} from "../../common/utils.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; +import { parseObject } from "../../common/utils.mjs"; import fileforge from "../../fileforge.app.mjs"; export default { key: "fileforge-generate-pdf", name: "Generate PDF", description: "Generate a PDF from provided HTML. [See the documentation](https://docs.fileforge.com/api-reference/api-reference/pdf/generate)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { fileforge, @@ -27,8 +24,8 @@ export default { }, files: { type: "string[]", - label: "HTML Files", - description: "The HTML files to convert to PDF. Each file should be a valid path to an HTML file saved to the `/tmp` directory (e.g. `/tmp/image.png`). [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory)..", + label: "File Paths or URLs", + description: "The HTML files to convert to PDF. For each entry, provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, test: { type: "boolean", @@ -66,7 +63,14 @@ export default { const parsedFiles = parseObject(files); for (const file of parsedFiles) { - formData.append("files", fs.createReadStream(checkTmp(file))); + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); + formData.append("files", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } formData.append("options", JSON.stringify({ diff --git a/components/fileforge/common/utils.mjs b/components/fileforge/common/utils.mjs index d1e7ed1a22d98..dcc9cc61f6f41 100644 --- a/components/fileforge/common/utils.mjs +++ b/components/fileforge/common/utils.mjs @@ -22,10 +22,3 @@ export const parseObject = (obj) => { } return obj; }; - -export const checkTmp = (filename) => { - if (!filename.startsWith("/tmp")) { - return `/tmp/${filename}`; - } - return filename; -}; diff --git a/components/fileforge/package.json b/components/fileforge/package.json index da73c8d49d8c2..e70e4d3565bce 100644 --- a/components/fileforge/package.json +++ b/components/fileforge/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/fileforge", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Fileforge Components", "main": "fileforge.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.1", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/components/leaddyno/leaddyno.app.mjs b/components/leaddyno/leaddyno.app.mjs index 65c90c8f99158..bfdb3fbcf085a 100644 --- a/components/leaddyno/leaddyno.app.mjs +++ b/components/leaddyno/leaddyno.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/components/llmwhisperer/actions/extract-text/extract-text.mjs b/components/llmwhisperer/actions/extract-text/extract-text.mjs index 50795dc7e8ce8..1b16c0efb32b6 100644 --- a/components/llmwhisperer/actions/extract-text/extract-text.mjs +++ b/components/llmwhisperer/actions/extract-text/extract-text.mjs @@ -1,11 +1,11 @@ -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; import app from "../../llmwhisperer.app.mjs"; export default { key: "llmwhisperer-extract-text", name: "Extract Text", description: "Convert your PDF/scanned documents to text format which can be used by LLMs. [See the documentation](https://docs.unstract.com/llm_whisperer/apis/llm_whisperer_text_extraction_api)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { app, @@ -91,40 +91,18 @@ export default { description: "Factor by which a horizontal stretch has to applied. It defaults to `1.0`. A stretch factor of `1.1` would mean at 10% stretch factor applied. Normally this factor need not be adjusted. You might want to use this parameter when multi column layouts back into each other. For example in a two column layout, the two columns get merged into one.", optional: true, }, - urlInPost: { - type: "boolean", - label: "URL In Post", - description: "If set to `true`, the headers will be set to `text/plain`. If set to `false`, the headers will be set to `application/octet-stream`.", - reloadProps: true, - default: true, + data: { + type: "string", + label: "File Path or URL", + description: "The document to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, - additionalProps() { - const { urlInPost } = this; - return { - data: { - type: "string", - label: urlInPost - ? "Document URL" - : "Document Path", - description: urlInPost - ? "The URL of the document to process." - : "Document path of the file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)", - }, - }; - }, methods: { - getHeaders(urlInPost) { - return { - "Content-Type": urlInPost - ? "text/plain" - : "application/octet-stream", - }; - }, - getData(urlInPost, data) { - return urlInPost - ? data - : fs.readFileSync(data); + getHeaders() { + return "application/octet-stream"; + }, + async getData(data) { + return getFileStream(data); }, extractText(args = {}) { return this.app.post({ @@ -138,7 +116,6 @@ export default { extractText, getHeaders, getData, - urlInPost, processingMode, outputMode, pageSeperator, @@ -156,9 +133,9 @@ export default { const response = await extractText({ $, - headers: getHeaders(urlInPost), + headers: getHeaders(), params: { - url_in_post: urlInPost, + url_in_post: false, processing_mode: processingMode, output_mode: outputMode, page_seperator: pageSeperator, @@ -172,7 +149,7 @@ export default { line_splitter_tolerance: lineSplitterTolerance, horizontal_stretch_factor: horizontalStretchFactor, }, - data: getData(urlInPost, data), + data: await getData(data), }); $.export("$summary", "Successfully extracted text from document."); diff --git a/components/llmwhisperer/package.json b/components/llmwhisperer/package.json index 9d9d06ba6ccd0..68766334b83fa 100644 --- a/components/llmwhisperer/package.json +++ b/components/llmwhisperer/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/llmwhisperer", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream LLMWhisperer Components", "main": "llmwhisperer.app.mjs", "keywords": [ @@ -13,7 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.0", - "fs": "^0.0.1-security" + "@pipedream/platform": "^3.1.0" } } diff --git a/components/mapbox/actions/create-tileset/create-tileset.mjs b/components/mapbox/actions/create-tileset/create-tileset.mjs index 2cc4423f4172f..52240a4833525 100644 --- a/components/mapbox/actions/create-tileset/create-tileset.mjs +++ b/components/mapbox/actions/create-tileset/create-tileset.mjs @@ -1,12 +1,12 @@ import mapbox from "../../mapbox.app.mjs"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "mapbox-create-tileset", name: "Create Tileset", description: "Uploads and creates a new tileset from a data source. [See the documentation](https://docs.mapbox.com/api/maps/mapbox-tiling-service/)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { mapbox, @@ -22,8 +22,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to a tileset source file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "A tileset source file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, recipe: { type: "object", @@ -44,15 +44,17 @@ export default { }, }, async run({ $ }) { - const filePath = this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`; - // Create Tileset Source try { const fileData = new FormData(); - const content = fs.createReadStream(filePath); - fileData.append("file", content); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + fileData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); await this.mapbox.createTilesetSource({ $, @@ -62,7 +64,7 @@ export default { headers: fileData.getHeaders(), }); } catch (e) { - throw new Error(`Error uploading file: \`${filePath}\`. Error: ${e}`); + throw new Error(`Error uploading file: \`${this.filePath}\`. Error: ${e}`); } const recipe = typeof this.recipe === "string" diff --git a/components/mapbox/package.json b/components/mapbox/package.json index 680b99cc3f4cb..2265772e4163e 100644 --- a/components/mapbox/package.json +++ b/components/mapbox/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/mapbox", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Mapbox Components", "main": "mapbox.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.1" } } diff --git a/components/microsoft_onedrive/actions/upload-file/upload-file.mjs b/components/microsoft_onedrive/actions/upload-file/upload-file.mjs index 23c25ec949cd8..cc3caa4c85aaa 100644 --- a/components/microsoft_onedrive/actions/upload-file/upload-file.mjs +++ b/components/microsoft_onedrive/actions/upload-file/upload-file.mjs @@ -1,13 +1,14 @@ import onedrive from "../../microsoft_onedrive.app.mjs"; -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; -import { fileTypeFromStream } from "file-type"; +import { + ConfigurationError, + getFileStreamAndMetadata, +} from "@pipedream/platform"; export default { name: "Upload File", description: "Upload a file to OneDrive. [See the documentation](https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online)", key: "microsoft_onedrive-upload-file", - version: "0.1.2", + version: "0.2.0", type: "action", props: { onedrive, @@ -21,8 +22,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to the file saved to the `/tmp` directory (e.g. `/tmp/image.png`). [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory).", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, filename: { type: "string", @@ -39,17 +40,10 @@ export default { throw new ConfigurationError("You must specify the **Upload Folder ID**."); } - let stream = fs.createReadStream(filePath); - let name = filename; - - if (!filename.includes(".")) { - const fileTypeResult = await fileTypeFromStream(stream); - const extension = fileTypeResult?.ext || ""; - name = `${filename}.${extension}`; - - stream.destroy(); - stream = fs.createReadStream(filePath); - } + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); + const name = filename || metadata.name; const response = await this.onedrive.uploadFile({ uploadFolderId, diff --git a/components/microsoft_onedrive/package.json b/components/microsoft_onedrive/package.json index 0e71d55362c72..7d35f2040e98f 100644 --- a/components/microsoft_onedrive/package.json +++ b/components/microsoft_onedrive/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/microsoft_onedrive", - "version": "1.6.2", + "version": "1.7.0", "description": "Pipedream Microsoft OneDrive components", "main": "microsoft_onedrive.app.mjs", "homepage": "https://pipedream.com/apps/microsoft-onedrive", @@ -10,9 +10,8 @@ }, "dependencies": { "@microsoft/microsoft-graph-client": "^3.0.1", - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "bottleneck": "^2.19.5", - "file-type": "^18.7.0", "isomorphic-fetch": "^3.0.0", "lodash.get": "^4.4.2" } diff --git a/components/mistral_ai/actions/upload-file/upload-file.mjs b/components/mistral_ai/actions/upload-file/upload-file.mjs index 6f6583088e4df..b061edb173714 100644 --- a/components/mistral_ai/actions/upload-file/upload-file.mjs +++ b/components/mistral_ai/actions/upload-file/upload-file.mjs @@ -1,20 +1,19 @@ import mistralAI from "../../mistral_ai.app.mjs"; -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "mistral_ai-upload-file", name: "Upload File", description: "Upload a file that can be used across various endpoints. [See the Documentation](https://docs.mistral.ai/api/#tag/files/operation/files_api_routes_upload_file)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { mistralAI, filePath: { type: "string", - label: "File Path", - description: "The path to a file in the `/tmp` directory. The size of individual files can be a maximum of 512 MB. The Fine-tuning API only supports .jsonl files. [See the Pipedream documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, purpose: { type: "string", @@ -29,17 +28,15 @@ export default { }, }, async run({ $ }) { - const filePath = this.filePath.startsWith("/tmp/") - ? this.filePath - : `/tmp/${this.filePath}`; - - if (!fs.existsSync(filePath)) { - throw new ConfigurationError(`File \`${filePath}\` not found`); - } - - const fileContent = fs.createReadStream(filePath); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); const form = new FormData(); - form.append("file", fileContent); + form.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); if (this.purpose) { form.append("purpose", this.purpose); } diff --git a/components/mistral_ai/package.json b/components/mistral_ai/package.json index 091e79e2483ca..aa80ae437bb0e 100644 --- a/components/mistral_ai/package.json +++ b/components/mistral_ai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/mistral_ai", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Mistral AI Components", "main": "mistral_ai.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.2" } } diff --git a/components/monday/actions/update-column-values/update-column-values.mjs b/components/monday/actions/update-column-values/update-column-values.mjs index 8c725ebc8f87c..7a77b4b06d263 100644 --- a/components/monday/actions/update-column-values/update-column-values.mjs +++ b/components/monday/actions/update-column-values/update-column-values.mjs @@ -1,6 +1,7 @@ import common from "../common/column-values.mjs"; -import { axios } from "@pipedream/platform"; -import fs from "fs"; +import { + axios, getFileStreamAndMetadata, +} from "@pipedream/platform"; import FormData from "form-data"; import { getColumnOptions } from "../../common/utils.mjs"; @@ -9,7 +10,7 @@ export default { key: "monday-update-column-values", name: "Update Column Values", description: "Update multiple column values of an item. [See the documentation](https://developer.monday.com/api-reference/reference/columns#change-multiple-column-values)", - version: "0.1.0", + version: "0.2.0", type: "action", props: { ...common.props, @@ -50,7 +51,7 @@ export default { options: getColumnOptions(columns, id), }; if (column.type === "file") { - props[column.id].description += ". The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)"; + props[column.id].description += ". Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)"; } } } @@ -62,13 +63,17 @@ export default { $, itemId, column, filePath, }) { const query = `mutation ($file: File!) { add_file_to_column (file: $file, item_id: ${itemId}, column_id: "${column.id}") { id } }`; - const content = fs.createReadStream(filePath.includes("tmp/") - ? filePath - : `/tmp/${filePath}`); + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); const formData = new FormData(); formData.append("query", query); - formData.append("variables[file]", content); + formData.append("variables[file]", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); return axios($, { method: "POST", diff --git a/components/monday/package.json b/components/monday/package.json index ce48c1bb44f7c..68d8e5edc3384 100644 --- a/components/monday/package.json +++ b/components/monday/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/monday", - "version": "0.7.0", + "version": "0.8.0", "description": "Pipedream Monday Components", "main": "monday.app.mjs", "keywords": [ @@ -14,7 +14,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0", "lodash.flatmap": "^4.5.0", "lodash.map": "^4.6.0", diff --git a/components/nyckel/actions/classify-image/classify-image.mjs b/components/nyckel/actions/classify-image/classify-image.mjs index c7b3f712e8657..9a7b46a1f6647 100644 --- a/components/nyckel/actions/classify-image/classify-image.mjs +++ b/components/nyckel/actions/classify-image/classify-image.mjs @@ -6,7 +6,7 @@ export default { key: "nyckel-classify-image", name: "Classify Image", description: "Classifies image data based on pre-trained classifiers in Nyckel. [See the documentation](https://www.nyckel.com/docs#invoke-image)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { nyckel, @@ -46,7 +46,7 @@ export default { const response = await this.nyckel.invokeFunction({ $, functionId: this.functionId, - ...this.getImageData(), + ...(await this.getImageData()), params: { labelCount: this.labelCount, includeMetadata: this.includeMetadata, diff --git a/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs b/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs index 289b0514f4103..ec0028ea8fedc 100644 --- a/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs +++ b/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs @@ -6,7 +6,7 @@ export default { key: "nyckel-extract-text-from-image", name: "Extract Text from Image", description: "Extracts text from an image URL. [See the documentation](https://www.nyckel.com/docs#ocr-image)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { nyckel, @@ -28,7 +28,7 @@ export default { const response = await this.nyckel.extractTextFromImageUrl({ $, functionId: this.functionId, - ...this.getImageData(), + ...(await this.getImageData()), params: { includeRegions: this.includeRegions, }, diff --git a/components/nyckel/common/common-image.mjs b/components/nyckel/common/common-image.mjs index a18f1fad59693..a89cee9b64417 100644 --- a/components/nyckel/common/common-image.mjs +++ b/components/nyckel/common/common-image.mjs @@ -1,6 +1,6 @@ import nyckel from "../nyckel.app.mjs"; import FormData from "form-data"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { props: { @@ -12,26 +12,16 @@ export default { }, }, methods: { - getImageData() { - const { imageOrUrl } = this; - const isUrl = imageOrUrl.startsWith("http"); - if (isUrl) { - return { - data: { - data: imageOrUrl, - }, - headers: { - "Content-Type": "application/json", - }, - }; - } - + async getImageData() { const data = new FormData(); - data.append("data", fs.createReadStream( - imageOrUrl.includes("tmp/") - ? imageOrUrl - : `/tmp/${imageOrUrl}`, - )); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.imageOrUrl); + data.append("data", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); return { data, diff --git a/components/nyckel/nyckel.app.mjs b/components/nyckel/nyckel.app.mjs index 23dedcf06617d..064d4570a9741 100644 --- a/components/nyckel/nyckel.app.mjs +++ b/components/nyckel/nyckel.app.mjs @@ -29,8 +29,8 @@ export default { }, imageOrUrl: { type: "string", - label: "Image Path or URL", - description: "The path to an image file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp). Alternatively, you can pass the direct URL to a file.", + label: "File Path or URL", + description: "An image file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", }, includeRegions: { type: "boolean", diff --git a/components/nyckel/package.json b/components/nyckel/package.json index 41b880a1a4950..e1301209310e6 100644 --- a/components/nyckel/package.json +++ b/components/nyckel/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/nyckel", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Nyckel Components", "main": "nyckel.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.6.2", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/components/ocrspace/actions/common/process-base.mjs b/components/ocrspace/actions/common/process-base.mjs index 2a8fbc60500d1..da822c6e2f479 100644 --- a/components/ocrspace/actions/common/process-base.mjs +++ b/components/ocrspace/actions/common/process-base.mjs @@ -45,11 +45,8 @@ export default { }, async run({ $ }) { const data = new FormData(); - const { - url, file, - } = getUrlOrFile(this.file); + const file = await getUrlOrFile(this.file); - if (url) data.append("url", url); if (file) data.append("base64Image", file); if (this.imageLanguage) data.append("language", this.imageLanguage); if (this.isOverlayRequired) data.append("isOverlayRequired", `${this.isOverlayRequired}`); diff --git a/components/ocrspace/actions/process-image/process-image.mjs b/components/ocrspace/actions/process-image/process-image.mjs index dace35e1e2ca1..bcbea25dfc269 100644 --- a/components/ocrspace/actions/process-image/process-image.mjs +++ b/components/ocrspace/actions/process-image/process-image.mjs @@ -5,7 +5,7 @@ export default { key: "ocrspace-process-image", name: "Process Image", description: "Submits an image file for OCR processing using OCR.space. [See the documentation](https://ocr.space/ocrapi)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { ...common.props, diff --git a/components/ocrspace/actions/process-pdf/process-pdf.mjs b/components/ocrspace/actions/process-pdf/process-pdf.mjs index cda0790f5c620..223e525fc3aa5 100644 --- a/components/ocrspace/actions/process-pdf/process-pdf.mjs +++ b/components/ocrspace/actions/process-pdf/process-pdf.mjs @@ -5,7 +5,7 @@ export default { key: "ocrspace-process-pdf", name: "Process PDF for OCR", description: "Submit a PDF for OCR processing. [See the documentation](https://ocr.space/ocrapi)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { ...common.props, diff --git a/components/ocrspace/common/utils.mjs b/components/ocrspace/common/utils.mjs index 3384dcc350320..d9fb05bac28f1 100644 --- a/components/ocrspace/common/utils.mjs +++ b/components/ocrspace/common/utils.mjs @@ -1,13 +1,15 @@ -import fs from "fs"; -import mime from "mime"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export const isValidUrl = (urlString) => { - var urlPattern = new RegExp("^(https?:\\/\\/)?" + // validate protocol -"((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|" + // validate domain name -"((\\d{1,3}\\.){3}\\d{1,3}))" + // validate OR ip (v4) address -"(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*" + // validate port and path -"(\\?[;&a-z\\d%_.~+=-]*)?" + // validate query string -"(\\#[-a-z\\d_]*)?$", "i"); // validate fragment locator + var urlPattern = new RegExp( + "^(https?:\\/\\/)?" + // validate protocol + "((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|" + // validate domain name + "((\\d{1,3}\\.){3}\\d{1,3}))" + // validate OR ip (v4) address + "(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*" + // validate port and path + "(\\?[;&a-z\\d%_.~+=-]*)?" + // validate query string + "(\\#[-a-z\\d_]*)?$", + "i", + ); // validate fragment locator return !!urlPattern.test(urlString); }; @@ -18,17 +20,16 @@ export const checkTmp = (filename) => { return filename; }; -export const getUrlOrFile = (url) => { - if (!isValidUrl(url)) { - const filePath = checkTmp(url); - const data = fs.readFileSync(filePath); - const mimeType = mime.getType(filePath); - const base64Image = Buffer.from(data, "binary").toString("base64"); - return { - file: `data:${mimeType};base64,${base64Image}`, - }; +export const getUrlOrFile = async (url) => { + const { + stream, + metadata: { contentType }, + } = await getFileStreamAndMetadata(url); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); } - return { - url, - }; + const buffer = Buffer.concat(chunks); + const base64Image = buffer.toString("base64"); + return `data:${contentType};base64,${base64Image}`; }; diff --git a/components/ocrspace/ocrspace.app.mjs b/components/ocrspace/ocrspace.app.mjs index ae96d7db00db7..c43d7de875022 100644 --- a/components/ocrspace/ocrspace.app.mjs +++ b/components/ocrspace/ocrspace.app.mjs @@ -11,8 +11,8 @@ export default { propDefinitions: { file: { type: "string", - label: "Image", - description: "The URL of the image or the path to the file saved to the `/tmp` directory (e.g. `/tmp/example.jpg`) to process. [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory).", + label: "File Path or URL", + description: "The file to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", }, language: { type: "string", diff --git a/components/ocrspace/package.json b/components/ocrspace/package.json index a0ab1ae330c10..656f2928aca88 100644 --- a/components/ocrspace/package.json +++ b/components/ocrspace/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/ocrspace", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream OCRSpace Components", "main": "ocrspace.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "mime": "^4.0.6" } } diff --git a/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs b/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs index daa5cb15e5fff..509b9bb478c1b 100644 --- a/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs +++ b/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs @@ -1,13 +1,12 @@ -import fs from "fs"; import FormData from "form-data"; -import { ConfigurationError } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import app from "../../onlinecheckwriter.app.mjs"; export default { key: "onlinecheckwriter-mail-pdf-document", name: "Mail PDF Document", description: "Mails a PDF document to a destination. [See the documentation](https://apiv3.onlinecheckwriter.com/#878daf05-e36e-44a2-bce8-15f24d72f82e).", - version: "0.0.1", + version: "0.1.0", type: "action", props: { app, @@ -19,8 +18,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to the pdf file saved to the `/tmp` directory (e.g. `/tmp/example.pdf`). [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory).", + label: "File Path or URL", + description: "The PDF file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, shippingTypeId: { optional: false, @@ -167,13 +166,15 @@ export default { destinationEmail, } = this; - if (!filePath?.startsWith("/tmp/")) { - throw new ConfigurationError("The file path must start with `/tmp/`."); - } - const data = new FormData(); - const file = fs.createReadStream(filePath); - data.append("document_details[file]", file); + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); + data.append("document_details[file]", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); data.append("document_details[title]", documentTitle || ""); data.append("shipping[shippingTypeId]", shippingTypeId || ""); data.append("destination[name]", destinationName || ""); diff --git a/components/onlinecheckwriter/package.json b/components/onlinecheckwriter/package.json index b3ad9e2566d5a..7ec6e37eca1d8 100644 --- a/components/onlinecheckwriter/package.json +++ b/components/onlinecheckwriter/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/onlinecheckwriter", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream OnlineCheckWriter Components", "main": "onlinecheckwriter.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/components/onlyoffice_docspace/actions/upload-file/upload-file.mjs b/components/onlyoffice_docspace/actions/upload-file/upload-file.mjs index e1dd3512f196f..4851c15d13f3b 100644 --- a/components/onlyoffice_docspace/actions/upload-file/upload-file.mjs +++ b/components/onlyoffice_docspace/actions/upload-file/upload-file.mjs @@ -5,7 +5,7 @@ export default { key: "onlyoffice_docspace-upload-file", name: "Upload File", description: "Uploads a file to the specified room. [See the documentation](https://api.onlyoffice.com/docspace/method/files/post/api/2.0/files/%7bfolderid%7d/upload)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { app, @@ -19,8 +19,8 @@ export default { }, file: { type: "string", - label: "File", - description: "File path of a file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, methods: { diff --git a/components/onlyoffice_docspace/common/utils.mjs b/components/onlyoffice_docspace/common/utils.mjs index a4710a240ed9f..953b4070c9cf6 100644 --- a/components/onlyoffice_docspace/common/utils.mjs +++ b/components/onlyoffice_docspace/common/utils.mjs @@ -1,26 +1,30 @@ -import { createReadStream } from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; import constants from "./constants.mjs"; -function buildFormData(formData, data, parentKey) { +async function buildFormData(formData, data, parentKey) { if (data && typeof(data) === "object") { - Object.keys(data) - .forEach((key) => { - buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); - }); - + for (const key of Object.keys(data)) { + await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); + } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, createReadStream(data)); - + const { + stream, metadata, + } = await getFileStreamAndMetadata(data); + formData.append(parentKey, stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } else if (data) { formData.append(parentKey, (data).toString()); } } -function getFormData(data) { +async function getFormData(data) { try { const formData = new FormData(); - buildFormData(formData, data); + await buildFormData(formData, data); return formData; } catch (error) { console.log("FormData Error", error); diff --git a/components/onlyoffice_docspace/onlyoffice_docspace.app.mjs b/components/onlyoffice_docspace/onlyoffice_docspace.app.mjs index a2380211af035..78e3812015a2a 100644 --- a/components/onlyoffice_docspace/onlyoffice_docspace.app.mjs +++ b/components/onlyoffice_docspace/onlyoffice_docspace.app.mjs @@ -46,12 +46,12 @@ export default { ...headers, }; }, - getConfig({ + async getConfig({ headers, data: preData, ...args } = {}) { const contentType = constants.CONTENT_TYPE_KEY_HEADER; const hasMultipartHeader = utils.hasMultipartHeader(headers); - const data = hasMultipartHeader && utils.getFormData(preData) || preData; + const data = hasMultipartHeader && await utils.getFormData(preData) || preData; const currentHeaders = this.getHeaders(headers); return { @@ -65,10 +65,10 @@ export default { ...args, }; }, - _makeRequest({ + async _makeRequest({ $ = this, path, headers, ...args } = {}) { - const config = this.getConfig({ + const config = await this.getConfig({ url: this.getUrl(path), headers: this.getHeaders(headers), ...args, diff --git a/components/onlyoffice_docspace/package.json b/components/onlyoffice_docspace/package.json index cab0756882afd..a5951e49fb3c8 100644 --- a/components/onlyoffice_docspace/package.json +++ b/components/onlyoffice_docspace/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/onlyoffice_docspace", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream ONLYOFFICE DocSpace Components", "main": "onlyoffice_docspace.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.0", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs b/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs index ce7caf8c31f87..91c77f10230ea 100644 --- a/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs +++ b/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs @@ -1,7 +1,5 @@ import app from "../../pandadoc.app.mjs"; -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; -import path from "path"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { @@ -9,7 +7,7 @@ export default { name: "Create Document Attachment", description: "Adds an attachment to a document. [See the documentation here](https://developers.pandadoc.com/reference/create-document-attachment)", type: "action", - version: "0.0.7", + version: "0.1.0", props: { app, documentId: { @@ -31,39 +29,17 @@ export default { }, }, methods: { - isValidFile(filePath) { - const filePathWithTmp = `/tmp/${filePath}`; - if (fs.existsSync(filePathWithTmp)) { - return filePathWithTmp; - } else if (fs.existsSync(filePath)) { - return filePath; - } - return false; - }, - getFileStream(filePath) { - return fs.createReadStream(filePath); - }, - getFileMeta(filePath) { - const stats = fs.statSync(filePath); - return { - name: path.basename(filePath), - size: stats.size, - }; - }, - getFormData(file, fileName) { - const fileValidation = this.isValidFile(file); - if (!fileValidation) { - throw new ConfigurationError("`file` must be a valid file path!"); - } - - const fileMeta = this.getFileMeta(fileValidation); - const fileContent = this.getFileStream(fileValidation); + async getFormData(file, fileName) { + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); const data = new FormData(); - if (fileName) data.append("name", fileName); - data.append("file", fileContent, { - knownLength: fileMeta.size, + data.append("name", fileName || metadata.name); + data.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, }); - return data; }, }, @@ -74,7 +50,7 @@ export default { fileName, } = this; - const data = this.getFormData(file, fileName); + const data = await this.getFormData(file, fileName); const response = await this.app.createDocumentAttachments({ $, diff --git a/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs b/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs index dcd5bc0d454f2..240af1e76bc00 100644 --- a/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs +++ b/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs @@ -7,7 +7,7 @@ export default { name: "Create Document From File", description: "Create a document from a file or public file URL. [See the documentation here](https://developers.pandadoc.com/reference/create-document-from-pdf)", type: "action", - version: "0.0.8", + version: "1.0.0", props: { app, name: { @@ -27,13 +27,6 @@ export default { app, "file", ], - optional: true, - }, - fileUrl: { - type: "string", - label: "File URL", - description: "A public file URL to use instead of a local file.", - optional: true, }, documentFolderId: { propDefinition: [ @@ -55,7 +48,6 @@ export default { name, recipients, file, - fileUrl, documentFolderId, } = this; @@ -66,7 +58,7 @@ export default { throw new ConfigurationError("**Error parsing recipients** - each must be a valid JSON-stringified object"); } - let data, contentType, json = { + const json = { name, recipients: parsedRecipients, folder_uuid: documentFolderId, @@ -78,15 +70,9 @@ export default { : this.fields; } - if (fileUrl) { - data = json; - contentType = "application/json"; - data.url = fileUrl; - } else { - data = this.getFormData(file); - contentType = `multipart/form-data; boundary=${data._boundary}`; - data.append("data", JSON.stringify(json)); - } + const data = await this.getFormData(file); + const contentType = `multipart/form-data; boundary=${data._boundary}`; + data.append("data", JSON.stringify(json)); const response = await this.app.createDocument({ $, diff --git a/components/pandadoc/package.json b/components/pandadoc/package.json index e1acc2088c9ea..f896236cfc31a 100644 --- a/components/pandadoc/package.json +++ b/components/pandadoc/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/pandadoc", - "version": "0.3.0", + "version": "1.0.0", "description": "Pipedream PandaDoc Components", "main": "pandadoc.app.mjs", "keywords": [ @@ -14,7 +14,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.6.0", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/components/pandadoc/pandadoc.app.mjs b/components/pandadoc/pandadoc.app.mjs index fd4de74e7b696..a3c2f59132532 100644 --- a/components/pandadoc/pandadoc.app.mjs +++ b/components/pandadoc/pandadoc.app.mjs @@ -88,8 +88,8 @@ export default { }, file: { type: "string", - label: "File", - description: "The file to upload from the `/tmp` folder. [See the docs here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp) on how to upload a file to `/tmp`.", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, name: { type: "string", diff --git a/components/pdf4me/actions/compress-pdf/compress-pdf.mjs b/components/pdf4me/actions/compress-pdf/compress-pdf.mjs index 7f259e3fc19f6..ebf39e88eec97 100644 --- a/components/pdf4me/actions/compress-pdf/compress-pdf.mjs +++ b/components/pdf4me/actions/compress-pdf/compress-pdf.mjs @@ -1,12 +1,11 @@ import pdf4me from "../../pdf4me.app.mjs"; import utils from "../../common/utils.mjs"; -import fs from "fs"; export default { key: "pdf4me-compress-pdf", name: "Compress PDF", description: "Compress a PDF file to reduce its size. [See the documentation](https://dev.pdf4me.com/apiv2/documentation/actions/compress-pdf/)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { pdf4me, @@ -41,10 +40,7 @@ export default { }, async run({ $ }) { const filename = utils.checkForExtension(this.filename, "pdf"); - const filePath = utils.normalizeFilePath(this.filePath); - const fileContent = fs.readFileSync(filePath, { - encoding: "base64", - }); + const fileContent = await utils.getBase64File(this.filePath); const response = await this.pdf4me.compressPdf({ $, diff --git a/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs b/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs index cf87f9f9f1144..d8880bfeb3219 100644 --- a/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs +++ b/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs @@ -1,12 +1,11 @@ import pdf4me from "../../pdf4me.app.mjs"; import utils from "../../common/utils.mjs"; -import fs from "fs"; export default { key: "pdf4me-convert-to-pdf", name: "Convert to PDF", description: "Convert a document (e.g., DOCX, XLSX, PPTX) to PDF. [See the documentation](https://dev.pdf4me.com/apiv2/documentation/actions/convert-to-pdf/)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { pdf4me, @@ -15,7 +14,6 @@ export default { pdf4me, "filePath", ], - description: "The path to a DOCX, XLSX, or PPTX file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", }, filename: { propDefinition: [ @@ -25,16 +23,14 @@ export default { }, }, async run({ $ }) { - const filePath = utils.normalizeFilePath(this.filePath); - const fileContent = fs.readFileSync(filePath, { - encoding: "base64", - }); + const filename = utils.checkForExtension(this.filename, "pdf"); + const fileContent = await utils.getBase64File(this.filePath); const response = await this.pdf4me.convertToPdf({ $, data: { docContent: fileContent, - docName: this.filename, + docName: filename, }, responseType: "arraybuffer", }); diff --git a/components/pdf4me/actions/merge-pdfs/merge-pdfs.mjs b/components/pdf4me/actions/merge-pdfs/merge-pdfs.mjs index df5857656831d..f2e9f51f3481b 100644 --- a/components/pdf4me/actions/merge-pdfs/merge-pdfs.mjs +++ b/components/pdf4me/actions/merge-pdfs/merge-pdfs.mjs @@ -1,19 +1,18 @@ import pdf4me from "../../pdf4me.app.mjs"; import utils from "../../common/utils.mjs"; -import fs from "fs"; export default { key: "pdf4me-merge-pdfs", name: "Merge PDF Files", description: "Merge multiple PDF files into a single PDF. [See the documentation](https://dev.pdf4me.com/apiv2/documentation/actions/merge/)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { pdf4me, filePaths: { type: "string[]", - label: "File Paths", - description: "An array of paths to a PDF files in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Paths or URLs", + description: "The files to process. For each entry, provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, filename: { propDefinition: [ @@ -24,10 +23,8 @@ export default { }, async run({ $ }) { const filename = utils.checkForExtension(this.filename, "pdf"); - const filePaths = this.filePaths.map((path) => utils.normalizeFilePath(path)); - const fileContents = filePaths.map((path) => fs.readFileSync(path, { - encoding: "base64", - })); + const promises = await Promise.allSettled(this.filePaths.map((p) => utils.getBase64File(p))); + const fileContents = promises.map((prom) => prom.value); const response = await this.pdf4me.mergePdfs({ $, diff --git a/components/pdf4me/common/utils.mjs b/components/pdf4me/common/utils.mjs index 6f46d4edf4569..68dd4219936c4 100644 --- a/components/pdf4me/common/utils.mjs +++ b/components/pdf4me/common/utils.mjs @@ -1,5 +1,7 @@ import fs from "fs"; -import { ConfigurationError } from "@pipedream/platform"; +import { + ConfigurationError, getFileStream, +} from "@pipedream/platform"; function normalizeFilePath(path) { return path.startsWith("/tmp/") @@ -41,9 +43,22 @@ function handleErrorMessage(error) { throw new ConfigurationError(errorMessage); } +async function getBase64File(filePath) { + try { + const stream = await getFileStream(filePath); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + return Buffer.concat(chunks).toString("base64"); + } catch (error) { + throw new ConfigurationError(`Error parsing file \`${filePath}\`: ${error.message}`); + } +} + export default { - normalizeFilePath, checkForExtension, downloadToTmp, handleErrorMessage, + getBase64File, }; diff --git a/components/pdf4me/package.json b/components/pdf4me/package.json index ba088f91d5b82..0dff14900e86e 100644 --- a/components/pdf4me/package.json +++ b/components/pdf4me/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/pdf4me", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream PDF4me Components", "main": "pdf4me.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3" + "@pipedream/platform": "^3.1.0" } } diff --git a/components/pdf4me/pdf4me.app.mjs b/components/pdf4me/pdf4me.app.mjs index ec179a9ff9730..6a552775d238f 100644 --- a/components/pdf4me/pdf4me.app.mjs +++ b/components/pdf4me/pdf4me.app.mjs @@ -7,8 +7,8 @@ export default { propDefinitions: { filePath: { type: "string", - label: "File Path", - description: "The path to a PDF file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.doc`)", }, filename: { type: "string", diff --git a/components/slack/actions/upload-file/upload-file.mjs b/components/slack/actions/upload-file/upload-file.mjs index 5489b4f2899b2..bbf4c2a3f34b7 100644 --- a/components/slack/actions/upload-file/upload-file.mjs +++ b/components/slack/actions/upload-file/upload-file.mjs @@ -1,7 +1,6 @@ import { - ConfigurationError, axios, + ConfigurationError, axios, getFileStreamAndMetadata, } from "@pipedream/platform"; -import fs from "fs"; import FormData from "form-data"; import slack from "../../slack.app.mjs"; @@ -9,7 +8,7 @@ export default { key: "slack-upload-file", name: "Upload File", description: "Upload a file. [See the documentation](https://api.slack.com/messaging/files#uploading_files)", - version: "0.0.27", + version: "0.1.0", type: "action", props: { slack, @@ -35,16 +34,16 @@ export default { }, }, async run({ $ }) { - if (!fs.existsSync(this.content)) { - throw new ConfigurationError(`\`${this.content}\` not found, a valid \`/tmp\` path is needed`); - } + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.content); const filename = this.content.split("/").pop(); // Get an upload URL from Slack const getUploadUrlResponse = await this.slack.getUploadUrl({ filename, - length: fs.statSync(this.content).size, + length: metadata.size, }); if (!getUploadUrlResponse.ok) { @@ -57,7 +56,11 @@ export default { // Upload the file to the provided URL const formData = new FormData(); - formData.append("file", fs.createReadStream(this.content)); + formData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); formData.append("filename", filename); await axios($, { diff --git a/components/slack/package.json b/components/slack/package.json index 8860416ac1880..a17294f9bd59a 100644 --- a/components/slack/package.json +++ b/components/slack/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/slack", - "version": "0.9.5", + "version": "0.10.0", "description": "Pipedream Slack Components", "main": "slack.app.mjs", "keywords": [ @@ -14,7 +14,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.0", + "@pipedream/platform": "^3.1.0", "@slack/web-api": "^7.9.0", "async-retry": "^1.3.3", "lodash": "^4.17.21" diff --git a/components/slack/slack.app.mjs b/components/slack/slack.app.mjs index 81b560452a7ac..401d853f0f6cb 100644 --- a/components/slack/slack.app.mjs +++ b/components/slack/slack.app.mjs @@ -356,8 +356,8 @@ export default { }, }, content: { - label: "File Path", - description: "Full path to the file in `/tmp/` directory. E.g. `/tmp/cute_cat.jpg`", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", type: "string", }, link_names: { diff --git a/components/stannp/actions/create-campaign/create-campaign.mjs b/components/stannp/actions/create-campaign/create-campaign.mjs index a09263de71159..b5f76d8474add 100644 --- a/components/stannp/actions/create-campaign/create-campaign.mjs +++ b/components/stannp/actions/create-campaign/create-campaign.mjs @@ -2,14 +2,14 @@ import FormData from "form-data"; import { TYPE_OPTIONS, WHAT_RECIPIENTS_OPTIONS, } from "../../common/constants.mjs"; -import { checkFile } from "../../common/utils.mjs"; +import { getFileData } from "../../common/utils.mjs"; import stannp from "../../stannp.app.mjs"; export default { key: "stannp-create-campaign", name: "Create a New Campaign", description: "Create a new campaign in Stannp. [See the documentation](https://www.stannp.com/us/direct-mail-api/campaigns)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { stannp, @@ -58,22 +58,22 @@ export default { }, file: { type: "string", - label: "File", - description: "A single or multi-page PDF file to use as the design artwork. can be a URL or a path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp). Make sure the image is in the correct format.", + label: "PDF File Path or URL", + description: "A PDF file to use as the design artwork. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", optional: true, reloadProps: true, }, front: { type: "string", - label: "Front", - description: "A PDF or JPG file to use as the front image. Can be a URL or a path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp). Make sure the image is in the correct format.", + label: "Front Image Path or URL", + description: "A PDF or JPG file to use as the front image. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", optional: true, reloadProps: true, }, back: { type: "string", - label: "Back", - description: "A PDF or JPG file to use as the back image. Can be a URL or a path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp). Make sure the image is in the correct format.", + label: "Back Image Path or URL", + description: "A PDF or JPG file to use as the back image. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", optional: true, reloadProps: true, }, @@ -107,9 +107,24 @@ export default { const formData = new FormData(); - if (file) formData.append("file", await checkFile(file)); - if (front) formData.append("front", await checkFile(front)); - if (back) formData.append("back", await checkFile(back)); + if (file) { + const { + stream, metadata, + } = await getFileData(file); + formData.append("file", stream, metadata); + } + if (front) { + const { + stream, metadata, + } = await getFileData(front); + formData.append("front", stream, metadata); + } + if (back) { + const { + stream, metadata, + } = await getFileData(back); + formData.append("back", stream, metadata); + } if (templateId) formData.append("template_id", templateId); if (groupId) formData.append("group_id", groupId); diff --git a/components/stannp/common/utils.mjs b/components/stannp/common/utils.mjs index 39c2e4c8e4409..516df2adbec60 100644 --- a/components/stannp/common/utils.mjs +++ b/components/stannp/common/utils.mjs @@ -1,5 +1,4 @@ -import fs from "fs"; -import urlExists from "url-exist"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export const parseObject = (obj) => { if (Array.isArray(obj)) { @@ -20,17 +19,16 @@ export const parseObject = (obj) => { return obj; }; -export const checkFile = async (url) => { - if (await urlExists(url)) { - return url; - } else { - return fs.createReadStream(checkTmp(url)); - } -}; - -const checkTmp = (filename) => { - if (!filename.startsWith("/tmp")) { - return `/tmp/${filename}`; - } - return filename; +export const getFileData = async (file) => { + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); + return { + stream, + metadata: { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }, + }; }; diff --git a/components/stannp/package.json b/components/stannp/package.json index 729a07205bde6..c5f3ab3608c32 100644 --- a/components/stannp/package.json +++ b/components/stannp/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/stannp", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Stannp Components", "main": "stannp.app.mjs", "keywords": [ @@ -13,8 +13,7 @@ "access": "public" }, "dependencies": { - "form-data": "^4.0.0", - "fs": "^0.0.1-security", - "url-exist": "^3.0.1" + "@pipedream/platform": "^3.1.0", + "form-data": "^4.0.0" } } diff --git a/components/zamzar/actions/start-job-from-file/start-job-from-file.mjs b/components/zamzar/actions/start-job-from-file/start-job-from-file.mjs index 9c8a66a3cd67d..80361b1405620 100644 --- a/components/zamzar/actions/start-job-from-file/start-job-from-file.mjs +++ b/components/zamzar/actions/start-job-from-file/start-job-from-file.mjs @@ -5,7 +5,7 @@ export default { key: "zamzar-start-job-from-file", name: "Start Job From File", description: "Starts a conversion job and upload a source file in a single request. [See the documentation](https://developers.zamzar.com/docs)", - version: "0.0.2", + version: "0.1.0", type: "action", props: { app, diff --git a/components/zamzar/common/utils.mjs b/components/zamzar/common/utils.mjs index c812fe7712329..712254a276dbe 100644 --- a/components/zamzar/common/utils.mjs +++ b/components/zamzar/common/utils.mjs @@ -1,26 +1,30 @@ -import { createReadStream } from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; import constants from "./constants.mjs"; -function buildFormData(formData, data, parentKey) { +async function buildFormData(formData, data, parentKey) { if (data && typeof(data) === "object") { - Object.keys(data) - .forEach(async (key) => { - buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); - }); - + for (const key of Object.keys(data)) { + await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); + } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, createReadStream(data)); - + const { + stream, metadata, + } = await getFileStreamAndMetadata(data); + formData.append(parentKey, stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } else if (data) { formData.append(parentKey, (data).toString()); } } -function getFormData(data) { +async function getFormData(data) { try { const formData = new FormData(); - buildFormData(formData, data); + await buildFormData(formData, data); return formData; } catch (error) { console.log("FormData Error", error); diff --git a/components/zamzar/package.json b/components/zamzar/package.json index 1422f5753baed..35fab82135a96 100644 --- a/components/zamzar/package.json +++ b/components/zamzar/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/zamzar", - "version": "0.1.1", + "version": "0.2.0", "description": "Pipedream Zamzar Components", "main": "zamzar.app.mjs", "keywords": [ @@ -13,8 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.6.2", - "form-data": "^4.0.0", - "fs": "^0.0.1-security" + "@pipedream/platform": "^3.1.0", + "form-data": "^4.0.0" } } diff --git a/components/zamzar/zamzar.app.mjs b/components/zamzar/zamzar.app.mjs index 7089d2c43c60a..7077c2575768e 100644 --- a/components/zamzar/zamzar.app.mjs +++ b/components/zamzar/zamzar.app.mjs @@ -8,8 +8,8 @@ export default { propDefinitions: { sourceFile: { type: "string", - label: "Source File", - description: "The path to the file saved to the `/tmp` directory (e.g. `/tmp/image.png`). [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory).", + label: "Source File Path or URL", + description: "The file to convert. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, targetFormat: { type: "string", @@ -45,7 +45,7 @@ export default { password: "", }; }, - _makeRequest({ + async _makeRequest({ $ = this, path, headers, data: rawData, ...args } = {}) { const { @@ -55,7 +55,7 @@ export default { const contentType = constants.CONTENT_TYPE_KEY_HEADER; const hasMultipartHeader = utils.hasMultipartHeader(headers); - const data = hasMultipartHeader && utils.getFormData(rawData) || rawData; + const data = hasMultipartHeader && await utils.getFormData(rawData) || rawData; const config = { ...args, diff --git a/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs b/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs index b20ee80d44a3f..9ed510dca8b21 100644 --- a/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs +++ b/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs @@ -1,12 +1,13 @@ import zohoDesk from "../../zoho_desk.app.mjs"; -import constants from "../../common/constants.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; +import FormData from "form-data"; export default { key: "zoho_desk-add-ticket-attachment", name: "Add Ticket Attachment", description: "Attaches a file to a ticket. [See the docs here](https://desk.zoho.com/DeskAPIDocument#TicketAttachments#TicketAttachments_CreateTicketattachment)", type: "action", - version: "0.0.2", + version: "0.1.0", props: { zohoDesk, orgId: { @@ -32,8 +33,8 @@ export default { }, file: { type: "string", - label: "File", - description: "File path of a file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)", + label: "File Path or URL", + description: "The file to attach. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, async run({ $ }) { @@ -44,19 +45,30 @@ export default { file, } = this; + const data = new FormData(); + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); + data.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); + const response = await this.zohoDesk.createTicketAttachment({ + $, ticketId, + params: { + isPublic, + }, headers: { orgId, - ...constants.MULTIPART_FORM_DATA_HEADERS, - }, - data: { - file, - isPublic, + ...data.getHeaders(), }, + data, }); - $.export("$summary", `Successfully created a new ticket attachment with ID ${response.id}`); + $.export("$summary", `Successfully created a new ticket attachment with ID ${response?.id}`); return response; }, diff --git a/components/zoho_desk/common/utils.mjs b/components/zoho_desk/common/utils.mjs index 21b502910e075..dff671bc33bc6 100644 --- a/components/zoho_desk/common/utils.mjs +++ b/components/zoho_desk/common/utils.mjs @@ -1,4 +1,4 @@ -import { createReadStream } from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; import retry from "async-retry"; import constants from "./constants.mjs"; @@ -11,25 +11,29 @@ async function streamIterator(stream) { return resources; } -function buildFormData(formData, data, parentKey) { +async function buildFormData(formData, data, parentKey) { if (data && typeof(data) === "object") { - Object.keys(data) - .forEach(async (key) => { - buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); - }); - + for (const key of Object.keys(data)) { + await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); + } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, createReadStream(data)); - + const { + stream, metadata, + } = await getFileStreamAndMetadata(data); + formData.append(parentKey, stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } else if (data) { formData.append(parentKey, (data).toString()); } } -function getFormData(data) { +async function getFormData(data) { try { const formData = new FormData(); - buildFormData(formData, data); + await buildFormData(formData, data); return formData; } catch (error) { console.log("FormData Error", error); diff --git a/components/zoho_desk/package.json b/components/zoho_desk/package.json index 9f0e0ff2849d1..9dee94e481eb8 100644 --- a/components/zoho_desk/package.json +++ b/components/zoho_desk/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/zoho_desk", - "version": "0.0.4", + "version": "0.1.0", "description": "Pipedream Zoho_desk Components", "main": "zoho_desk.app.mjs", "keywords": [ @@ -10,10 +10,9 @@ "homepage": "https://pipedream.com/apps/zoho_desk", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^1.6.2", + "@pipedream/platform": "^3.1.0", "async-retry": "^1.3.3", - "form-data": "^4.0.0", - "fs": "^0.0.1-security" + "form-data": "^4.0.0" }, "publishConfig": { "access": "public" diff --git a/components/zoho_desk/zoho_desk.app.mjs b/components/zoho_desk/zoho_desk.app.mjs index 8a531dd6d3f7d..5df21a3aa84aa 100644 --- a/components/zoho_desk/zoho_desk.app.mjs +++ b/components/zoho_desk/zoho_desk.app.mjs @@ -124,31 +124,16 @@ export default { $ = this, url, path, - headers: preHeaders, params, - data: preData, + headers, versionPath = constants.VERSION_PATH, withRetries = true, ...args } = {}) { - const contentType = constants.CONTENT_TYPE_KEY_HEADER; - - const hasMultipartHeader = utils.hasMultipartHeader(preHeaders); - const data = hasMultipartHeader && utils.getFormData(preData) || preData; - - const currentHeaders = this.getHeaders(preHeaders); - const headers = hasMultipartHeader - ? { - ...currentHeaders, - [contentType]: data.getHeaders()[contentType.toLowerCase()], - } - : currentHeaders; - const config = { - headers, url: this.getUrl(url, path, versionPath), params: this.getParams(url, params), - data, + headers: this.getHeaders(headers), ...args, }; try { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 56ffbfe751134..607c8ee027d8a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -154,8 +154,8 @@ importers: components/_2markdown: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.1 version: 4.0.1 @@ -609,8 +609,8 @@ importers: components/akeneo: dependencies: '@pipedream/platform': - specifier: ^1.3.0 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -990,8 +990,7 @@ importers: specifier: ^3.0.0 version: 3.0.3 - components/arpoone: - specifiers: {} + components/arpoone: {} components/arxiv: {} @@ -1019,14 +1018,11 @@ importers: components/askyourpdf: dependencies: '@pipedream/platform': - specifier: ^1.6.2 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security components/assembla: dependencies: @@ -1747,8 +1743,8 @@ importers: components/box: dependencies: '@pipedream/platform': - specifier: ^1.5.1 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -4585,8 +4581,8 @@ importers: components/fileforge: dependencies: '@pipedream/platform': - specifier: ^3.0.1 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -7613,11 +7609,8 @@ importers: components/llmwhisperer: dependencies: '@pipedream/platform': - specifier: ^3.0.0 - version: 3.0.3 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security + specifier: ^3.1.0 + version: 3.1.0 components/lmnt: dependencies: @@ -7955,8 +7948,8 @@ importers: components/mapbox: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.1 version: 4.0.1 @@ -8254,14 +8247,11 @@ importers: specifier: ^3.0.1 version: 3.0.7 '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 bottleneck: specifier: ^2.19.5 version: 2.19.5 - file-type: - specifier: ^18.7.0 - version: 18.7.0 isomorphic-fetch: specifier: ^3.0.0 version: 3.0.0 @@ -8383,8 +8373,8 @@ importers: components/mistral_ai: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.2 version: 4.0.2 @@ -8464,8 +8454,8 @@ importers: components/monday: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9048,8 +9038,8 @@ importers: components/nyckel: dependencies: '@pipedream/platform': - specifier: ^1.6.2 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9078,8 +9068,8 @@ importers: components/ocrspace: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 mime: specifier: ^4.0.6 version: 4.0.6 @@ -9243,8 +9233,8 @@ importers: components/onlinecheckwriter: dependencies: '@pipedream/platform': - specifier: 3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9252,8 +9242,8 @@ importers: components/onlyoffice_docspace: dependencies: '@pipedream/platform': - specifier: ^3.0.0 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9276,7 +9266,7 @@ importers: dependencies: '@pipedream/platform': specifier: ^3.0.3 - version: 3.0.3 + version: 3.1.0 '@pipedream/types': specifier: ^0.1.4 version: 0.1.6 @@ -9557,8 +9547,8 @@ importers: components/pandadoc: dependencies: '@pipedream/platform': - specifier: ^1.6.0 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9736,8 +9726,8 @@ importers: components/pdf4me: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 components/pdf_api_io: {} @@ -10363,8 +10353,7 @@ importers: specifier: ^1.5.1 version: 1.6.6 - components/postmaster: - specifiers: {} + components/postmaster: {} components/power_automate: {} @@ -12414,8 +12403,8 @@ importers: components/slack: dependencies: '@pipedream/platform': - specifier: ^3.0.0 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 '@slack/web-api': specifier: ^7.9.0 version: 7.9.1 @@ -12881,15 +12870,12 @@ importers: components/stannp: dependencies: + '@pipedream/platform': + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security - url-exist: - specifier: ^3.0.1 - version: 3.0.1(web-streams-polyfill@3.3.3) components/starloop: {} @@ -15163,14 +15149,11 @@ importers: components/zamzar: dependencies: '@pipedream/platform': - specifier: ^1.6.2 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security components/zendesk: dependencies: @@ -15378,17 +15361,14 @@ importers: components/zoho_desk: dependencies: '@pipedream/platform': - specifier: ^1.6.2 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 async-retry: specifier: ^1.3.3 version: 1.3.3 form-data: specifier: ^4.0.0 version: 4.0.1 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security components/zoho_docs: dependencies: @@ -15697,14 +15677,6 @@ importers: specifier: ^6.0.0 version: 6.2.0 - modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/cjs: {} - - modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/esm: {} - - modelcontextprotocol/node_modules2/zod-to-json-schema/dist/cjs: {} - - modelcontextprotocol/node_modules2/zod-to-json-schema/dist/esm: {} - packages/ai: dependencies: '@pipedream/sdk': @@ -23865,10 +23837,6 @@ packages: resolution: {integrity: sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==} engines: {node: '>=10'} - file-type@18.7.0: - resolution: {integrity: sha512-ihHtXRzXEziMrQ56VSgU7wkxh55iNchFkosu7Y9/S+tXHdKyrGjVK0ujbqNnsxzea+78MaLhN6PGmfYSAv1ACw==} - engines: {node: '>=14.16'} - file-type@3.9.0: resolution: {integrity: sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==} engines: {node: '>=0.10.0'} @@ -27557,10 +27525,6 @@ packages: resolution: {integrity: sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==} engines: {node: '>=8'} - peek-readable@5.3.1: - resolution: {integrity: sha512-GVlENSDW6KHaXcd9zkZltB7tCLosKB/4Hg0fqBJkAoBgYG2Tn1xtMgXtSUuMU9AK/gCm/tTdT8mgAeF4YNeeqw==} - engines: {node: '>=14.16'} - pend@1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} @@ -29183,10 +29147,6 @@ packages: resolution: {integrity: sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==} engines: {node: '>=10'} - strtok3@7.1.1: - resolution: {integrity: sha512-mKX8HA/cdBqMKUr0MMZAFssCkIGoZeSCMXgnt79yKxNFguMLVFgRe6wB+fsL0NmoHDbeyZXczy7vEPSoo3rkzg==} - engines: {node: '>=16'} - stubs@3.0.0: resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==} @@ -29512,10 +29472,6 @@ packages: resolution: {integrity: sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==} engines: {node: '>=10'} - token-types@5.0.1: - resolution: {integrity: sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==} - engines: {node: '>=14.16'} - touch@3.1.1: resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==} hasBin: true @@ -41889,12 +41845,6 @@ snapshots: strtok3: 6.3.0 token-types: 4.2.1 - file-type@18.7.0: - dependencies: - readable-web-to-node-stream: 3.0.2 - strtok3: 7.1.1 - token-types: 5.0.1 - file-type@3.9.0: {} file-uri-to-path@1.0.0: {} @@ -46975,8 +46925,6 @@ snapshots: peek-readable@4.1.0: {} - peek-readable@5.3.1: {} - pend@1.2.0: {} performance-now@2.1.0: {} @@ -49317,11 +49265,6 @@ snapshots: '@tokenizer/token': 0.3.0 peek-readable: 4.1.0 - strtok3@7.1.1: - dependencies: - '@tokenizer/token': 0.3.0 - peek-readable: 5.3.1 - stubs@3.0.0: {} style-to-object@0.4.4: @@ -49797,11 +49740,6 @@ snapshots: '@tokenizer/token': 0.3.0 ieee754: 1.2.1 - token-types@5.0.1: - dependencies: - '@tokenizer/token': 0.3.0 - ieee754: 1.2.1 - touch@3.1.1: {} tough-cookie@2.5.0: