From 1aef0404f1f70a423cbd986824fdd5984e7fac58 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 4 Jun 2025 21:07:56 -0300 Subject: [PATCH 01/47] pnpm --- pnpm-lock.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 223aab8e6b264..ea90c8906e2c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15604,14 +15604,6 @@ importers: specifier: ^6.0.0 version: 6.2.0 - modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/cjs: {} - - modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/esm: {} - - modelcontextprotocol/node_modules2/zod-to-json-schema/dist/cjs: {} - - modelcontextprotocol/node_modules2/zod-to-json-schema/dist/esm: {} - packages/ai: dependencies: '@pipedream/sdk': From 3d6ce07945fcc26cb4b638a4f00283866c1d3076 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 4 Jun 2025 21:57:21 -0300 Subject: [PATCH 02/47] Updating _2markdown and mapbox --- components/_2markdown/_2markdown.app.mjs | 4 ++-- .../html-file-to-markdown/html-file-to-markdown.mjs | 8 +++----- .../actions/pdf-to-markdown/pdf-to-markdown.mjs | 10 ++++------ components/_2markdown/package.json | 2 +- .../mapbox/actions/create-tileset/create-tileset.mjs | 10 +++++----- components/mapbox/package.json | 2 +- 6 files changed, 16 insertions(+), 20 deletions(-) diff --git a/components/_2markdown/_2markdown.app.mjs b/components/_2markdown/_2markdown.app.mjs index 5996f3464f0b9..92b9781802daf 100644 --- a/components/_2markdown/_2markdown.app.mjs +++ b/components/_2markdown/_2markdown.app.mjs @@ -6,8 +6,8 @@ export default { propDefinitions: { filePath: { type: "string", - label: "File Path", - description: "The path to an HTML file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "An HTML file. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", }, }, methods: { diff --git a/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs b/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs index c5e8510e1e639..3e80fd0ad3302 100644 --- a/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs +++ b/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs @@ -1,12 +1,12 @@ import _2markdown from "../../_2markdown.app.mjs"; -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "_2markdown-html-file-to-markdown", name: "HTML File to Markdown", description: "Convert an HTML file to Markdown format. [See the documentation](https://2markdown.com/docs#file2md)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { _2markdown, @@ -20,9 +20,7 @@ export default { async run({ $ }) { const form = new FormData(); - form.append("document", fs.createReadStream(this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`)); + form.append("document", await getFileStream(this.filePath)); const response = await this._2markdown.htmlFileToMarkdown({ $, diff --git a/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs b/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs index 597a80fdb6812..0baad9505a16a 100644 --- a/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs +++ b/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs @@ -1,12 +1,12 @@ import _2markdown from "../../_2markdown.app.mjs"; -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "_2markdown-pdf-to-markdown", name: "PDF to Markdown", description: "Convert a PDF document to Markdown format. [See the documentation](https://2markdown.com/docs#pdf2md)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { _2markdown, @@ -15,7 +15,7 @@ export default { _2markdown, "filePath", ], - description: "The path to a PDF file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + description: "A PDF file. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.pdf`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", }, waitForCompletion: { type: "boolean", @@ -27,9 +27,7 @@ export default { async run({ $ }) { const form = new FormData(); - form.append("document", fs.createReadStream(this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`)); + form.append("document", await getFileStream(this.filePath)); let response = await this._2markdown.pdfToMarkdown({ $, diff --git a/components/_2markdown/package.json b/components/_2markdown/package.json index 9df443a7fe99d..23829b114d56c 100644 --- a/components/_2markdown/package.json +++ b/components/_2markdown/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/_2markdown", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream 2markdown Components", "main": "_2markdown.app.mjs", "keywords": [ diff --git a/components/mapbox/actions/create-tileset/create-tileset.mjs b/components/mapbox/actions/create-tileset/create-tileset.mjs index 2cc4423f4172f..3f47e9791adc0 100644 --- a/components/mapbox/actions/create-tileset/create-tileset.mjs +++ b/components/mapbox/actions/create-tileset/create-tileset.mjs @@ -1,12 +1,12 @@ import mapbox from "../../mapbox.app.mjs"; -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "mapbox-create-tileset", name: "Create Tileset", description: "Uploads and creates a new tileset from a data source. [See the documentation](https://docs.mapbox.com/api/maps/mapbox-tiling-service/)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { mapbox, @@ -22,8 +22,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to a tileset source file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "A tileset source file. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", }, recipe: { type: "object", @@ -51,7 +51,7 @@ export default { // Create Tileset Source try { const fileData = new FormData(); - const content = fs.createReadStream(filePath); + const content = await getFileStream(filePath); fileData.append("file", content); await this.mapbox.createTilesetSource({ diff --git a/components/mapbox/package.json b/components/mapbox/package.json index 680b99cc3f4cb..a5d202a96091d 100644 --- a/components/mapbox/package.json +++ b/components/mapbox/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/mapbox", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Mapbox Components", "main": "mapbox.app.mjs", "keywords": [ From 54a0c8d68af1cc5c7c55376a707a60d56a3e07f4 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 4 Jun 2025 22:17:53 -0300 Subject: [PATCH 03/47] Akeneo and LLMWhisperer apps --- .../create-a-new-product-media-file.mjs | 24 ++++----- components/akeneo/package.json | 4 +- .../actions/extract-text/extract-text.mjs | 51 +++++-------------- components/llmwhisperer/package.json | 5 +- 4 files changed, 29 insertions(+), 55 deletions(-) diff --git a/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs b/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs index 7a2715c5ce0c2..874f05a61d332 100644 --- a/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs +++ b/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs @@ -1,13 +1,13 @@ import app from "../../akeneo.app.mjs"; -import utils from "../../common/utils.mjs"; -import { ConfigurationError } from "@pipedream/platform"; +import { + ConfigurationError, getFileStream, +} from "@pipedream/platform"; import FormData from "form-data"; -import fs from "fs"; export default { type: "action", key: "akeneo-create-a-new-product-media-file", - version: "0.0.1", + version: "0.1.0", name: "Create A New Product Media File", description: "Allows you to create a new media file and associate it to an attribute value of a given product or product model. [See the docs](https://api.akeneo.com/api-reference.html#post_media_files)", props: { @@ -32,18 +32,15 @@ export default { }, filename: { type: "string", - label: "File", - description: "The file to be uploaded, please provide a file from `/tmp`. To upload a file to `/tmp` folder, please follow the doc [here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to be uploaded. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", }, }, async run ({ $ }) { if (!this.productId && !this.productModelCode) { throw new ConfigurationError("Either `Product Identifier` or `Product Model Code` should be set!"); } - const path = utils.checkTmp(this.filename); - if (!fs.existsSync(path)) { - throw new ConfigurationError("File does not exist!"); - } + const payload = { attribute: this.mediaFileAttributeCode, scope: null, @@ -57,9 +54,10 @@ export default { payload.code = this.productModelCode; data.append("product_model", JSON.stringify(payload)); } - const file = fs.readFileSync(path); - const fileParts = path.split("/"); - data.append("file", file, fileParts[fileParts.length - 1]); + + const fileStream = await getFileStream(this.filename); + const fileParts = this.filename.split("/"); + data.append("file", fileStream, fileParts[fileParts.length - 1]); const contentLength = data.getLengthSync(); await this.app.createProductMediaFile({ $, diff --git a/components/akeneo/package.json b/components/akeneo/package.json index b53068ab24a0b..35b9c6f20ed10 100644 --- a/components/akeneo/package.json +++ b/components/akeneo/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/akeneo", - "version": "0.0.2", + "version": "0.1.0", "description": "Pipedream Akeneo Components", "main": "akeneo.app.mjs", "keywords": [ @@ -10,7 +10,7 @@ "homepage": "https://pipedream.com/apps/akeneo", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^1.3.0", + "@pipedream/platform": "^3.0.1", "form-data": "^4.0.0" }, "publishConfig": { diff --git a/components/llmwhisperer/actions/extract-text/extract-text.mjs b/components/llmwhisperer/actions/extract-text/extract-text.mjs index 50795dc7e8ce8..cdc528809eafe 100644 --- a/components/llmwhisperer/actions/extract-text/extract-text.mjs +++ b/components/llmwhisperer/actions/extract-text/extract-text.mjs @@ -1,11 +1,11 @@ -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; import app from "../../llmwhisperer.app.mjs"; export default { key: "llmwhisperer-extract-text", name: "Extract Text", description: "Convert your PDF/scanned documents to text format which can be used by LLMs. [See the documentation](https://docs.unstract.com/llm_whisperer/apis/llm_whisperer_text_extraction_api)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { app, @@ -91,40 +91,18 @@ export default { description: "Factor by which a horizontal stretch has to applied. It defaults to `1.0`. A stretch factor of `1.1` would mean at 10% stretch factor applied. Normally this factor need not be adjusted. You might want to use this parameter when multi column layouts back into each other. For example in a two column layout, the two columns get merged into one.", optional: true, }, - urlInPost: { - type: "boolean", - label: "URL In Post", - description: "If set to `true`, the headers will be set to `text/plain`. If set to `false`, the headers will be set to `application/octet-stream`.", - reloadProps: true, - default: true, + data: { + type: "string", + label: "File Path or URL", + description: "The document to process. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", }, }, - additionalProps() { - const { urlInPost } = this; - return { - data: { - type: "string", - label: urlInPost - ? "Document URL" - : "Document Path", - description: urlInPost - ? "The URL of the document to process." - : "Document path of the file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)", - }, - }; - }, methods: { - getHeaders(urlInPost) { - return { - "Content-Type": urlInPost - ? "text/plain" - : "application/octet-stream", - }; - }, - getData(urlInPost, data) { - return urlInPost - ? data - : fs.readFileSync(data); + getHeaders() { + return "application/octet-stream"; + }, + async getData(data) { + return getFileStream(data); }, extractText(args = {}) { return this.app.post({ @@ -138,7 +116,6 @@ export default { extractText, getHeaders, getData, - urlInPost, processingMode, outputMode, pageSeperator, @@ -156,9 +133,9 @@ export default { const response = await extractText({ $, - headers: getHeaders(urlInPost), + headers: getHeaders(), params: { - url_in_post: urlInPost, + url_in_post: false, processing_mode: processingMode, output_mode: outputMode, page_seperator: pageSeperator, @@ -172,7 +149,7 @@ export default { line_splitter_tolerance: lineSplitterTolerance, horizontal_stretch_factor: horizontalStretchFactor, }, - data: getData(urlInPost, data), + data: await getData(data), }); $.export("$summary", "Successfully extracted text from document."); diff --git a/components/llmwhisperer/package.json b/components/llmwhisperer/package.json index 9d9d06ba6ccd0..68766334b83fa 100644 --- a/components/llmwhisperer/package.json +++ b/components/llmwhisperer/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/llmwhisperer", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream LLMWhisperer Components", "main": "llmwhisperer.app.mjs", "keywords": [ @@ -13,7 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.0", - "fs": "^0.0.1-security" + "@pipedream/platform": "^3.1.0" } } From 49b328f409710d003c8e1642fe21611ff4db4ea2 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 4 Jun 2025 22:34:25 -0300 Subject: [PATCH 04/47] pnpm --- pnpm-lock.yaml | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ea90c8906e2c4..4f55baa539674 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -609,8 +609,8 @@ importers: components/akeneo: dependencies: '@pipedream/platform': - specifier: ^1.3.0 - version: 1.6.6 + specifier: ^3.0.1 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -7558,11 +7558,8 @@ importers: components/llmwhisperer: dependencies: '@pipedream/platform': - specifier: ^3.0.0 - version: 3.0.3 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security + specifier: ^3.1.0 + version: 3.1.0 components/lmnt: dependencies: @@ -18722,6 +18719,9 @@ packages: '@pipedream/platform@3.0.3': resolution: {integrity: sha512-7elalas41lnT8i6EAFkqB7fT/+hkLGEQ1njS6A7CVguTrEswaIYk/seKmkfkRY7+O6qncgnXswYIKCBML9Co7w==} + '@pipedream/platform@3.1.0': + resolution: {integrity: sha512-ELmV3fpd9PIUm74/RPaqmA7hJaY4ryIn6PCNT1NP/gnqMrkRNOAr+DTV4xPaJgPiRm74WC5HvrkC8YdRveXzZA==} + '@pipedream/postgresql@2.2.3': resolution: {integrity: sha512-Co9r4UKvvimEPo9T4v+EfVn/Sqqw8+X7PtvsAF7tydq7CHpkmOT9eYAiD0Kuybg5WbuWuqDaXHHmhVtsAxgpBw==} @@ -35373,7 +35373,7 @@ snapshots: '@pipedream/canva@0.2.3': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 transitivePeerDependencies: - debug @@ -35392,7 +35392,7 @@ snapshots: '@pipedream/docusign@0.2.1': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 transitivePeerDependencies: - debug @@ -35442,7 +35442,7 @@ snapshots: '@pipedream/linear_app@0.7.1': dependencies: '@linear/sdk': 13.0.0 - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 transitivePeerDependencies: - debug - encoding @@ -35457,7 +35457,7 @@ snapshots: '@pipedream/monday@0.7.0': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 form-data: 4.0.2 lodash.flatmap: 4.5.0 lodash.map: 4.6.0 @@ -35470,7 +35470,7 @@ snapshots: '@pipedream/notion@0.6.2': dependencies: '@notionhq/client': 2.3.0 - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 '@tryfabric/martian': 1.2.4 lodash-es: 4.17.21 notion-to-md: 3.1.8 @@ -35576,6 +35576,17 @@ snapshots: transitivePeerDependencies: - debug + '@pipedream/platform@3.1.0': + dependencies: + axios: 1.9.0 + fp-ts: 2.16.10 + io-ts: 2.2.22(fp-ts@2.16.10) + mime-types: 3.0.1 + querystring: 0.2.1 + uuid: 11.1.0 + transitivePeerDependencies: + - debug + '@pipedream/postgresql@2.2.3': dependencies: '@pipedream/platform': 2.0.0 @@ -35587,13 +35598,13 @@ snapshots: '@pipedream/quickbooks@0.5.1': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 transitivePeerDependencies: - debug '@pipedream/ramp@0.1.2': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 uuid: 10.0.0 transitivePeerDependencies: - debug @@ -35630,14 +35641,14 @@ snapshots: '@pipedream/sftp@0.4.1': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 ssh2-sftp-client: 11.0.0 transitivePeerDependencies: - debug '@pipedream/shopify@0.7.0': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 async-retry: 1.3.3 bottleneck: 2.19.5 form-data: 4.0.2 From d174051cd00420aa4db9d4ef980905762f5dea0b Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Thu, 5 Jun 2025 22:31:23 -0300 Subject: [PATCH 05/47] add: askyourpdf --- .../add-document-via-file-upload.mjs | 6 +++--- components/askyourpdf/common/utils.mjs | 15 ++++++--------- components/askyourpdf/package.json | 7 +++---- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs b/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs index 4b4b50d7a0631..ddfe1291fd318 100644 --- a/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs +++ b/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs @@ -6,13 +6,13 @@ export default { name: "Add Document Via File Upload", description: "Add a document via file upload. [See the documentation](https://docs.askyourpdf.com/askyourpdf-docs/#2.-adding-document-via-file-upload)", type: "action", - version: "0.0.2", + version: "0.1.0", props: { app, file: { type: "string", - label: "File Path", - description: "File path of a file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)", + label: "File Path or URL", + description: "The file to upload. Provide either a path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", }, }, methods: { diff --git a/components/askyourpdf/common/utils.mjs b/components/askyourpdf/common/utils.mjs index fb5909561429a..049bb0670a17e 100644 --- a/components/askyourpdf/common/utils.mjs +++ b/components/askyourpdf/common/utils.mjs @@ -1,17 +1,14 @@ -import { createReadStream } from "fs"; +import { getFileStream } from "@pipedream/platform"; import FormData from "form-data"; import constants from "./constants.mjs"; -function buildFormData(formData, data, parentKey) { +async function buildFormData(formData, data, parentKey) { if (data && typeof(data) === "object") { - Object.keys(data) - .forEach(async (key) => { - buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); - }); - + for (const key of Object.keys(data)) { + await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); + } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, createReadStream(data)); - + formData.append(parentKey, await getFileStream(data)); } else if (data) { formData.append(parentKey, (data).toString()); } diff --git a/components/askyourpdf/package.json b/components/askyourpdf/package.json index afdaf84cde945..7b8dbdcd072e9 100644 --- a/components/askyourpdf/package.json +++ b/components/askyourpdf/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/askyourpdf", - "version": "0.1.1", + "version": "0.2.0", "description": "Pipedream AskYourPDF Components", "main": "askyourpdf.app.mjs", "keywords": [ @@ -10,9 +10,8 @@ "homepage": "https://pipedream.com/apps/askyourpdf", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^1.6.2", - "form-data": "^4.0.0", - "fs": "^0.0.1-security" + "@pipedream/platform": "^3.1.0", + "form-data": "^4.0.0" }, "publishConfig": { "access": "public" From 9e1147726b2e47bd1d9adbd44da1921fb37a2d2e Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Thu, 5 Jun 2025 22:36:59 -0300 Subject: [PATCH 06/47] add: fileforge --- .../actions/generate-pdf/generate-pdf.mjs | 15 ++++++--------- components/fileforge/common/utils.mjs | 7 ------- components/fileforge/package.json | 4 ++-- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/components/fileforge/actions/generate-pdf/generate-pdf.mjs b/components/fileforge/actions/generate-pdf/generate-pdf.mjs index 9294fc2d2798a..37afc7672ff20 100644 --- a/components/fileforge/actions/generate-pdf/generate-pdf.mjs +++ b/components/fileforge/actions/generate-pdf/generate-pdf.mjs @@ -1,16 +1,13 @@ import FormData from "form-data"; -import fs from "fs"; -import { - checkTmp, - parseObject, -} from "../../common/utils.mjs"; +import { getFileStream } from "@pipedream/platform"; +import { parseObject } from "../../common/utils.mjs"; import fileforge from "../../fileforge.app.mjs"; export default { key: "fileforge-generate-pdf", name: "Generate PDF", description: "Generate a PDF from provided HTML. [See the documentation](https://docs.fileforge.com/api-reference/api-reference/pdf/generate)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { fileforge, @@ -27,8 +24,8 @@ export default { }, files: { type: "string[]", - label: "HTML Files", - description: "The HTML files to convert to PDF. Each file should be a valid path to an HTML file saved to the `/tmp` directory (e.g. `/tmp/image.png`). [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory)..", + label: "File Paths or URLs", + description: "The HTML files to convert to PDF. For each entry, provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, test: { type: "boolean", @@ -66,7 +63,7 @@ export default { const parsedFiles = parseObject(files); for (const file of parsedFiles) { - formData.append("files", fs.createReadStream(checkTmp(file))); + formData.append("files", await getFileStream(file)); } formData.append("options", JSON.stringify({ diff --git a/components/fileforge/common/utils.mjs b/components/fileforge/common/utils.mjs index d1e7ed1a22d98..dcc9cc61f6f41 100644 --- a/components/fileforge/common/utils.mjs +++ b/components/fileforge/common/utils.mjs @@ -22,10 +22,3 @@ export const parseObject = (obj) => { } return obj; }; - -export const checkTmp = (filename) => { - if (!filename.startsWith("/tmp")) { - return `/tmp/${filename}`; - } - return filename; -}; diff --git a/components/fileforge/package.json b/components/fileforge/package.json index da73c8d49d8c2..e70e4d3565bce 100644 --- a/components/fileforge/package.json +++ b/components/fileforge/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/fileforge", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Fileforge Components", "main": "fileforge.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.1", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } From 8a57ff356446837e376dea533088ca3d7cfd6c71 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Thu, 5 Jun 2025 22:52:55 -0300 Subject: [PATCH 07/47] adjusting existing descriptions and platform versions --- components/_2markdown/_2markdown.app.mjs | 2 +- components/_2markdown/package.json | 2 +- .../create-a-new-product-media-file.mjs | 2 +- components/akeneo/package.json | 2 +- .../actions/extract-text/extract-text.mjs | 2 +- .../actions/create-tileset/create-tileset.mjs | 2 +- components/mapbox/package.json | 2 +- pnpm-lock.yaml | 53 ++++++++----------- 8 files changed, 29 insertions(+), 38 deletions(-) diff --git a/components/_2markdown/_2markdown.app.mjs b/components/_2markdown/_2markdown.app.mjs index 92b9781802daf..ccdb8127019fa 100644 --- a/components/_2markdown/_2markdown.app.mjs +++ b/components/_2markdown/_2markdown.app.mjs @@ -7,7 +7,7 @@ export default { filePath: { type: "string", label: "File Path or URL", - description: "An HTML file. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", + description: "An HTML file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, methods: { diff --git a/components/_2markdown/package.json b/components/_2markdown/package.json index 23829b114d56c..7fcf281a32410 100644 --- a/components/_2markdown/package.json +++ b/components/_2markdown/package.json @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.1" } } diff --git a/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs b/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs index 874f05a61d332..c534cfe428023 100644 --- a/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs +++ b/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs @@ -33,7 +33,7 @@ export default { filename: { type: "string", label: "File Path or URL", - description: "The file to be uploaded. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", + description: "The file to be uploaded. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, async run ({ $ }) { diff --git a/components/akeneo/package.json b/components/akeneo/package.json index 35b9c6f20ed10..7c2aa0121119f 100644 --- a/components/akeneo/package.json +++ b/components/akeneo/package.json @@ -10,7 +10,7 @@ "homepage": "https://pipedream.com/apps/akeneo", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^3.0.1", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" }, "publishConfig": { diff --git a/components/llmwhisperer/actions/extract-text/extract-text.mjs b/components/llmwhisperer/actions/extract-text/extract-text.mjs index cdc528809eafe..1b16c0efb32b6 100644 --- a/components/llmwhisperer/actions/extract-text/extract-text.mjs +++ b/components/llmwhisperer/actions/extract-text/extract-text.mjs @@ -94,7 +94,7 @@ export default { data: { type: "string", label: "File Path or URL", - description: "The document to process. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", + description: "The document to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, methods: { diff --git a/components/mapbox/actions/create-tileset/create-tileset.mjs b/components/mapbox/actions/create-tileset/create-tileset.mjs index 3f47e9791adc0..e403ed291b2f9 100644 --- a/components/mapbox/actions/create-tileset/create-tileset.mjs +++ b/components/mapbox/actions/create-tileset/create-tileset.mjs @@ -23,7 +23,7 @@ export default { filePath: { type: "string", label: "File Path or URL", - description: "A tileset source file. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", + description: "A tileset source file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, recipe: { type: "object", diff --git a/components/mapbox/package.json b/components/mapbox/package.json index a5d202a96091d..2265772e4163e 100644 --- a/components/mapbox/package.json +++ b/components/mapbox/package.json @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.1" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4f55baa539674..6aa73af3c87f9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -154,8 +154,8 @@ importers: components/_2markdown: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.1 version: 4.0.1 @@ -609,7 +609,7 @@ importers: components/akeneo: dependencies: '@pipedream/platform': - specifier: ^3.0.1 + specifier: ^3.1.0 version: 3.1.0 form-data: specifier: ^4.0.0 @@ -1010,14 +1010,11 @@ importers: components/askyourpdf: dependencies: '@pipedream/platform': - specifier: ^1.6.2 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security components/assembla: dependencies: @@ -1738,8 +1735,8 @@ importers: components/box: dependencies: '@pipedream/platform': - specifier: ^1.5.1 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -4545,8 +4542,8 @@ importers: components/fileforge: dependencies: '@pipedream/platform': - specifier: ^3.0.1 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -6484,8 +6481,8 @@ importers: components/imagga: dependencies: '@pipedream/platform': - specifier: ^1.5.1 - version: 1.6.6 + specifier: ^3.0.3 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -7897,8 +7894,8 @@ importers: components/mapbox: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.1 version: 4.0.1 @@ -9188,8 +9185,8 @@ importers: components/onlyoffice_docspace: dependencies: '@pipedream/platform': - specifier: ^3.0.0 - version: 3.0.3 + specifier: ^3.0.3 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -12789,12 +12786,12 @@ importers: components/stannp: dependencies: + '@pipedream/platform': + specifier: ^3.0.3 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security url-exist: specifier: ^3.0.1 version: 3.0.1(web-streams-polyfill@3.3.3) @@ -15067,14 +15064,11 @@ importers: components/zamzar: dependencies: '@pipedream/platform': - specifier: ^1.6.2 - version: 1.6.6 + specifier: ^3.0.3 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security components/zendesk: dependencies: @@ -15282,17 +15276,14 @@ importers: components/zoho_desk: dependencies: '@pipedream/platform': - specifier: ^1.6.2 - version: 1.6.6 + specifier: ^3.0.3 + version: 3.1.0 async-retry: specifier: ^1.3.3 version: 1.3.3 form-data: specifier: ^4.0.0 version: 4.0.1 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security components/zoho_docs: dependencies: From 9bf8aad8b694dc60388b248dcfa0d78d514ddd95 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Thu, 5 Jun 2025 23:05:54 -0300 Subject: [PATCH 08/47] Description updates --- components/_2markdown/_2markdown.app.mjs | 2 +- .../_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs | 2 +- .../add-document-via-file-upload.mjs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/_2markdown/_2markdown.app.mjs b/components/_2markdown/_2markdown.app.mjs index ccdb8127019fa..4fa47f6e6f432 100644 --- a/components/_2markdown/_2markdown.app.mjs +++ b/components/_2markdown/_2markdown.app.mjs @@ -7,7 +7,7 @@ export default { filePath: { type: "string", label: "File Path or URL", - description: "An HTML file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", + description: "An HTML file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.html`)", }, }, methods: { diff --git a/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs b/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs index 0baad9505a16a..306196acab213 100644 --- a/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs +++ b/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs @@ -15,7 +15,7 @@ export default { _2markdown, "filePath", ], - description: "A PDF file. Provide either the path to a file in the `/tmp` directory (e.g. `/tmp/myFile.pdf`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", + description: "A PDF file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, waitForCompletion: { type: "boolean", diff --git a/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs b/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs index ddfe1291fd318..e93de416ceb2e 100644 --- a/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs +++ b/components/askyourpdf/actions/add-document-via-file-upload/add-document-via-file-upload.mjs @@ -12,7 +12,7 @@ export default { file: { type: "string", label: "File Path or URL", - description: "The file to upload. Provide either a path to a file in the `/tmp` directory (e.g. `/tmp/myFile.ext`) or a file URL. [See the documentation on working with files](https://pipedream.com/docs/workflows/building-workflows/code/nodejs/working-with-files/)", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, methods: { From c10404564c61d1cdecb12bbb5b958446969e40b2 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Thu, 5 Jun 2025 23:40:13 -0300 Subject: [PATCH 09/47] add: microsoft_onedrive, mistral_ai, onlyoffice_docspace --- .../actions/upload-file/upload-file.mjs | 15 ++++++++------- components/microsoft_onedrive/package.json | 4 ++-- .../actions/upload-file/upload-file.mjs | 19 +++++-------------- components/mistral_ai/package.json | 4 ++-- .../actions/upload-file/upload-file.mjs | 6 +++--- .../onlyoffice_docspace/common/utils.mjs | 15 ++++++--------- components/onlyoffice_docspace/package.json | 4 ++-- pnpm-lock.yaml | 12 ++++++------ 8 files changed, 34 insertions(+), 45 deletions(-) diff --git a/components/microsoft_onedrive/actions/upload-file/upload-file.mjs b/components/microsoft_onedrive/actions/upload-file/upload-file.mjs index 23c25ec949cd8..968a04953b6f1 100644 --- a/components/microsoft_onedrive/actions/upload-file/upload-file.mjs +++ b/components/microsoft_onedrive/actions/upload-file/upload-file.mjs @@ -1,13 +1,14 @@ import onedrive from "../../microsoft_onedrive.app.mjs"; -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; +import { + ConfigurationError, getFileStream, +} from "@pipedream/platform"; import { fileTypeFromStream } from "file-type"; export default { name: "Upload File", description: "Upload a file to OneDrive. [See the documentation](https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online)", key: "microsoft_onedrive-upload-file", - version: "0.1.2", + version: "0.2.0", type: "action", props: { onedrive, @@ -21,8 +22,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to the file saved to the `/tmp` directory (e.g. `/tmp/image.png`). [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory).", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, filename: { type: "string", @@ -39,7 +40,7 @@ export default { throw new ConfigurationError("You must specify the **Upload Folder ID**."); } - let stream = fs.createReadStream(filePath); + let stream = await getFileStream(filePath); let name = filename; if (!filename.includes(".")) { @@ -48,7 +49,7 @@ export default { name = `${filename}.${extension}`; stream.destroy(); - stream = fs.createReadStream(filePath); + stream = await getFileStream(filePath); } const response = await this.onedrive.uploadFile({ diff --git a/components/microsoft_onedrive/package.json b/components/microsoft_onedrive/package.json index 0e71d55362c72..a914b82e5a62b 100644 --- a/components/microsoft_onedrive/package.json +++ b/components/microsoft_onedrive/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/microsoft_onedrive", - "version": "1.6.2", + "version": "1.7.0", "description": "Pipedream Microsoft OneDrive components", "main": "microsoft_onedrive.app.mjs", "homepage": "https://pipedream.com/apps/microsoft-onedrive", @@ -10,7 +10,7 @@ }, "dependencies": { "@microsoft/microsoft-graph-client": "^3.0.1", - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "bottleneck": "^2.19.5", "file-type": "^18.7.0", "isomorphic-fetch": "^3.0.0", diff --git a/components/mistral_ai/actions/upload-file/upload-file.mjs b/components/mistral_ai/actions/upload-file/upload-file.mjs index 6f6583088e4df..71eaea3e90e0a 100644 --- a/components/mistral_ai/actions/upload-file/upload-file.mjs +++ b/components/mistral_ai/actions/upload-file/upload-file.mjs @@ -1,20 +1,19 @@ import mistralAI from "../../mistral_ai.app.mjs"; -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "mistral_ai-upload-file", name: "Upload File", description: "Upload a file that can be used across various endpoints. [See the Documentation](https://docs.mistral.ai/api/#tag/files/operation/files_api_routes_upload_file)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { mistralAI, filePath: { type: "string", - label: "File Path", - description: "The path to a file in the `/tmp` directory. The size of individual files can be a maximum of 512 MB. The Fine-tuning API only supports .jsonl files. [See the Pipedream documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, purpose: { type: "string", @@ -29,15 +28,7 @@ export default { }, }, async run({ $ }) { - const filePath = this.filePath.startsWith("/tmp/") - ? this.filePath - : `/tmp/${this.filePath}`; - - if (!fs.existsSync(filePath)) { - throw new ConfigurationError(`File \`${filePath}\` not found`); - } - - const fileContent = fs.createReadStream(filePath); + const fileContent = await getFileStream(this.filePath); const form = new FormData(); form.append("file", fileContent); if (this.purpose) { diff --git a/components/mistral_ai/package.json b/components/mistral_ai/package.json index 091e79e2483ca..aa80ae437bb0e 100644 --- a/components/mistral_ai/package.json +++ b/components/mistral_ai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/mistral_ai", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Mistral AI Components", "main": "mistral_ai.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.2" } } diff --git a/components/onlyoffice_docspace/actions/upload-file/upload-file.mjs b/components/onlyoffice_docspace/actions/upload-file/upload-file.mjs index e1dd3512f196f..4851c15d13f3b 100644 --- a/components/onlyoffice_docspace/actions/upload-file/upload-file.mjs +++ b/components/onlyoffice_docspace/actions/upload-file/upload-file.mjs @@ -5,7 +5,7 @@ export default { key: "onlyoffice_docspace-upload-file", name: "Upload File", description: "Uploads a file to the specified room. [See the documentation](https://api.onlyoffice.com/docspace/method/files/post/api/2.0/files/%7bfolderid%7d/upload)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { app, @@ -19,8 +19,8 @@ export default { }, file: { type: "string", - label: "File", - description: "File path of a file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, methods: { diff --git a/components/onlyoffice_docspace/common/utils.mjs b/components/onlyoffice_docspace/common/utils.mjs index a4710a240ed9f..800a52c3ae6c4 100644 --- a/components/onlyoffice_docspace/common/utils.mjs +++ b/components/onlyoffice_docspace/common/utils.mjs @@ -1,17 +1,14 @@ -import { createReadStream } from "fs"; +import { getFileStream } from "@pipedream/platform"; import FormData from "form-data"; import constants from "./constants.mjs"; -function buildFormData(formData, data, parentKey) { +async function buildFormData(formData, data, parentKey) { if (data && typeof(data) === "object") { - Object.keys(data) - .forEach((key) => { - buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); - }); - + for (const key of Object.keys(data)) { + await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); + } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, createReadStream(data)); - + formData.append(parentKey, await getFileStream(data)); } else if (data) { formData.append(parentKey, (data).toString()); } diff --git a/components/onlyoffice_docspace/package.json b/components/onlyoffice_docspace/package.json index cab0756882afd..a5951e49fb3c8 100644 --- a/components/onlyoffice_docspace/package.json +++ b/components/onlyoffice_docspace/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/onlyoffice_docspace", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream ONLYOFFICE DocSpace Components", "main": "onlyoffice_docspace.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.0", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6aa73af3c87f9..3d025a579a4fc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6481,7 +6481,7 @@ importers: components/imagga: dependencies: '@pipedream/platform': - specifier: ^3.0.3 + specifier: ^3.1.0 version: 3.1.0 form-data: specifier: ^4.0.0 @@ -8189,8 +8189,8 @@ importers: specifier: ^3.0.1 version: 3.0.7 '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 bottleneck: specifier: ^2.19.5 version: 2.19.5 @@ -8318,8 +8318,8 @@ importers: components/mistral_ai: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.2 version: 4.0.2 @@ -9185,7 +9185,7 @@ importers: components/onlyoffice_docspace: dependencies: '@pipedream/platform': - specifier: ^3.0.3 + specifier: ^3.1.0 version: 3.1.0 form-data: specifier: ^4.0.0 From 5624c249f0e86c9a9a1c65acc82df476c60210bb Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Thu, 5 Jun 2025 23:58:51 -0300 Subject: [PATCH 10/47] add: slack --- .../slack/actions/upload-file/upload-file.mjs | 15 +++++++-------- components/slack/package.json | 4 ++-- components/slack/slack.app.mjs | 4 ++-- pnpm-lock.yaml | 4 ++-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/components/slack/actions/upload-file/upload-file.mjs b/components/slack/actions/upload-file/upload-file.mjs index 5489b4f2899b2..3aea48d309fc9 100644 --- a/components/slack/actions/upload-file/upload-file.mjs +++ b/components/slack/actions/upload-file/upload-file.mjs @@ -1,7 +1,6 @@ import { - ConfigurationError, axios, + ConfigurationError, axios, getFileStreamAndMetadata, } from "@pipedream/platform"; -import fs from "fs"; import FormData from "form-data"; import slack from "../../slack.app.mjs"; @@ -9,7 +8,7 @@ export default { key: "slack-upload-file", name: "Upload File", description: "Upload a file. [See the documentation](https://api.slack.com/messaging/files#uploading_files)", - version: "0.0.27", + version: "0.1.0", type: "action", props: { slack, @@ -35,16 +34,16 @@ export default { }, }, async run({ $ }) { - if (!fs.existsSync(this.content)) { - throw new ConfigurationError(`\`${this.content}\` not found, a valid \`/tmp\` path is needed`); - } + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.content); const filename = this.content.split("/").pop(); // Get an upload URL from Slack const getUploadUrlResponse = await this.slack.getUploadUrl({ filename, - length: fs.statSync(this.content).size, + length: metadata.size, }); if (!getUploadUrlResponse.ok) { @@ -57,7 +56,7 @@ export default { // Upload the file to the provided URL const formData = new FormData(); - formData.append("file", fs.createReadStream(this.content)); + formData.append("file", stream); formData.append("filename", filename); await axios($, { diff --git a/components/slack/package.json b/components/slack/package.json index 8860416ac1880..a17294f9bd59a 100644 --- a/components/slack/package.json +++ b/components/slack/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/slack", - "version": "0.9.5", + "version": "0.10.0", "description": "Pipedream Slack Components", "main": "slack.app.mjs", "keywords": [ @@ -14,7 +14,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.0", + "@pipedream/platform": "^3.1.0", "@slack/web-api": "^7.9.0", "async-retry": "^1.3.3", "lodash": "^4.17.21" diff --git a/components/slack/slack.app.mjs b/components/slack/slack.app.mjs index 81b560452a7ac..401d853f0f6cb 100644 --- a/components/slack/slack.app.mjs +++ b/components/slack/slack.app.mjs @@ -356,8 +356,8 @@ export default { }, }, content: { - label: "File Path", - description: "Full path to the file in `/tmp/` directory. E.g. `/tmp/cute_cat.jpg`", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", type: "string", }, link_names: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3d025a579a4fc..03954c68a55db 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12319,8 +12319,8 @@ importers: components/slack: dependencies: '@pipedream/platform': - specifier: ^3.0.0 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 '@slack/web-api': specifier: ^7.9.0 version: 7.9.1 From fe3bb7ea8039741a7d3eb4487f75363255e3678b Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Fri, 6 Jun 2025 00:15:22 -0300 Subject: [PATCH 11/47] add: stannp, zamzar, zoho_desk --- .../actions/create-campaign/create-campaign.mjs | 14 +++++++------- components/stannp/common/utils.mjs | 11 ++--------- components/stannp/package.json | 4 ++-- .../start-job-from-file/start-job-from-file.mjs | 2 +- components/zamzar/common/utils.mjs | 15 ++++++--------- components/zamzar/package.json | 7 +++---- components/zamzar/zamzar.app.mjs | 4 ++-- .../add-ticket-attachment.mjs | 6 +++--- components/zoho_desk/common/utils.mjs | 4 ++-- components/zoho_desk/package.json | 7 +++---- pnpm-lock.yaml | 6 +++--- 11 files changed, 34 insertions(+), 46 deletions(-) diff --git a/components/stannp/actions/create-campaign/create-campaign.mjs b/components/stannp/actions/create-campaign/create-campaign.mjs index a09263de71159..18b69b58f4d56 100644 --- a/components/stannp/actions/create-campaign/create-campaign.mjs +++ b/components/stannp/actions/create-campaign/create-campaign.mjs @@ -9,7 +9,7 @@ export default { key: "stannp-create-campaign", name: "Create a New Campaign", description: "Create a new campaign in Stannp. [See the documentation](https://www.stannp.com/us/direct-mail-api/campaigns)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { stannp, @@ -58,22 +58,22 @@ export default { }, file: { type: "string", - label: "File", - description: "A single or multi-page PDF file to use as the design artwork. can be a URL or a path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp). Make sure the image is in the correct format.", + label: "PDF File Path or URL", + description: "A PDF file to use as the design artwork. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", optional: true, reloadProps: true, }, front: { type: "string", - label: "Front", - description: "A PDF or JPG file to use as the front image. Can be a URL or a path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp). Make sure the image is in the correct format.", + label: "Front Image Path or URL", + description: "A PDF or JPG file to use as the front image. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", optional: true, reloadProps: true, }, back: { type: "string", - label: "Back", - description: "A PDF or JPG file to use as the back image. Can be a URL or a path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp). Make sure the image is in the correct format.", + label: "Back Image Path or URL", + description: "A PDF or JPG file to use as the back image. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", optional: true, reloadProps: true, }, diff --git a/components/stannp/common/utils.mjs b/components/stannp/common/utils.mjs index 39c2e4c8e4409..5855330e494a8 100644 --- a/components/stannp/common/utils.mjs +++ b/components/stannp/common/utils.mjs @@ -1,5 +1,5 @@ -import fs from "fs"; import urlExists from "url-exist"; +import { getFileStream } from "@pipedream/platform"; export const parseObject = (obj) => { if (Array.isArray(obj)) { @@ -24,13 +24,6 @@ export const checkFile = async (url) => { if (await urlExists(url)) { return url; } else { - return fs.createReadStream(checkTmp(url)); + return getFileStream(url); } }; - -const checkTmp = (filename) => { - if (!filename.startsWith("/tmp")) { - return `/tmp/${filename}`; - } - return filename; -}; diff --git a/components/stannp/package.json b/components/stannp/package.json index 729a07205bde6..98a651b14f424 100644 --- a/components/stannp/package.json +++ b/components/stannp/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/stannp", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Stannp Components", "main": "stannp.app.mjs", "keywords": [ @@ -13,8 +13,8 @@ "access": "public" }, "dependencies": { + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0", - "fs": "^0.0.1-security", "url-exist": "^3.0.1" } } diff --git a/components/zamzar/actions/start-job-from-file/start-job-from-file.mjs b/components/zamzar/actions/start-job-from-file/start-job-from-file.mjs index 9c8a66a3cd67d..80361b1405620 100644 --- a/components/zamzar/actions/start-job-from-file/start-job-from-file.mjs +++ b/components/zamzar/actions/start-job-from-file/start-job-from-file.mjs @@ -5,7 +5,7 @@ export default { key: "zamzar-start-job-from-file", name: "Start Job From File", description: "Starts a conversion job and upload a source file in a single request. [See the documentation](https://developers.zamzar.com/docs)", - version: "0.0.2", + version: "0.1.0", type: "action", props: { app, diff --git a/components/zamzar/common/utils.mjs b/components/zamzar/common/utils.mjs index c812fe7712329..a59db04ebe003 100644 --- a/components/zamzar/common/utils.mjs +++ b/components/zamzar/common/utils.mjs @@ -1,17 +1,14 @@ -import { createReadStream } from "fs"; +import { getFileStream } from "@pipedream/platform"; import FormData from "form-data"; import constants from "./constants.mjs"; -function buildFormData(formData, data, parentKey) { +async function buildFormData(formData, data, parentKey) { if (data && typeof(data) === "object") { - Object.keys(data) - .forEach(async (key) => { - buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); - }); - + for (const key of Object.keys(data)) { + await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); + } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, createReadStream(data)); - + formData.append(parentKey, await getFileStream(data)); } else if (data) { formData.append(parentKey, (data).toString()); } diff --git a/components/zamzar/package.json b/components/zamzar/package.json index 1422f5753baed..35fab82135a96 100644 --- a/components/zamzar/package.json +++ b/components/zamzar/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/zamzar", - "version": "0.1.1", + "version": "0.2.0", "description": "Pipedream Zamzar Components", "main": "zamzar.app.mjs", "keywords": [ @@ -13,8 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.6.2", - "form-data": "^4.0.0", - "fs": "^0.0.1-security" + "@pipedream/platform": "^3.1.0", + "form-data": "^4.0.0" } } diff --git a/components/zamzar/zamzar.app.mjs b/components/zamzar/zamzar.app.mjs index 7089d2c43c60a..f9b57c16de243 100644 --- a/components/zamzar/zamzar.app.mjs +++ b/components/zamzar/zamzar.app.mjs @@ -8,8 +8,8 @@ export default { propDefinitions: { sourceFile: { type: "string", - label: "Source File", - description: "The path to the file saved to the `/tmp` directory (e.g. `/tmp/image.png`). [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory).", + label: "Source File Path or URL", + description: "The file to convert. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, targetFormat: { type: "string", diff --git a/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs b/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs index b20ee80d44a3f..8db712defab74 100644 --- a/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs +++ b/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs @@ -6,7 +6,7 @@ export default { name: "Add Ticket Attachment", description: "Attaches a file to a ticket. [See the docs here](https://desk.zoho.com/DeskAPIDocument#TicketAttachments#TicketAttachments_CreateTicketattachment)", type: "action", - version: "0.0.2", + version: "0.1.0", props: { zohoDesk, orgId: { @@ -32,8 +32,8 @@ export default { }, file: { type: "string", - label: "File", - description: "File path of a file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)", + label: "File Path or URL", + description: "The file to attach. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, async run({ $ }) { diff --git a/components/zoho_desk/common/utils.mjs b/components/zoho_desk/common/utils.mjs index 21b502910e075..57fcc7fe18db5 100644 --- a/components/zoho_desk/common/utils.mjs +++ b/components/zoho_desk/common/utils.mjs @@ -1,4 +1,4 @@ -import { createReadStream } from "fs"; +import { getFileStream } from "@pipedream/platform"; import FormData from "form-data"; import retry from "async-retry"; import constants from "./constants.mjs"; @@ -19,7 +19,7 @@ function buildFormData(formData, data, parentKey) { }); } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, createReadStream(data)); + formData.append(parentKey, getFileStream(data)); } else if (data) { formData.append(parentKey, (data).toString()); diff --git a/components/zoho_desk/package.json b/components/zoho_desk/package.json index 9f0e0ff2849d1..9dee94e481eb8 100644 --- a/components/zoho_desk/package.json +++ b/components/zoho_desk/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/zoho_desk", - "version": "0.0.4", + "version": "0.1.0", "description": "Pipedream Zoho_desk Components", "main": "zoho_desk.app.mjs", "keywords": [ @@ -10,10 +10,9 @@ "homepage": "https://pipedream.com/apps/zoho_desk", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^1.6.2", + "@pipedream/platform": "^3.1.0", "async-retry": "^1.3.3", - "form-data": "^4.0.0", - "fs": "^0.0.1-security" + "form-data": "^4.0.0" }, "publishConfig": { "access": "public" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 03954c68a55db..314ba145de131 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12787,7 +12787,7 @@ importers: components/stannp: dependencies: '@pipedream/platform': - specifier: ^3.0.3 + specifier: ^3.1.0 version: 3.1.0 form-data: specifier: ^4.0.0 @@ -15064,7 +15064,7 @@ importers: components/zamzar: dependencies: '@pipedream/platform': - specifier: ^3.0.3 + specifier: ^3.1.0 version: 3.1.0 form-data: specifier: ^4.0.0 @@ -15276,7 +15276,7 @@ importers: components/zoho_desk: dependencies: '@pipedream/platform': - specifier: ^3.0.3 + specifier: ^3.1.0 version: 3.1.0 async-retry: specifier: ^1.3.3 From fed67fa2b6863de3cc5e2c7049ae24dc09e754a4 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Fri, 6 Jun 2025 03:53:13 -0300 Subject: [PATCH 12/47] add box upload-file --- .../upload-file-version.mjs | 2 +- .../box/actions/upload-file/upload-file.mjs | 2 +- components/box/box.app.mjs | 4 +- components/box/common/common-file-upload.mjs | 10 ++--- components/box/common/utils.mjs | 41 ++++++++----------- components/box/package.json | 4 +- pnpm-lock.yaml | 4 +- 7 files changed, 28 insertions(+), 39 deletions(-) diff --git a/components/box/actions/upload-file-version/upload-file-version.mjs b/components/box/actions/upload-file-version/upload-file-version.mjs index 48a9ed40212a6..a94d96124d937 100644 --- a/components/box/actions/upload-file-version/upload-file-version.mjs +++ b/components/box/actions/upload-file-version/upload-file-version.mjs @@ -5,7 +5,7 @@ export default { name: "Upload File Version", description: "Update a file's content. [See the documentation](https://developer.box.com/reference/post-files-id-content/).", key: "box-upload-file-version", - version: "0.0.2", + version: "0.1.0", type: "action", props: { app, diff --git a/components/box/actions/upload-file/upload-file.mjs b/components/box/actions/upload-file/upload-file.mjs index 6e9f42e3ec6b8..46a3ae0618330 100644 --- a/components/box/actions/upload-file/upload-file.mjs +++ b/components/box/actions/upload-file/upload-file.mjs @@ -5,7 +5,7 @@ export default { name: "Upload a File", description: "Uploads a small file to Box. [See the documentation](https://developer.box.com/reference/post-files-content/).", key: "box-upload-file", - version: "0.0.4", + version: "0.1.0", type: "action", props: { app, diff --git a/components/box/box.app.mjs b/components/box/box.app.mjs index f7a2cbd47cb44..84302cb68efa6 100644 --- a/components/box/box.app.mjs +++ b/components/box/box.app.mjs @@ -13,8 +13,8 @@ export default { }, file: { type: "string", - label: "File", - description: "The file to upload to Box, please provide a file from `/tmp`. To upload a file to `/tmp` folder, please follow the doc [here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, createdAt: { type: "string", diff --git a/components/box/common/common-file-upload.mjs b/components/box/common/common-file-upload.mjs index 8bcc6f70af13e..d24b687206691 100644 --- a/components/box/common/common-file-upload.mjs +++ b/components/box/common/common-file-upload.mjs @@ -1,4 +1,3 @@ -import { ConfigurationError } from "@pipedream/platform"; import FormData from "form-data"; import utils from "./utils.mjs"; @@ -9,12 +8,9 @@ export function getFileUploadBody({ fileName, parentId, }) { - const fileValidation = utils.isValidFile(file); - if (!fileValidation) { - throw new ConfigurationError("`file` must be a valid file path!"); - } - const fileMeta = utils.getFileMeta(fileValidation); - const fileContent = utils.getFileStream(fileValidation); + const { + fileMeta, fileContent, + } = utils.getFileData(file); const attributes = fileMeta.attributes; if (createdAt && utils.checkRFC3339(createdAt)) { attributes.content_created_at = createdAt; diff --git a/components/box/common/utils.mjs b/components/box/common/utils.mjs index 3bc4affe137d8..2897f8d946c48 100644 --- a/components/box/common/utils.mjs +++ b/components/box/common/utils.mjs @@ -1,34 +1,27 @@ -import fs from "fs"; import path from "path"; import constants from "./constants.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { - isValidFile(filePath) { - const filePathWithTmp = `/tmp/${filePath}`; - if (fs.existsSync(filePathWithTmp)) { - return filePathWithTmp; - } else if (fs.existsSync(filePath)) { - return filePath; - } - return false; - }, - getFileStream(filePath) { - return fs.createReadStream(filePath); - }, - getFileMeta(filePath) { - const stats = fs.statSync(filePath); + async getFileData(filePath) { + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); return { - attributes: { - content_created_at: new Date(stats.ctimeMs).toISOString() - .split(".")[0] + "Z", - content_modified_at: new Date(stats.mtimeMs).toISOString() - .split(".")[0] + "Z", - name: path.basename(filePath), - parent: { - id: 0, + fileContent: stream, + fileMeta: { + attributes: { + content_created_at: new Date(metadata.lastModified).toISOString() + .split(".")[0] + "Z", + content_modified_at: new Date(metadata.lastModified).toISOString() + .split(".")[0] + "Z", + name: path.basename(filePath), + parent: { + id: 0, + }, }, + size: metadata.size, }, - size: stats.size, }; }, checkRFC3339(dateTimeStr) { diff --git a/components/box/package.json b/components/box/package.json index 40752ed28373d..67da5b3df1053 100644 --- a/components/box/package.json +++ b/components/box/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/box", - "version": "0.3.0", + "version": "0.4.0", "description": "Pipedream Box Components", "main": "box.app.mjs", "keywords": [ @@ -10,7 +10,7 @@ "homepage": "https://pipedream.com/apps/box", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^1.5.1", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0", "path": "^0.12.7", "stream": "^0.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 314ba145de131..024e5d414ab3f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5518,8 +5518,8 @@ importers: specifier: ^2.3.0 version: 2.4.0 '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 cron-parser: specifier: ^4.9.0 version: 4.9.0 From b560b057e1cc81d5ffd2688d086b4000b7da8fd0 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Fri, 6 Jun 2025 16:28:52 -0300 Subject: [PATCH 13/47] add: monday, nyckel --- .../update-column-values.mjs | 13 ++++----- components/monday/package.json | 4 +-- .../actions/classify-image/classify-image.mjs | 2 +- .../extract-text-from-image.mjs | 2 +- components/nyckel/common/common-image.mjs | 11 +++----- components/nyckel/nyckel.app.mjs | 4 +-- components/nyckel/package.json | 4 +-- pnpm-lock.yaml | 28 +++++++++---------- 8 files changed, 32 insertions(+), 36 deletions(-) diff --git a/components/monday/actions/update-column-values/update-column-values.mjs b/components/monday/actions/update-column-values/update-column-values.mjs index 8c725ebc8f87c..35b46b6452501 100644 --- a/components/monday/actions/update-column-values/update-column-values.mjs +++ b/components/monday/actions/update-column-values/update-column-values.mjs @@ -1,6 +1,7 @@ import common from "../common/column-values.mjs"; -import { axios } from "@pipedream/platform"; -import fs from "fs"; +import { + axios, getFileStream, +} from "@pipedream/platform"; import FormData from "form-data"; import { getColumnOptions } from "../../common/utils.mjs"; @@ -9,7 +10,7 @@ export default { key: "monday-update-column-values", name: "Update Column Values", description: "Update multiple column values of an item. [See the documentation](https://developer.monday.com/api-reference/reference/columns#change-multiple-column-values)", - version: "0.1.0", + version: "0.2.0", type: "action", props: { ...common.props, @@ -50,7 +51,7 @@ export default { options: getColumnOptions(columns, id), }; if (column.type === "file") { - props[column.id].description += ". The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)"; + props[column.id].description += ". Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)"; } } } @@ -62,9 +63,7 @@ export default { $, itemId, column, filePath, }) { const query = `mutation ($file: File!) { add_file_to_column (file: $file, item_id: ${itemId}, column_id: "${column.id}") { id } }`; - const content = fs.createReadStream(filePath.includes("tmp/") - ? filePath - : `/tmp/${filePath}`); + const content = await getFileStream(filePath); const formData = new FormData(); formData.append("query", query); diff --git a/components/monday/package.json b/components/monday/package.json index ce48c1bb44f7c..68d8e5edc3384 100644 --- a/components/monday/package.json +++ b/components/monday/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/monday", - "version": "0.7.0", + "version": "0.8.0", "description": "Pipedream Monday Components", "main": "monday.app.mjs", "keywords": [ @@ -14,7 +14,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0", "lodash.flatmap": "^4.5.0", "lodash.map": "^4.6.0", diff --git a/components/nyckel/actions/classify-image/classify-image.mjs b/components/nyckel/actions/classify-image/classify-image.mjs index c7b3f712e8657..f59687e30014a 100644 --- a/components/nyckel/actions/classify-image/classify-image.mjs +++ b/components/nyckel/actions/classify-image/classify-image.mjs @@ -6,7 +6,7 @@ export default { key: "nyckel-classify-image", name: "Classify Image", description: "Classifies image data based on pre-trained classifiers in Nyckel. [See the documentation](https://www.nyckel.com/docs#invoke-image)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { nyckel, diff --git a/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs b/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs index 289b0514f4103..bad8921c64745 100644 --- a/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs +++ b/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs @@ -6,7 +6,7 @@ export default { key: "nyckel-extract-text-from-image", name: "Extract Text from Image", description: "Extracts text from an image URL. [See the documentation](https://www.nyckel.com/docs#ocr-image)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { nyckel, diff --git a/components/nyckel/common/common-image.mjs b/components/nyckel/common/common-image.mjs index a18f1fad59693..8d4ad41a77860 100644 --- a/components/nyckel/common/common-image.mjs +++ b/components/nyckel/common/common-image.mjs @@ -1,6 +1,6 @@ import nyckel from "../nyckel.app.mjs"; import FormData from "form-data"; -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; export default { props: { @@ -12,7 +12,7 @@ export default { }, }, methods: { - getImageData() { + async getImageData() { const { imageOrUrl } = this; const isUrl = imageOrUrl.startsWith("http"); if (isUrl) { @@ -27,11 +27,8 @@ export default { } const data = new FormData(); - data.append("data", fs.createReadStream( - imageOrUrl.includes("tmp/") - ? imageOrUrl - : `/tmp/${imageOrUrl}`, - )); + const stream = await getFileStream(imageOrUrl); + data.append("data", stream); return { data, diff --git a/components/nyckel/nyckel.app.mjs b/components/nyckel/nyckel.app.mjs index 23dedcf06617d..064d4570a9741 100644 --- a/components/nyckel/nyckel.app.mjs +++ b/components/nyckel/nyckel.app.mjs @@ -29,8 +29,8 @@ export default { }, imageOrUrl: { type: "string", - label: "Image Path or URL", - description: "The path to an image file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp). Alternatively, you can pass the direct URL to a file.", + label: "File Path or URL", + description: "An image file. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", }, includeRegions: { type: "boolean", diff --git a/components/nyckel/package.json b/components/nyckel/package.json index 41b880a1a4950..e1301209310e6 100644 --- a/components/nyckel/package.json +++ b/components/nyckel/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/nyckel", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Nyckel Components", "main": "nyckel.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.6.2", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 024e5d414ab3f..8bb60e6a8d3e6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8399,8 +8399,8 @@ importers: components/monday: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -8983,8 +8983,8 @@ importers: components/nyckel: dependencies: '@pipedream/platform': - specifier: ^1.6.2 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9013,8 +9013,8 @@ importers: components/ocrspace: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 mime: specifier: ^4.0.6 version: 4.0.6 @@ -9176,8 +9176,8 @@ importers: components/onlinecheckwriter: dependencies: '@pipedream/platform': - specifier: 3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9208,8 +9208,8 @@ importers: components/openai: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 '@pipedream/types': specifier: ^0.1.4 version: 0.1.6 @@ -9490,8 +9490,8 @@ importers: components/pandadoc: dependencies: '@pipedream/platform': - specifier: ^1.6.0 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9669,8 +9669,8 @@ importers: components/pdf4me: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 components/pdf_api_io: {} From 04ae59af6857fd5cf75988fa0677bf624309cee1 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Fri, 6 Jun 2025 18:48:29 -0300 Subject: [PATCH 14/47] add: ocrspace --- .../ocrspace/actions/common/process-base.mjs | 5 +---- .../actions/process-image/process-image.mjs | 2 +- .../actions/process-pdf/process-pdf.mjs | 2 +- components/ocrspace/common/utils.mjs | 20 +++++++------------ components/ocrspace/ocrspace.app.mjs | 4 ++-- components/ocrspace/package.json | 4 ++-- 6 files changed, 14 insertions(+), 23 deletions(-) diff --git a/components/ocrspace/actions/common/process-base.mjs b/components/ocrspace/actions/common/process-base.mjs index 2a8fbc60500d1..da822c6e2f479 100644 --- a/components/ocrspace/actions/common/process-base.mjs +++ b/components/ocrspace/actions/common/process-base.mjs @@ -45,11 +45,8 @@ export default { }, async run({ $ }) { const data = new FormData(); - const { - url, file, - } = getUrlOrFile(this.file); + const file = await getUrlOrFile(this.file); - if (url) data.append("url", url); if (file) data.append("base64Image", file); if (this.imageLanguage) data.append("language", this.imageLanguage); if (this.isOverlayRequired) data.append("isOverlayRequired", `${this.isOverlayRequired}`); diff --git a/components/ocrspace/actions/process-image/process-image.mjs b/components/ocrspace/actions/process-image/process-image.mjs index dace35e1e2ca1..bcbea25dfc269 100644 --- a/components/ocrspace/actions/process-image/process-image.mjs +++ b/components/ocrspace/actions/process-image/process-image.mjs @@ -5,7 +5,7 @@ export default { key: "ocrspace-process-image", name: "Process Image", description: "Submits an image file for OCR processing using OCR.space. [See the documentation](https://ocr.space/ocrapi)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { ...common.props, diff --git a/components/ocrspace/actions/process-pdf/process-pdf.mjs b/components/ocrspace/actions/process-pdf/process-pdf.mjs index cda0790f5c620..223e525fc3aa5 100644 --- a/components/ocrspace/actions/process-pdf/process-pdf.mjs +++ b/components/ocrspace/actions/process-pdf/process-pdf.mjs @@ -5,7 +5,7 @@ export default { key: "ocrspace-process-pdf", name: "Process PDF for OCR", description: "Submit a PDF for OCR processing. [See the documentation](https://ocr.space/ocrapi)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { ...common.props, diff --git a/components/ocrspace/common/utils.mjs b/components/ocrspace/common/utils.mjs index 3384dcc350320..3a194dcb0e1ea 100644 --- a/components/ocrspace/common/utils.mjs +++ b/components/ocrspace/common/utils.mjs @@ -1,5 +1,4 @@ -import fs from "fs"; -import mime from "mime"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export const isValidUrl = (urlString) => { var urlPattern = new RegExp("^(https?:\\/\\/)?" + // validate protocol @@ -18,17 +17,12 @@ export const checkTmp = (filename) => { return filename; }; -export const getUrlOrFile = (url) => { - if (!isValidUrl(url)) { - const filePath = checkTmp(url); - const data = fs.readFileSync(filePath); - const mimeType = mime.getType(filePath); - const base64Image = Buffer.from(data, "binary").toString("base64"); - return { - file: `data:${mimeType};base64,${base64Image}`, - }; - } +export const getUrlOrFile = async (url) => { + const { + stream, metadata: { contentType }, + } = await getFileStreamAndMetadata(url); + const base64Image = Buffer.from(stream, "binary").toString("base64"); return { - url, + file: `data:${contentType};base64,${base64Image}`, }; }; diff --git a/components/ocrspace/ocrspace.app.mjs b/components/ocrspace/ocrspace.app.mjs index ae96d7db00db7..c43d7de875022 100644 --- a/components/ocrspace/ocrspace.app.mjs +++ b/components/ocrspace/ocrspace.app.mjs @@ -11,8 +11,8 @@ export default { propDefinitions: { file: { type: "string", - label: "Image", - description: "The URL of the image or the path to the file saved to the `/tmp` directory (e.g. `/tmp/example.jpg`) to process. [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory).", + label: "File Path or URL", + description: "The file to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", }, language: { type: "string", diff --git a/components/ocrspace/package.json b/components/ocrspace/package.json index a0ab1ae330c10..656f2928aca88 100644 --- a/components/ocrspace/package.json +++ b/components/ocrspace/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/ocrspace", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream OCRSpace Components", "main": "ocrspace.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "mime": "^4.0.6" } } From 69bbb92c1eec80f26aa89c2efd716f770453fd0c Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Fri, 6 Jun 2025 22:46:55 -0300 Subject: [PATCH 15/47] add: onlinecheckwriter, pandadoc, pdf4me --- .../mail-pdf-document/mail-pdf-document.mjs | 15 ++---- components/onlinecheckwriter/package.json | 4 +- .../create-document-attachment.mjs | 48 +++++-------------- .../create-document-from-file.mjs | 24 ++-------- components/pandadoc/package.json | 4 +- components/pandadoc/pandadoc.app.mjs | 4 +- .../actions/compress-pdf/compress-pdf.mjs | 8 +--- .../actions/convert-to-pdf/convert-to-pdf.mjs | 8 +--- .../pdf4me/actions/merge-pdfs/merge-pdfs.mjs | 13 ++--- components/pdf4me/common/utils.mjs | 22 +++++---- components/pdf4me/package.json | 4 +- components/pdf4me/pdf4me.app.mjs | 4 +- 12 files changed, 54 insertions(+), 104 deletions(-) diff --git a/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs b/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs index daa5cb15e5fff..205f796055d67 100644 --- a/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs +++ b/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs @@ -1,13 +1,12 @@ -import fs from "fs"; import FormData from "form-data"; -import { ConfigurationError } from "@pipedream/platform"; +import { getFileStream } from "@pipedream/platform"; import app from "../../onlinecheckwriter.app.mjs"; export default { key: "onlinecheckwriter-mail-pdf-document", name: "Mail PDF Document", description: "Mails a PDF document to a destination. [See the documentation](https://apiv3.onlinecheckwriter.com/#878daf05-e36e-44a2-bce8-15f24d72f82e).", - version: "0.0.1", + version: "0.1.0", type: "action", props: { app, @@ -19,8 +18,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to the pdf file saved to the `/tmp` directory (e.g. `/tmp/example.pdf`). [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory).", + label: "File Path or URL", + description: "The PDF file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, shippingTypeId: { optional: false, @@ -167,12 +166,8 @@ export default { destinationEmail, } = this; - if (!filePath?.startsWith("/tmp/")) { - throw new ConfigurationError("The file path must start with `/tmp/`."); - } - const data = new FormData(); - const file = fs.createReadStream(filePath); + const file = await getFileStream(filePath); data.append("document_details[file]", file); data.append("document_details[title]", documentTitle || ""); data.append("shipping[shippingTypeId]", shippingTypeId || ""); diff --git a/components/onlinecheckwriter/package.json b/components/onlinecheckwriter/package.json index b3ad9e2566d5a..7ec6e37eca1d8 100644 --- a/components/onlinecheckwriter/package.json +++ b/components/onlinecheckwriter/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/onlinecheckwriter", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream OnlineCheckWriter Components", "main": "onlinecheckwriter.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "3.0.3", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs b/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs index ce7caf8c31f87..34e03ca8c53cb 100644 --- a/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs +++ b/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs @@ -1,7 +1,5 @@ import app from "../../pandadoc.app.mjs"; -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; -import path from "path"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { @@ -9,7 +7,7 @@ export default { name: "Create Document Attachment", description: "Adds an attachment to a document. [See the documentation here](https://developers.pandadoc.com/reference/create-document-attachment)", type: "action", - version: "0.0.7", + version: "0.1.0", props: { app, documentId: { @@ -31,39 +29,17 @@ export default { }, }, methods: { - isValidFile(filePath) { - const filePathWithTmp = `/tmp/${filePath}`; - if (fs.existsSync(filePathWithTmp)) { - return filePathWithTmp; - } else if (fs.existsSync(filePath)) { - return filePath; - } - return false; - }, - getFileStream(filePath) { - return fs.createReadStream(filePath); - }, - getFileMeta(filePath) { - const stats = fs.statSync(filePath); - return { - name: path.basename(filePath), - size: stats.size, - }; - }, - getFormData(file, fileName) { - const fileValidation = this.isValidFile(file); - if (!fileValidation) { - throw new ConfigurationError("`file` must be a valid file path!"); - } - - const fileMeta = this.getFileMeta(fileValidation); - const fileContent = this.getFileStream(fileValidation); + async getFormData(file, fileName) { + const { + stream, metadata: { + name, size, + }, + } = await getFileStreamAndMetadata(file); const data = new FormData(); - if (fileName) data.append("name", fileName); - data.append("file", fileContent, { - knownLength: fileMeta.size, + data.append("name", fileName || name); + data.append("file", stream, { + knownLength: size, }); - return data; }, }, @@ -74,7 +50,7 @@ export default { fileName, } = this; - const data = this.getFormData(file, fileName); + const data = await this.getFormData(file, fileName); const response = await this.app.createDocumentAttachments({ $, diff --git a/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs b/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs index dcd5bc0d454f2..5faf64bb0c36f 100644 --- a/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs +++ b/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs @@ -7,7 +7,7 @@ export default { name: "Create Document From File", description: "Create a document from a file or public file URL. [See the documentation here](https://developers.pandadoc.com/reference/create-document-from-pdf)", type: "action", - version: "0.0.8", + version: "1.0.0", props: { app, name: { @@ -27,13 +27,6 @@ export default { app, "file", ], - optional: true, - }, - fileUrl: { - type: "string", - label: "File URL", - description: "A public file URL to use instead of a local file.", - optional: true, }, documentFolderId: { propDefinition: [ @@ -55,7 +48,6 @@ export default { name, recipients, file, - fileUrl, documentFolderId, } = this; @@ -66,7 +58,7 @@ export default { throw new ConfigurationError("**Error parsing recipients** - each must be a valid JSON-stringified object"); } - let data, contentType, json = { + const json = { name, recipients: parsedRecipients, folder_uuid: documentFolderId, @@ -78,15 +70,9 @@ export default { : this.fields; } - if (fileUrl) { - data = json; - contentType = "application/json"; - data.url = fileUrl; - } else { - data = this.getFormData(file); - contentType = `multipart/form-data; boundary=${data._boundary}`; - data.append("data", JSON.stringify(json)); - } + const data = this.getFormData(file); + const contentType = `multipart/form-data; boundary=${data._boundary}`; + data.append("data", JSON.stringify(json)); const response = await this.app.createDocument({ $, diff --git a/components/pandadoc/package.json b/components/pandadoc/package.json index e1acc2088c9ea..f896236cfc31a 100644 --- a/components/pandadoc/package.json +++ b/components/pandadoc/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/pandadoc", - "version": "0.3.0", + "version": "1.0.0", "description": "Pipedream PandaDoc Components", "main": "pandadoc.app.mjs", "keywords": [ @@ -14,7 +14,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.6.0", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/components/pandadoc/pandadoc.app.mjs b/components/pandadoc/pandadoc.app.mjs index fd4de74e7b696..a3c2f59132532 100644 --- a/components/pandadoc/pandadoc.app.mjs +++ b/components/pandadoc/pandadoc.app.mjs @@ -88,8 +88,8 @@ export default { }, file: { type: "string", - label: "File", - description: "The file to upload from the `/tmp` folder. [See the docs here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp) on how to upload a file to `/tmp`.", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, name: { type: "string", diff --git a/components/pdf4me/actions/compress-pdf/compress-pdf.mjs b/components/pdf4me/actions/compress-pdf/compress-pdf.mjs index 7f259e3fc19f6..ebf39e88eec97 100644 --- a/components/pdf4me/actions/compress-pdf/compress-pdf.mjs +++ b/components/pdf4me/actions/compress-pdf/compress-pdf.mjs @@ -1,12 +1,11 @@ import pdf4me from "../../pdf4me.app.mjs"; import utils from "../../common/utils.mjs"; -import fs from "fs"; export default { key: "pdf4me-compress-pdf", name: "Compress PDF", description: "Compress a PDF file to reduce its size. [See the documentation](https://dev.pdf4me.com/apiv2/documentation/actions/compress-pdf/)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { pdf4me, @@ -41,10 +40,7 @@ export default { }, async run({ $ }) { const filename = utils.checkForExtension(this.filename, "pdf"); - const filePath = utils.normalizeFilePath(this.filePath); - const fileContent = fs.readFileSync(filePath, { - encoding: "base64", - }); + const fileContent = await utils.getBase64File(this.filePath); const response = await this.pdf4me.compressPdf({ $, diff --git a/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs b/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs index cf87f9f9f1144..9bbfdf296033e 100644 --- a/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs +++ b/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs @@ -1,12 +1,11 @@ import pdf4me from "../../pdf4me.app.mjs"; import utils from "../../common/utils.mjs"; -import fs from "fs"; export default { key: "pdf4me-convert-to-pdf", name: "Convert to PDF", description: "Convert a document (e.g., DOCX, XLSX, PPTX) to PDF. [See the documentation](https://dev.pdf4me.com/apiv2/documentation/actions/convert-to-pdf/)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { pdf4me, @@ -25,10 +24,7 @@ export default { }, }, async run({ $ }) { - const filePath = utils.normalizeFilePath(this.filePath); - const fileContent = fs.readFileSync(filePath, { - encoding: "base64", - }); + const fileContent = await utils.getBase64File(this.filePath); const response = await this.pdf4me.convertToPdf({ $, diff --git a/components/pdf4me/actions/merge-pdfs/merge-pdfs.mjs b/components/pdf4me/actions/merge-pdfs/merge-pdfs.mjs index df5857656831d..f2e9f51f3481b 100644 --- a/components/pdf4me/actions/merge-pdfs/merge-pdfs.mjs +++ b/components/pdf4me/actions/merge-pdfs/merge-pdfs.mjs @@ -1,19 +1,18 @@ import pdf4me from "../../pdf4me.app.mjs"; import utils from "../../common/utils.mjs"; -import fs from "fs"; export default { key: "pdf4me-merge-pdfs", name: "Merge PDF Files", description: "Merge multiple PDF files into a single PDF. [See the documentation](https://dev.pdf4me.com/apiv2/documentation/actions/merge/)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { pdf4me, filePaths: { type: "string[]", - label: "File Paths", - description: "An array of paths to a PDF files in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Paths or URLs", + description: "The files to process. For each entry, provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, filename: { propDefinition: [ @@ -24,10 +23,8 @@ export default { }, async run({ $ }) { const filename = utils.checkForExtension(this.filename, "pdf"); - const filePaths = this.filePaths.map((path) => utils.normalizeFilePath(path)); - const fileContents = filePaths.map((path) => fs.readFileSync(path, { - encoding: "base64", - })); + const promises = await Promise.allSettled(this.filePaths.map((p) => utils.getBase64File(p))); + const fileContents = promises.map((prom) => prom.value); const response = await this.pdf4me.mergePdfs({ $, diff --git a/components/pdf4me/common/utils.mjs b/components/pdf4me/common/utils.mjs index 6f46d4edf4569..353736c8d790c 100644 --- a/components/pdf4me/common/utils.mjs +++ b/components/pdf4me/common/utils.mjs @@ -1,5 +1,7 @@ import fs from "fs"; -import { ConfigurationError } from "@pipedream/platform"; +import { + ConfigurationError, getFileStream, +} from "@pipedream/platform"; function normalizeFilePath(path) { return path.startsWith("/tmp/") @@ -7,12 +9,6 @@ function normalizeFilePath(path) { : `/tmp/${path}`; } -function checkForExtension(filename, ext = "pdf") { - return filename.endsWith(`.${ext}`) - ? filename - : `${filename}.${ext}`; -} - function downloadToTmp(response, filename) { const rawcontent = response.toString("base64"); const buffer = Buffer.from(rawcontent, "base64"); @@ -41,9 +37,17 @@ function handleErrorMessage(error) { throw new ConfigurationError(errorMessage); } +async function getBase64File(filePath) { + const stream = await getFileStream(filePath); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + return Buffer.concat(chunks).toString("base64"); +} + export default { - normalizeFilePath, - checkForExtension, downloadToTmp, handleErrorMessage, + getBase64File, }; diff --git a/components/pdf4me/package.json b/components/pdf4me/package.json index ba088f91d5b82..0dff14900e86e 100644 --- a/components/pdf4me/package.json +++ b/components/pdf4me/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/pdf4me", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream PDF4me Components", "main": "pdf4me.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3" + "@pipedream/platform": "^3.1.0" } } diff --git a/components/pdf4me/pdf4me.app.mjs b/components/pdf4me/pdf4me.app.mjs index ec179a9ff9730..6a552775d238f 100644 --- a/components/pdf4me/pdf4me.app.mjs +++ b/components/pdf4me/pdf4me.app.mjs @@ -7,8 +7,8 @@ export default { propDefinitions: { filePath: { type: "string", - label: "File Path", - description: "The path to a PDF file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.doc`)", }, filename: { type: "string", From 094bf97cf400c97116289f1a8534e28e25636494 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 9 Jun 2025 00:03:51 -0300 Subject: [PATCH 16/47] Onedrive adjustment --- .../actions/upload-file/upload-file.mjs | 19 ++++++------------- components/microsoft_onedrive/package.json | 1 - 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/components/microsoft_onedrive/actions/upload-file/upload-file.mjs b/components/microsoft_onedrive/actions/upload-file/upload-file.mjs index 968a04953b6f1..cc3caa4c85aaa 100644 --- a/components/microsoft_onedrive/actions/upload-file/upload-file.mjs +++ b/components/microsoft_onedrive/actions/upload-file/upload-file.mjs @@ -1,8 +1,8 @@ import onedrive from "../../microsoft_onedrive.app.mjs"; import { - ConfigurationError, getFileStream, + ConfigurationError, + getFileStreamAndMetadata, } from "@pipedream/platform"; -import { fileTypeFromStream } from "file-type"; export default { name: "Upload File", @@ -40,17 +40,10 @@ export default { throw new ConfigurationError("You must specify the **Upload Folder ID**."); } - let stream = await getFileStream(filePath); - let name = filename; - - if (!filename.includes(".")) { - const fileTypeResult = await fileTypeFromStream(stream); - const extension = fileTypeResult?.ext || ""; - name = `${filename}.${extension}`; - - stream.destroy(); - stream = await getFileStream(filePath); - } + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); + const name = filename || metadata.name; const response = await this.onedrive.uploadFile({ uploadFolderId, diff --git a/components/microsoft_onedrive/package.json b/components/microsoft_onedrive/package.json index a914b82e5a62b..7d35f2040e98f 100644 --- a/components/microsoft_onedrive/package.json +++ b/components/microsoft_onedrive/package.json @@ -12,7 +12,6 @@ "@microsoft/microsoft-graph-client": "^3.0.1", "@pipedream/platform": "^3.1.0", "bottleneck": "^2.19.5", - "file-type": "^18.7.0", "isomorphic-fetch": "^3.0.0", "lodash.get": "^4.4.2" } From 0107ed68b996d4965fb3a73ab3653b9c7f36abfc Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 9 Jun 2025 00:08:15 -0300 Subject: [PATCH 17/47] pnpm --- pnpm-lock.yaml | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8bb60e6a8d3e6..27b18c7fc9847 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8194,9 +8194,6 @@ importers: bottleneck: specifier: ^2.19.5 version: 2.19.5 - file-type: - specifier: ^18.7.0 - version: 18.7.0 isomorphic-fetch: specifier: ^3.0.0 version: 3.0.0 @@ -23752,10 +23749,6 @@ packages: resolution: {integrity: sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==} engines: {node: '>=10'} - file-type@18.7.0: - resolution: {integrity: sha512-ihHtXRzXEziMrQ56VSgU7wkxh55iNchFkosu7Y9/S+tXHdKyrGjVK0ujbqNnsxzea+78MaLhN6PGmfYSAv1ACw==} - engines: {node: '>=14.16'} - file-type@3.9.0: resolution: {integrity: sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==} engines: {node: '>=0.10.0'} @@ -27424,10 +27417,6 @@ packages: resolution: {integrity: sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==} engines: {node: '>=8'} - peek-readable@5.3.1: - resolution: {integrity: sha512-GVlENSDW6KHaXcd9zkZltB7tCLosKB/4Hg0fqBJkAoBgYG2Tn1xtMgXtSUuMU9AK/gCm/tTdT8mgAeF4YNeeqw==} - engines: {node: '>=14.16'} - pend@1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} @@ -29047,10 +29036,6 @@ packages: resolution: {integrity: sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==} engines: {node: '>=10'} - strtok3@7.1.1: - resolution: {integrity: sha512-mKX8HA/cdBqMKUr0MMZAFssCkIGoZeSCMXgnt79yKxNFguMLVFgRe6wB+fsL0NmoHDbeyZXczy7vEPSoo3rkzg==} - engines: {node: '>=16'} - stubs@3.0.0: resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==} @@ -29376,10 +29361,6 @@ packages: resolution: {integrity: sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==} engines: {node: '>=10'} - token-types@5.0.1: - resolution: {integrity: sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==} - engines: {node: '>=14.16'} - touch@3.1.1: resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==} hasBin: true @@ -41751,12 +41732,6 @@ snapshots: strtok3: 6.3.0 token-types: 4.2.1 - file-type@18.7.0: - dependencies: - readable-web-to-node-stream: 3.0.2 - strtok3: 7.1.1 - token-types: 5.0.1 - file-type@3.9.0: {} file-uri-to-path@1.0.0: {} @@ -46799,8 +46774,6 @@ snapshots: peek-readable@4.1.0: {} - peek-readable@5.3.1: {} - pend@1.2.0: {} performance-now@2.1.0: {} @@ -49139,11 +49112,6 @@ snapshots: '@tokenizer/token': 0.3.0 peek-readable: 4.1.0 - strtok3@7.1.1: - dependencies: - '@tokenizer/token': 0.3.0 - peek-readable: 5.3.1 - stubs@3.0.0: {} style-to-object@0.4.4: @@ -49619,11 +49587,6 @@ snapshots: '@tokenizer/token': 0.3.0 ieee754: 1.2.1 - token-types@5.0.1: - dependencies: - '@tokenizer/token': 0.3.0 - ieee754: 1.2.1 - touch@3.1.1: {} tough-cookie@2.5.0: From 8aa2266347a1da0d2e7b6b45ad78ff324dc19d1b Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 9 Jun 2025 16:05:17 -0300 Subject: [PATCH 18/47] adjusting Slack and Pandadoc --- .../create-document-attachment.mjs | 10 +++++----- .../create-document-from-file.mjs | 2 +- components/slack/actions/upload-file/upload-file.mjs | 6 +++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs b/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs index 34e03ca8c53cb..91c77f10230ea 100644 --- a/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs +++ b/components/pandadoc/actions/create-document-attachment/create-document-attachment.mjs @@ -31,14 +31,14 @@ export default { methods: { async getFormData(file, fileName) { const { - stream, metadata: { - name, size, - }, + stream, metadata, } = await getFileStreamAndMetadata(file); const data = new FormData(); - data.append("name", fileName || name); + data.append("name", fileName || metadata.name); data.append("file", stream, { - knownLength: size, + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, }); return data; }, diff --git a/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs b/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs index 5faf64bb0c36f..240af1e76bc00 100644 --- a/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs +++ b/components/pandadoc/actions/create-document-from-file/create-document-from-file.mjs @@ -70,7 +70,7 @@ export default { : this.fields; } - const data = this.getFormData(file); + const data = await this.getFormData(file); const contentType = `multipart/form-data; boundary=${data._boundary}`; data.append("data", JSON.stringify(json)); diff --git a/components/slack/actions/upload-file/upload-file.mjs b/components/slack/actions/upload-file/upload-file.mjs index 3aea48d309fc9..bbf4c2a3f34b7 100644 --- a/components/slack/actions/upload-file/upload-file.mjs +++ b/components/slack/actions/upload-file/upload-file.mjs @@ -56,7 +56,11 @@ export default { // Upload the file to the provided URL const formData = new FormData(); - formData.append("file", stream); + formData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); formData.append("filename", filename); await axios($, { From ed11f6ec633b213762654d7ae4976776a44d0ca8 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 9 Jun 2025 20:56:51 -0300 Subject: [PATCH 19/47] updates to include metadata in all formdatas --- .../html-file-to-markdown.mjs | 11 ++++++-- .../pdf-to-markdown/pdf-to-markdown.mjs | 11 ++++++-- .../create-a-new-product-media-file.mjs | 13 +++++++--- components/askyourpdf/askyourpdf.app.mjs | 6 ++--- components/askyourpdf/common/utils.mjs | 15 ++++++++--- .../actions/generate-pdf/generate-pdf.mjs | 11 ++++++-- .../actions/create-tileset/create-tileset.mjs | 12 ++++++--- .../actions/upload-file/upload-file.mjs | 12 ++++++--- .../update-column-values.mjs | 12 ++++++--- components/nyckel/common/common-image.mjs | 25 +++++++----------- .../mail-pdf-document/mail-pdf-document.mjs | 12 ++++++--- .../onlyoffice_docspace/common/utils.mjs | 15 ++++++++--- .../create-campaign/create-campaign.mjs | 23 +++++++++++++--- components/stannp/common/utils.mjs | 21 +++++++++------ components/stannp/package.json | 3 +-- components/zamzar/common/utils.mjs | 15 ++++++++--- components/zamzar/zamzar.app.mjs | 4 +-- components/zoho_desk/common/utils.mjs | 26 +++++++++++-------- components/zoho_desk/zoho_desk.app.mjs | 2 +- 19 files changed, 168 insertions(+), 81 deletions(-) diff --git a/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs b/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs index 3e80fd0ad3302..c238c14ff71c0 100644 --- a/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs +++ b/components/_2markdown/actions/html-file-to-markdown/html-file-to-markdown.mjs @@ -1,5 +1,5 @@ import _2markdown from "../../_2markdown.app.mjs"; -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { @@ -20,7 +20,14 @@ export default { async run({ $ }) { const form = new FormData(); - form.append("document", await getFileStream(this.filePath)); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + form.append("document", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); const response = await this._2markdown.htmlFileToMarkdown({ $, diff --git a/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs b/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs index 306196acab213..2a27deafd32cf 100644 --- a/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs +++ b/components/_2markdown/actions/pdf-to-markdown/pdf-to-markdown.mjs @@ -1,5 +1,5 @@ import _2markdown from "../../_2markdown.app.mjs"; -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { @@ -27,7 +27,14 @@ export default { async run({ $ }) { const form = new FormData(); - form.append("document", await getFileStream(this.filePath)); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + form.append("document", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); let response = await this._2markdown.pdfToMarkdown({ $, diff --git a/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs b/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs index c534cfe428023..f43f439b34223 100644 --- a/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs +++ b/components/akeneo/actions/create-a-new-product-media-file/create-a-new-product-media-file.mjs @@ -1,6 +1,6 @@ import app from "../../akeneo.app.mjs"; import { - ConfigurationError, getFileStream, + ConfigurationError, getFileStreamAndMetadata, } from "@pipedream/platform"; import FormData from "form-data"; @@ -55,9 +55,14 @@ export default { data.append("product_model", JSON.stringify(payload)); } - const fileStream = await getFileStream(this.filename); - const fileParts = this.filename.split("/"); - data.append("file", fileStream, fileParts[fileParts.length - 1]); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filename); + data.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); const contentLength = data.getLengthSync(); await this.app.createProductMediaFile({ $, diff --git a/components/askyourpdf/askyourpdf.app.mjs b/components/askyourpdf/askyourpdf.app.mjs index 94a7e34bfb456..72244e8a3256f 100644 --- a/components/askyourpdf/askyourpdf.app.mjs +++ b/components/askyourpdf/askyourpdf.app.mjs @@ -27,12 +27,12 @@ export default { ...headers, }; }, - getConfig({ + async getConfig({ headers, data: preData, ...args } = {}) { const contentType = constants.CONTENT_TYPE_KEY_HEADER; const hasMultipartHeader = utils.hasMultipartHeader(headers); - const data = hasMultipartHeader && utils.getFormData(preData) || preData; + const data = hasMultipartHeader && await utils.getFormData(preData) || preData; const currentHeaders = this.getHeaders(headers); return { @@ -49,7 +49,7 @@ export default { async makeRequest({ step = this, path, headers, data, summary, ...args } = {}) { - const config = this.getConfig({ + const config = await this.getConfig({ url: this.getUrl(path), headers, data, diff --git a/components/askyourpdf/common/utils.mjs b/components/askyourpdf/common/utils.mjs index 049bb0670a17e..4b22a4e5dbae4 100644 --- a/components/askyourpdf/common/utils.mjs +++ b/components/askyourpdf/common/utils.mjs @@ -1,4 +1,4 @@ -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; import constants from "./constants.mjs"; @@ -8,16 +8,23 @@ async function buildFormData(formData, data, parentKey) { await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, await getFileStream(data)); + const { + stream, metadata, + } = await getFileStreamAndMetadata(data); + formData.append(parentKey, stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } else if (data) { formData.append(parentKey, (data).toString()); } } -function getFormData(data) { +async function getFormData(data) { try { const formData = new FormData(); - buildFormData(formData, data); + await buildFormData(formData, data); return formData; } catch (error) { console.log("FormData Error", error); diff --git a/components/fileforge/actions/generate-pdf/generate-pdf.mjs b/components/fileforge/actions/generate-pdf/generate-pdf.mjs index 37afc7672ff20..83593ed0c9c5f 100644 --- a/components/fileforge/actions/generate-pdf/generate-pdf.mjs +++ b/components/fileforge/actions/generate-pdf/generate-pdf.mjs @@ -1,5 +1,5 @@ import FormData from "form-data"; -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import { parseObject } from "../../common/utils.mjs"; import fileforge from "../../fileforge.app.mjs"; @@ -63,7 +63,14 @@ export default { const parsedFiles = parseObject(files); for (const file of parsedFiles) { - formData.append("files", await getFileStream(file)); + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); + formData.append("files", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } formData.append("options", JSON.stringify({ diff --git a/components/mapbox/actions/create-tileset/create-tileset.mjs b/components/mapbox/actions/create-tileset/create-tileset.mjs index e403ed291b2f9..0772dbbfadfd8 100644 --- a/components/mapbox/actions/create-tileset/create-tileset.mjs +++ b/components/mapbox/actions/create-tileset/create-tileset.mjs @@ -1,5 +1,5 @@ import mapbox from "../../mapbox.app.mjs"; -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { @@ -51,8 +51,14 @@ export default { // Create Tileset Source try { const fileData = new FormData(); - const content = await getFileStream(filePath); - fileData.append("file", content); + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); + fileData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); await this.mapbox.createTilesetSource({ $, diff --git a/components/mistral_ai/actions/upload-file/upload-file.mjs b/components/mistral_ai/actions/upload-file/upload-file.mjs index 71eaea3e90e0a..b061edb173714 100644 --- a/components/mistral_ai/actions/upload-file/upload-file.mjs +++ b/components/mistral_ai/actions/upload-file/upload-file.mjs @@ -1,5 +1,5 @@ import mistralAI from "../../mistral_ai.app.mjs"; -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { @@ -28,9 +28,15 @@ export default { }, }, async run({ $ }) { - const fileContent = await getFileStream(this.filePath); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); const form = new FormData(); - form.append("file", fileContent); + form.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); if (this.purpose) { form.append("purpose", this.purpose); } diff --git a/components/monday/actions/update-column-values/update-column-values.mjs b/components/monday/actions/update-column-values/update-column-values.mjs index 35b46b6452501..7a77b4b06d263 100644 --- a/components/monday/actions/update-column-values/update-column-values.mjs +++ b/components/monday/actions/update-column-values/update-column-values.mjs @@ -1,6 +1,6 @@ import common from "../common/column-values.mjs"; import { - axios, getFileStream, + axios, getFileStreamAndMetadata, } from "@pipedream/platform"; import FormData from "form-data"; import { getColumnOptions } from "../../common/utils.mjs"; @@ -63,11 +63,17 @@ export default { $, itemId, column, filePath, }) { const query = `mutation ($file: File!) { add_file_to_column (file: $file, item_id: ${itemId}, column_id: "${column.id}") { id } }`; - const content = await getFileStream(filePath); + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); const formData = new FormData(); formData.append("query", query); - formData.append("variables[file]", content); + formData.append("variables[file]", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); return axios($, { method: "POST", diff --git a/components/nyckel/common/common-image.mjs b/components/nyckel/common/common-image.mjs index 8d4ad41a77860..a89cee9b64417 100644 --- a/components/nyckel/common/common-image.mjs +++ b/components/nyckel/common/common-image.mjs @@ -1,6 +1,6 @@ import nyckel from "../nyckel.app.mjs"; import FormData from "form-data"; -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { props: { @@ -13,22 +13,15 @@ export default { }, methods: { async getImageData() { - const { imageOrUrl } = this; - const isUrl = imageOrUrl.startsWith("http"); - if (isUrl) { - return { - data: { - data: imageOrUrl, - }, - headers: { - "Content-Type": "application/json", - }, - }; - } - const data = new FormData(); - const stream = await getFileStream(imageOrUrl); - data.append("data", stream); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.imageOrUrl); + data.append("data", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); return { data, diff --git a/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs b/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs index 205f796055d67..509b9bb478c1b 100644 --- a/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs +++ b/components/onlinecheckwriter/actions/mail-pdf-document/mail-pdf-document.mjs @@ -1,5 +1,5 @@ import FormData from "form-data"; -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import app from "../../onlinecheckwriter.app.mjs"; export default { @@ -167,8 +167,14 @@ export default { } = this; const data = new FormData(); - const file = await getFileStream(filePath); - data.append("document_details[file]", file); + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); + data.append("document_details[file]", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); data.append("document_details[title]", documentTitle || ""); data.append("shipping[shippingTypeId]", shippingTypeId || ""); data.append("destination[name]", destinationName || ""); diff --git a/components/onlyoffice_docspace/common/utils.mjs b/components/onlyoffice_docspace/common/utils.mjs index 800a52c3ae6c4..953b4070c9cf6 100644 --- a/components/onlyoffice_docspace/common/utils.mjs +++ b/components/onlyoffice_docspace/common/utils.mjs @@ -1,4 +1,4 @@ -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; import constants from "./constants.mjs"; @@ -8,16 +8,23 @@ async function buildFormData(formData, data, parentKey) { await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, await getFileStream(data)); + const { + stream, metadata, + } = await getFileStreamAndMetadata(data); + formData.append(parentKey, stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } else if (data) { formData.append(parentKey, (data).toString()); } } -function getFormData(data) { +async function getFormData(data) { try { const formData = new FormData(); - buildFormData(formData, data); + await buildFormData(formData, data); return formData; } catch (error) { console.log("FormData Error", error); diff --git a/components/stannp/actions/create-campaign/create-campaign.mjs b/components/stannp/actions/create-campaign/create-campaign.mjs index 18b69b58f4d56..b5f76d8474add 100644 --- a/components/stannp/actions/create-campaign/create-campaign.mjs +++ b/components/stannp/actions/create-campaign/create-campaign.mjs @@ -2,7 +2,7 @@ import FormData from "form-data"; import { TYPE_OPTIONS, WHAT_RECIPIENTS_OPTIONS, } from "../../common/constants.mjs"; -import { checkFile } from "../../common/utils.mjs"; +import { getFileData } from "../../common/utils.mjs"; import stannp from "../../stannp.app.mjs"; export default { @@ -107,9 +107,24 @@ export default { const formData = new FormData(); - if (file) formData.append("file", await checkFile(file)); - if (front) formData.append("front", await checkFile(front)); - if (back) formData.append("back", await checkFile(back)); + if (file) { + const { + stream, metadata, + } = await getFileData(file); + formData.append("file", stream, metadata); + } + if (front) { + const { + stream, metadata, + } = await getFileData(front); + formData.append("front", stream, metadata); + } + if (back) { + const { + stream, metadata, + } = await getFileData(back); + formData.append("back", stream, metadata); + } if (templateId) formData.append("template_id", templateId); if (groupId) formData.append("group_id", groupId); diff --git a/components/stannp/common/utils.mjs b/components/stannp/common/utils.mjs index 5855330e494a8..516df2adbec60 100644 --- a/components/stannp/common/utils.mjs +++ b/components/stannp/common/utils.mjs @@ -1,5 +1,4 @@ -import urlExists from "url-exist"; -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export const parseObject = (obj) => { if (Array.isArray(obj)) { @@ -20,10 +19,16 @@ export const parseObject = (obj) => { return obj; }; -export const checkFile = async (url) => { - if (await urlExists(url)) { - return url; - } else { - return getFileStream(url); - } +export const getFileData = async (file) => { + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); + return { + stream, + metadata: { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }, + }; }; diff --git a/components/stannp/package.json b/components/stannp/package.json index 98a651b14f424..c5f3ab3608c32 100644 --- a/components/stannp/package.json +++ b/components/stannp/package.json @@ -14,7 +14,6 @@ }, "dependencies": { "@pipedream/platform": "^3.1.0", - "form-data": "^4.0.0", - "url-exist": "^3.0.1" + "form-data": "^4.0.0" } } diff --git a/components/zamzar/common/utils.mjs b/components/zamzar/common/utils.mjs index a59db04ebe003..712254a276dbe 100644 --- a/components/zamzar/common/utils.mjs +++ b/components/zamzar/common/utils.mjs @@ -1,4 +1,4 @@ -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; import constants from "./constants.mjs"; @@ -8,16 +8,23 @@ async function buildFormData(formData, data, parentKey) { await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, await getFileStream(data)); + const { + stream, metadata, + } = await getFileStreamAndMetadata(data); + formData.append(parentKey, stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } else if (data) { formData.append(parentKey, (data).toString()); } } -function getFormData(data) { +async function getFormData(data) { try { const formData = new FormData(); - buildFormData(formData, data); + await buildFormData(formData, data); return formData; } catch (error) { console.log("FormData Error", error); diff --git a/components/zamzar/zamzar.app.mjs b/components/zamzar/zamzar.app.mjs index f9b57c16de243..7077c2575768e 100644 --- a/components/zamzar/zamzar.app.mjs +++ b/components/zamzar/zamzar.app.mjs @@ -45,7 +45,7 @@ export default { password: "", }; }, - _makeRequest({ + async _makeRequest({ $ = this, path, headers, data: rawData, ...args } = {}) { const { @@ -55,7 +55,7 @@ export default { const contentType = constants.CONTENT_TYPE_KEY_HEADER; const hasMultipartHeader = utils.hasMultipartHeader(headers); - const data = hasMultipartHeader && utils.getFormData(rawData) || rawData; + const data = hasMultipartHeader && await utils.getFormData(rawData) || rawData; const config = { ...args, diff --git a/components/zoho_desk/common/utils.mjs b/components/zoho_desk/common/utils.mjs index 57fcc7fe18db5..dff671bc33bc6 100644 --- a/components/zoho_desk/common/utils.mjs +++ b/components/zoho_desk/common/utils.mjs @@ -1,4 +1,4 @@ -import { getFileStream } from "@pipedream/platform"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; import retry from "async-retry"; import constants from "./constants.mjs"; @@ -11,25 +11,29 @@ async function streamIterator(stream) { return resources; } -function buildFormData(formData, data, parentKey) { +async function buildFormData(formData, data, parentKey) { if (data && typeof(data) === "object") { - Object.keys(data) - .forEach(async (key) => { - buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); - }); - + for (const key of Object.keys(data)) { + await buildFormData(formData, data[key], parentKey && `${parentKey}[${key}]` || key); + } } else if (data && constants.FILE_PROP_NAMES.some((prop) => parentKey.includes(prop))) { - formData.append(parentKey, getFileStream(data)); - + const { + stream, metadata, + } = await getFileStreamAndMetadata(data); + formData.append(parentKey, stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); } else if (data) { formData.append(parentKey, (data).toString()); } } -function getFormData(data) { +async function getFormData(data) { try { const formData = new FormData(); - buildFormData(formData, data); + await buildFormData(formData, data); return formData; } catch (error) { console.log("FormData Error", error); diff --git a/components/zoho_desk/zoho_desk.app.mjs b/components/zoho_desk/zoho_desk.app.mjs index 8a531dd6d3f7d..de2840cfae167 100644 --- a/components/zoho_desk/zoho_desk.app.mjs +++ b/components/zoho_desk/zoho_desk.app.mjs @@ -134,7 +134,7 @@ export default { const contentType = constants.CONTENT_TYPE_KEY_HEADER; const hasMultipartHeader = utils.hasMultipartHeader(preHeaders); - const data = hasMultipartHeader && utils.getFormData(preData) || preData; + const data = hasMultipartHeader && await utils.getFormData(preData) || preData; const currentHeaders = this.getHeaders(preHeaders); const headers = hasMultipartHeader From 1afc1fa0db6b8c9d286d954882ee7f4c046defd7 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 9 Jun 2025 20:57:49 -0300 Subject: [PATCH 20/47] pnpm --- pnpm-lock.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 27b18c7fc9847..c2ae6b3e13d76 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12789,9 +12789,6 @@ importers: form-data: specifier: ^4.0.0 version: 4.0.1 - url-exist: - specifier: ^3.0.1 - version: 3.0.1(web-streams-polyfill@3.3.3) components/starloop: {} From c0438fba67e03e5e8b878269d3d9786017a3dffc Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 9 Jun 2025 21:29:59 -0300 Subject: [PATCH 21/47] fix --- .../onlyoffice_docspace/onlyoffice_docspace.app.mjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/onlyoffice_docspace/onlyoffice_docspace.app.mjs b/components/onlyoffice_docspace/onlyoffice_docspace.app.mjs index a2380211af035..78e3812015a2a 100644 --- a/components/onlyoffice_docspace/onlyoffice_docspace.app.mjs +++ b/components/onlyoffice_docspace/onlyoffice_docspace.app.mjs @@ -46,12 +46,12 @@ export default { ...headers, }; }, - getConfig({ + async getConfig({ headers, data: preData, ...args } = {}) { const contentType = constants.CONTENT_TYPE_KEY_HEADER; const hasMultipartHeader = utils.hasMultipartHeader(headers); - const data = hasMultipartHeader && utils.getFormData(preData) || preData; + const data = hasMultipartHeader && await utils.getFormData(preData) || preData; const currentHeaders = this.getHeaders(headers); return { @@ -65,10 +65,10 @@ export default { ...args, }; }, - _makeRequest({ + async _makeRequest({ $ = this, path, headers, ...args } = {}) { - const config = this.getConfig({ + const config = await this.getConfig({ url: this.getUrl(path), headers: this.getHeaders(headers), ...args, From 055a2e4811f4c4f24c7f8a7cbd39ba22b58bed70 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 9 Jun 2025 21:32:11 -0300 Subject: [PATCH 22/47] pnpm --- pnpm-lock.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c2ae6b3e13d76..727e2d75308ff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5518,7 +5518,7 @@ importers: specifier: ^2.3.0 version: 2.4.0 '@pipedream/platform': - specifier: ^3.1.0 + specifier: ^3.0.3 version: 3.1.0 cron-parser: specifier: ^4.9.0 @@ -6481,8 +6481,8 @@ importers: components/imagga: dependencies: '@pipedream/platform': - specifier: ^3.1.0 - version: 3.1.0 + specifier: ^1.5.1 + version: 1.6.6 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9205,7 +9205,7 @@ importers: components/openai: dependencies: '@pipedream/platform': - specifier: ^3.1.0 + specifier: ^3.0.3 version: 3.1.0 '@pipedream/types': specifier: ^0.1.4 From b6a68ad270432fc26117320827df63a4c6bf0628 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 9 Jun 2025 21:58:38 -0300 Subject: [PATCH 23/47] CodeRabbit fixes --- components/ocrspace/common/utils.mjs | 29 +++++++++++++++++----------- components/pdf4me/common/utils.mjs | 21 +++++++++++++++----- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/components/ocrspace/common/utils.mjs b/components/ocrspace/common/utils.mjs index 3a194dcb0e1ea..d9fb05bac28f1 100644 --- a/components/ocrspace/common/utils.mjs +++ b/components/ocrspace/common/utils.mjs @@ -1,12 +1,15 @@ import { getFileStreamAndMetadata } from "@pipedream/platform"; export const isValidUrl = (urlString) => { - var urlPattern = new RegExp("^(https?:\\/\\/)?" + // validate protocol -"((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|" + // validate domain name -"((\\d{1,3}\\.){3}\\d{1,3}))" + // validate OR ip (v4) address -"(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*" + // validate port and path -"(\\?[;&a-z\\d%_.~+=-]*)?" + // validate query string -"(\\#[-a-z\\d_]*)?$", "i"); // validate fragment locator + var urlPattern = new RegExp( + "^(https?:\\/\\/)?" + // validate protocol + "((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|" + // validate domain name + "((\\d{1,3}\\.){3}\\d{1,3}))" + // validate OR ip (v4) address + "(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*" + // validate port and path + "(\\?[;&a-z\\d%_.~+=-]*)?" + // validate query string + "(\\#[-a-z\\d_]*)?$", + "i", + ); // validate fragment locator return !!urlPattern.test(urlString); }; @@ -19,10 +22,14 @@ export const checkTmp = (filename) => { export const getUrlOrFile = async (url) => { const { - stream, metadata: { contentType }, + stream, + metadata: { contentType }, } = await getFileStreamAndMetadata(url); - const base64Image = Buffer.from(stream, "binary").toString("base64"); - return { - file: `data:${contentType};base64,${base64Image}`, - }; + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const buffer = Buffer.concat(chunks); + const base64Image = buffer.toString("base64"); + return `data:${contentType};base64,${base64Image}`; }; diff --git a/components/pdf4me/common/utils.mjs b/components/pdf4me/common/utils.mjs index 353736c8d790c..68dd4219936c4 100644 --- a/components/pdf4me/common/utils.mjs +++ b/components/pdf4me/common/utils.mjs @@ -9,6 +9,12 @@ function normalizeFilePath(path) { : `/tmp/${path}`; } +function checkForExtension(filename, ext = "pdf") { + return filename.endsWith(`.${ext}`) + ? filename + : `${filename}.${ext}`; +} + function downloadToTmp(response, filename) { const rawcontent = response.toString("base64"); const buffer = Buffer.from(rawcontent, "base64"); @@ -38,15 +44,20 @@ function handleErrorMessage(error) { } async function getBase64File(filePath) { - const stream = await getFileStream(filePath); - const chunks = []; - for await (const chunk of stream) { - chunks.push(chunk); + try { + const stream = await getFileStream(filePath); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + return Buffer.concat(chunks).toString("base64"); + } catch (error) { + throw new ConfigurationError(`Error parsing file \`${filePath}\`: ${error.message}`); } - return Buffer.concat(chunks).toString("base64"); } export default { + checkForExtension, downloadToTmp, handleErrorMessage, getBase64File, From 9158a07b9e494ff61e7c48fe5d004ef3dadd1ad0 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Tue, 10 Jun 2025 17:25:18 -0300 Subject: [PATCH 24/47] Fixing box and nyckel --- .../box/actions/upload-file-version/upload-file-version.mjs | 2 +- components/box/actions/upload-file/upload-file.mjs | 2 +- components/box/common/common-file-upload.mjs | 4 ++-- components/nyckel/actions/classify-image/classify-image.mjs | 2 +- .../extract-text-from-image/extract-text-from-image.mjs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/box/actions/upload-file-version/upload-file-version.mjs b/components/box/actions/upload-file-version/upload-file-version.mjs index a94d96124d937..9884061a1c8af 100644 --- a/components/box/actions/upload-file-version/upload-file-version.mjs +++ b/components/box/actions/upload-file-version/upload-file-version.mjs @@ -55,7 +55,7 @@ export default { const { file, fileId, createdAt, modifiedAt, fileName, parentId, } = this; - const data = this.getFileUploadBody({ + const data = await this.getFileUploadBody({ file, createdAt, modifiedAt, diff --git a/components/box/actions/upload-file/upload-file.mjs b/components/box/actions/upload-file/upload-file.mjs index 46a3ae0618330..476d98cf924bb 100644 --- a/components/box/actions/upload-file/upload-file.mjs +++ b/components/box/actions/upload-file/upload-file.mjs @@ -47,7 +47,7 @@ export default { const { file, createdAt, modifiedAt, fileName, parentId, } = this; - const data = this.getFileUploadBody({ + const data = await this.getFileUploadBody({ file, createdAt, modifiedAt, diff --git a/components/box/common/common-file-upload.mjs b/components/box/common/common-file-upload.mjs index d24b687206691..a5e4434af54bc 100644 --- a/components/box/common/common-file-upload.mjs +++ b/components/box/common/common-file-upload.mjs @@ -1,7 +1,7 @@ import FormData from "form-data"; import utils from "./utils.mjs"; -export function getFileUploadBody({ +export async function getFileUploadBody({ file, createdAt, modifiedAt, @@ -10,7 +10,7 @@ export function getFileUploadBody({ }) { const { fileMeta, fileContent, - } = utils.getFileData(file); + } = await utils.getFileData(file); const attributes = fileMeta.attributes; if (createdAt && utils.checkRFC3339(createdAt)) { attributes.content_created_at = createdAt; diff --git a/components/nyckel/actions/classify-image/classify-image.mjs b/components/nyckel/actions/classify-image/classify-image.mjs index f59687e30014a..9a7b46a1f6647 100644 --- a/components/nyckel/actions/classify-image/classify-image.mjs +++ b/components/nyckel/actions/classify-image/classify-image.mjs @@ -46,7 +46,7 @@ export default { const response = await this.nyckel.invokeFunction({ $, functionId: this.functionId, - ...this.getImageData(), + ...(await this.getImageData()), params: { labelCount: this.labelCount, includeMetadata: this.includeMetadata, diff --git a/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs b/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs index bad8921c64745..ec0028ea8fedc 100644 --- a/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs +++ b/components/nyckel/actions/extract-text-from-image/extract-text-from-image.mjs @@ -28,7 +28,7 @@ export default { const response = await this.nyckel.extractTextFromImageUrl({ $, functionId: this.functionId, - ...this.getImageData(), + ...(await this.getImageData()), params: { includeRegions: this.includeRegions, }, From c34250fe872c1a1dc94c9ab33de84c2fd862d9bf Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Tue, 10 Jun 2025 17:45:44 -0300 Subject: [PATCH 25/47] fix mapbox --- .../mapbox/actions/create-tileset/create-tileset.mjs | 8 ++------ .../pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs | 1 - 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/components/mapbox/actions/create-tileset/create-tileset.mjs b/components/mapbox/actions/create-tileset/create-tileset.mjs index 0772dbbfadfd8..52240a4833525 100644 --- a/components/mapbox/actions/create-tileset/create-tileset.mjs +++ b/components/mapbox/actions/create-tileset/create-tileset.mjs @@ -44,16 +44,12 @@ export default { }, }, async run({ $ }) { - const filePath = this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`; - // Create Tileset Source try { const fileData = new FormData(); const { stream, metadata, - } = await getFileStreamAndMetadata(filePath); + } = await getFileStreamAndMetadata(this.filePath); fileData.append("file", stream, { contentType: metadata.contentType, knownLength: metadata.size, @@ -68,7 +64,7 @@ export default { headers: fileData.getHeaders(), }); } catch (e) { - throw new Error(`Error uploading file: \`${filePath}\`. Error: ${e}`); + throw new Error(`Error uploading file: \`${this.filePath}\`. Error: ${e}`); } const recipe = typeof this.recipe === "string" diff --git a/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs b/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs index 9bbfdf296033e..b7010ed2166b0 100644 --- a/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs +++ b/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs @@ -14,7 +14,6 @@ export default { pdf4me, "filePath", ], - description: "The path to a DOCX, XLSX, or PPTX file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", }, filename: { propDefinition: [ From 13d0e9f7bd92cdffc4641c0b53d060cdc8120ebc Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Tue, 10 Jun 2025 18:31:36 -0300 Subject: [PATCH 26/47] add: autodesk, cradl_ai, docsbot_ai --- .../actions/upload-file/upload-file.mjs | 23 +++++++++---------- components/autodesk/package.json | 4 ++-- components/cradl_ai/actions/common/common.mjs | 12 ++++------ .../parse-document-human-in-loop.mjs | 2 +- .../actions/parse-document/parse-document.mjs | 2 +- components/cradl_ai/package.json | 4 ++-- .../upload-source-file/upload-source-file.mjs | 18 +++++++-------- components/docsbot_ai/package.json | 4 ++-- 8 files changed, 32 insertions(+), 37 deletions(-) diff --git a/components/autodesk/actions/upload-file/upload-file.mjs b/components/autodesk/actions/upload-file/upload-file.mjs index f04a1718e6a08..81f5a04cc2255 100644 --- a/components/autodesk/actions/upload-file/upload-file.mjs +++ b/components/autodesk/actions/upload-file/upload-file.mjs @@ -1,12 +1,13 @@ import autodesk from "../../autodesk.app.mjs"; -import { axios } from "@pipedream/platform"; -import fs from "fs"; +import { + axios, getFileStreamAndMetadata, +} from "@pipedream/platform"; export default { key: "autodesk-upload-file", name: "Upload File", description: "Uploads a new file to a specified folder in Autodesk. [See the documentation](https://aps.autodesk.com/en/docs/data/v2/tutorials/upload-file/).", - version: "0.0.1", + version: "0.1.0", type: "action", props: { autodesk, @@ -42,8 +43,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, type: { type: "string", @@ -107,19 +108,17 @@ export default { const signedUrl = urls[0]; // Upload to signed URL - const filePath = this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`; - const fileStream = fs.createReadStream(filePath); - const fileSize = fs.statSync(filePath).size; + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); await axios($, { url: signedUrl, - data: fileStream, + data: stream, method: "PUT", headers: { "Content-Type": "application/octet-stream", - "Content-Length": fileSize, + "Content-Length": metadata.size, }, maxContentLength: Infinity, maxBodyLength: Infinity, diff --git a/components/autodesk/package.json b/components/autodesk/package.json index edab34b5a08b0..92df5dd1bfc38 100644 --- a/components/autodesk/package.json +++ b/components/autodesk/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/autodesk", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Autodesk Components", "main": "autodesk.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3" + "@pipedream/platform": "^3.1.0" } } diff --git a/components/cradl_ai/actions/common/common.mjs b/components/cradl_ai/actions/common/common.mjs index fafa7eb27e763..49f492070487f 100644 --- a/components/cradl_ai/actions/common/common.mjs +++ b/components/cradl_ai/actions/common/common.mjs @@ -1,14 +1,14 @@ import cradlAi from "../../cradl_ai.app.mjs"; import constants from "../../common/constants.mjs"; -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; export default { props: { cradlAi, filePath: { type: "string", - label: "File Path", - description: "The path to the document file saved to the `/tmp` directory (e.g. `/tmp/example.pdf`). [See the documentation](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory).", + label: "File Path or URL", + description: "The file to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`).", }, contentType: { type: "string", @@ -46,10 +46,8 @@ export default { }, }); }, - uploadFile($, fileUrl) { - const fileData = fs.readFileSync(this.filePath.includes("/tmp") - ? this.filePath - : `/tmp/${this.filePath}`); + async uploadFile($, fileUrl) { + const fileData = await getFileStream(this.filePath); return this.cradlAi.uploadDocument({ $, fileUrl, diff --git a/components/cradl_ai/actions/parse-document-human-in-loop/parse-document-human-in-loop.mjs b/components/cradl_ai/actions/parse-document-human-in-loop/parse-document-human-in-loop.mjs index c4a16d6eff9c1..edb243e3f37c4 100644 --- a/components/cradl_ai/actions/parse-document-human-in-loop/parse-document-human-in-loop.mjs +++ b/components/cradl_ai/actions/parse-document-human-in-loop/parse-document-human-in-loop.mjs @@ -5,7 +5,7 @@ export default { key: "cradl_ai-parse-document-human-in-loop", name: "Parse Document with Human in the Loop", description: "Sends a document to an existing flow for human-in-the-loop processing. [See the documentation](https://docs.cradl.ai/integrations/rest-api)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { ...common.props, diff --git a/components/cradl_ai/actions/parse-document/parse-document.mjs b/components/cradl_ai/actions/parse-document/parse-document.mjs index 5b320fb1af25b..b65d210acaaca 100644 --- a/components/cradl_ai/actions/parse-document/parse-document.mjs +++ b/components/cradl_ai/actions/parse-document/parse-document.mjs @@ -5,7 +5,7 @@ export default { key: "cradl_ai-parse-document", name: "Parse Document", description: "Parses data from a document using a custom selected model. [See the documentation](https://docs.cradl.ai/integrations/rest-api)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { ...common.props, diff --git a/components/cradl_ai/package.json b/components/cradl_ai/package.json index 5e42698c0847f..0bf2ed7903f0a 100644 --- a/components/cradl_ai/package.json +++ b/components/cradl_ai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/cradl_ai", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Cradl AI Components", "main": "cradl_ai.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.6.5" + "@pipedream/platform": "^3.1.0" } } diff --git a/components/docsbot_ai/actions/upload-source-file/upload-source-file.mjs b/components/docsbot_ai/actions/upload-source-file/upload-source-file.mjs index 9ef1e50e4dd05..762f007dc2bf0 100644 --- a/components/docsbot_ai/actions/upload-source-file/upload-source-file.mjs +++ b/components/docsbot_ai/actions/upload-source-file/upload-source-file.mjs @@ -1,11 +1,11 @@ -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import docsbotAi from "../../docsbot_ai.app.mjs"; export default { key: "docsbot_ai-upload-source-file", name: "Upload Source File", description: "Upload a file to be used as a source. [See the documentation](https://docsbot.ai/documentation/developer/source-api#source-file-uploads)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { docsbotAi, @@ -33,11 +33,9 @@ export default { }, async run({ $ }) { const { filePath } = this; - const data = fs.createReadStream(filePath.startsWith("/tmp") - ? filePath - : `/tmp/${filePath}`.replace(/\/\//g, "/")); - - const fileName = filePath.split("/").pop(); + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); const { file, url, @@ -46,16 +44,16 @@ export default { teamId: this.teamId, botId: this.botId, params: { - fileName, + fileName: metadata.name || filePath.split("/").pop(), }, }); await this.docsbotAi.uploadSourceFile({ $, url, - data, + data: stream, }); - $.export("$summary", `Successfully uploaded "${fileName}"`); + $.export("$summary", `Successfully uploaded "${metadata.name}"`); return { file, }; diff --git a/components/docsbot_ai/package.json b/components/docsbot_ai/package.json index 6acfd2ed0785b..92ac28d2e7d3c 100644 --- a/components/docsbot_ai/package.json +++ b/components/docsbot_ai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/docsbot_ai", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream DocsBot AI Components", "main": "docsbot_ai.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.5.1" + "@pipedream/platform": "^3.1.0" } } From 30210542871a604b7a8650a0da0824e1143eb19b Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Tue, 10 Jun 2025 18:32:03 -0300 Subject: [PATCH 27/47] pnpm --- pnpm-lock.yaml | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 727e2d75308ff..bffadb7c6fdd0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1075,8 +1075,8 @@ importers: components/autodesk: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 components/autoklose: {} @@ -3028,8 +3028,8 @@ importers: components/cradl_ai: dependencies: '@pipedream/platform': - specifier: ^1.6.5 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 components/craftboxx: dependencies: @@ -3661,8 +3661,8 @@ importers: components/docsbot_ai: dependencies: '@pipedream/platform': - specifier: ^1.5.1 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 components/docsgenflow: {} @@ -4070,17 +4070,14 @@ importers: components/egnyte: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 bottleneck: specifier: ^2.19.5 version: 2.19.5 form-data: specifier: ^4.0.1 version: 4.0.1 - mime: - specifier: ^4.0.6 - version: 4.0.6 path: specifier: ^0.12.7 version: 0.12.7 @@ -4691,8 +4688,8 @@ importers: components/flippingbook: dependencies: '@pipedream/platform': - specifier: ^2.0.0 - version: 2.0.0 + specifier: ^3.1.0 + version: 3.1.0 components/float: {} @@ -5919,8 +5916,8 @@ importers: components/happy_scribe: dependencies: '@pipedream/platform': - specifier: ^1.5.1 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 components/happyfox_chat: dependencies: @@ -6460,8 +6457,8 @@ importers: components/ilovepdf: dependencies: '@pipedream/platform': - specifier: 1.5.1 - version: 1.5.1 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -7618,8 +7615,8 @@ importers: components/lokalise: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 components/looker: {} @@ -10170,8 +10167,8 @@ importers: components/podio: dependencies: '@pipedream/platform': - specifier: ^1.5.1 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -15258,8 +15255,8 @@ importers: components/zoho_crm: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 crypto: specifier: ^1.0.1 version: 1.0.1 From 11928654c3d66a8e7ef90a73853369e76fcb8e7e Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Tue, 10 Jun 2025 22:09:34 -0300 Subject: [PATCH 28/47] add: egnyte, flippingbook --- .../actions/upload-file/upload-file.mjs | 25 +++++++------- components/egnyte/package.json | 5 ++- .../create-flipbook/create-flipbook.mjs | 33 ++++++++----------- .../update-flipbook/update-flipbook.mjs | 24 +++----------- components/flippingbook/flippingbook.app.mjs | 11 ++----- components/flippingbook/package.json | 4 +-- 6 files changed, 34 insertions(+), 68 deletions(-) diff --git a/components/egnyte/actions/upload-file/upload-file.mjs b/components/egnyte/actions/upload-file/upload-file.mjs index 09b9c1afab861..50df16c70fea9 100644 --- a/components/egnyte/actions/upload-file/upload-file.mjs +++ b/components/egnyte/actions/upload-file/upload-file.mjs @@ -1,21 +1,19 @@ import egnyte from "../../egnyte.app.mjs"; import FormData from "form-data"; -import fs from "fs"; -import path from "path"; -import mime from "mime"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "egnyte-upload-file", name: "Upload File", description: "Uploads a file to a specified folder in Egnyte. [See the documentation](https://developers.egnyte.com/docs/File_System_Management_API_Documentation#Upload-a-File)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { egnyte, filePath: { type: "string", - label: "File Path", - description: "The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, folderPath: { type: "string", @@ -26,16 +24,15 @@ export default { async run({ $ }) { const form = new FormData(); - const filePath = this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`; + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + const filename = metadata.name; - const filename = path.basename(filePath); - const contentType = mime.getType(filePath) || "application/octet-stream"; - - form.append("file", fs.createReadStream(filePath), { + form.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, filename, - contentType, }); let folderPath = this.folderPath; diff --git a/components/egnyte/package.json b/components/egnyte/package.json index d23f556e09fdd..efd9a305b59fc 100644 --- a/components/egnyte/package.json +++ b/components/egnyte/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/egnyte", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Egnyte Components", "main": "egnyte.app.mjs", "keywords": [ @@ -13,10 +13,9 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "bottleneck": "^2.19.5", "form-data": "^4.0.1", - "mime": "^4.0.6", "path": "^0.12.7" } } diff --git a/components/flippingbook/actions/create-flipbook/create-flipbook.mjs b/components/flippingbook/actions/create-flipbook/create-flipbook.mjs index 4e096e8cccf1f..4c1f5c13fdded 100644 --- a/components/flippingbook/actions/create-flipbook/create-flipbook.mjs +++ b/components/flippingbook/actions/create-flipbook/create-flipbook.mjs @@ -1,12 +1,11 @@ import flippingbook from "../../flippingbook.app.mjs"; -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; export default { key: "flippingbook-create-flipbook", name: "Create Flipbook", description: "Generates a new flipbook from an input PDF file. [See the documentation](https://apidocs.flippingbook.com/#create-a-new-publication-possibly-attaching-a-new-source-file)", - version: "0.0.1", + version: "1.0.0", type: "action", props: { flippingbook, @@ -22,12 +21,6 @@ export default { "info", ], }, - fileUrl: { - propDefinition: [ - flippingbook, - "fileUrl", - ], - }, filePath: { propDefinition: [ flippingbook, @@ -47,23 +40,23 @@ export default { ], }, }, + methods: { + async getFileData() { + const stream = await getFileStream(this.filePath); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + return Buffer.concat(chunks).toString("base64"); + }, + }, async run({ $ }) { - if ((!this.fileUrl && !this.filePath) || (this.fileUrl && this.filePath)) { - throw new ConfigurationError("Please provide exactly one of File URL or File Path"); - } const data = { filename: this.filename, name: this.name, description: this.description, + data: await this.getFileData(), }; - if (this.fileUrl) { - data.url = this.fileUrl; - } else { - const content = fs.readFileSync(this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`); - data.data = content.toString("base64"); - } const response = await this.flippingbook.createFlipbook({ $, diff --git a/components/flippingbook/actions/update-flipbook/update-flipbook.mjs b/components/flippingbook/actions/update-flipbook/update-flipbook.mjs index 8e1b31a6a527a..c77aabde59bcb 100644 --- a/components/flippingbook/actions/update-flipbook/update-flipbook.mjs +++ b/components/flippingbook/actions/update-flipbook/update-flipbook.mjs @@ -1,12 +1,11 @@ import flippingbook from "../../flippingbook.app.mjs"; -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; +import createFlipbook from "../create-flipbook/create-flipbook.mjs"; export default { key: "flippingbook-update-flipbook", name: "Update Flipbook", description: "Edits an existing flipbook by replacing it with a new input PDF file. [See the documentation](https://apidocs.flippingbook.com/#update-the-metadata-for-one-publication-possibly-attaching-a-new-source-file)", - version: "0.0.1", + version: "1.0.0", type: "action", props: { flippingbook, @@ -22,12 +21,6 @@ export default { "info", ], }, - fileUrl: { - propDefinition: [ - flippingbook, - "fileUrl", - ], - }, filePath: { propDefinition: [ flippingbook, @@ -54,23 +47,14 @@ export default { ], }, }, + methods: createFlipbook.methods, async run({ $ }) { - if ((!this.fileUrl && !this.filePath) || (this.fileUrl && this.filePath)) { - throw new ConfigurationError("Please provide exactly one of File URL or File Path"); - } const data = { filename: this.filename, name: this.name, description: this.description, + data: await this.getFileData(), }; - if (this.fileUrl) { - data.url = this.fileUrl; - } else { - const content = fs.readFileSync(this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`); - data.data = content.toString("base64"); - } const response = await this.flippingbook.updateFlipbook({ $, diff --git a/components/flippingbook/flippingbook.app.mjs b/components/flippingbook/flippingbook.app.mjs index 5f6079ce6a759..45156a8f29160 100644 --- a/components/flippingbook/flippingbook.app.mjs +++ b/components/flippingbook/flippingbook.app.mjs @@ -34,17 +34,10 @@ export default { alertType: "info", content: "Please provide exactly one of File URL or File Path.", }, - fileUrl: { - type: "string", - label: "File URL", - description: "The URL of the .pdf file that will be used to create the source for the publication. The URL must be publicly accessible for at least for several minutes.", - optional: true, - }, filePath: { type: "string", - label: "File Path", - description: "The path to a .pdf file in the `/tmp` directory be used to create the source for the publication. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", - optional: true, + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, filename: { type: "string", diff --git a/components/flippingbook/package.json b/components/flippingbook/package.json index ba1607744d054..2b59a03f4724f 100644 --- a/components/flippingbook/package.json +++ b/components/flippingbook/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/flippingbook", - "version": "0.1.0", + "version": "1.0.0", "description": "Pipedream FlippingBook Components", "main": "flippingbook.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^2.0.0" + "@pipedream/platform": "^3.1.0" } } From 38a9f0d537101127c0ddd4746ca4ff8f1bac4d8b Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Tue, 10 Jun 2025 22:44:20 -0300 Subject: [PATCH 29/47] add: happy_scribe, ilovepdf --- .../actions/submit-file/submit-file.mjs | 11 ++-- components/happy_scribe/package.json | 4 +- .../actions/process-files/process-files.mjs | 57 +++++-------------- components/ilovepdf/ilovepdf.app.mjs | 11 +--- components/ilovepdf/package.json | 4 +- 5 files changed, 26 insertions(+), 61 deletions(-) diff --git a/components/happy_scribe/actions/submit-file/submit-file.mjs b/components/happy_scribe/actions/submit-file/submit-file.mjs index ac2a2adef4d36..a5c3d5f157f27 100644 --- a/components/happy_scribe/actions/submit-file/submit-file.mjs +++ b/components/happy_scribe/actions/submit-file/submit-file.mjs @@ -1,9 +1,9 @@ -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; import app from "../../happy_scribe.app.mjs"; export default { name: "Submit File", - version: "0.0.1", + version: "0.1.0", key: "happy_scribe-submit-file", description: "Submit a file. [See the documentation](https://dev.happyscribe.com/sections/product/#uploads-2-upload-your-file-with-the-signed-url)", type: "action", @@ -16,15 +16,16 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path of the file you want to upload. E.g. `/tmp/filename.aac`", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, async run({ $ }) { + const data = await getFileStream(this.filePath); const response = await this.app.uploadFile({ $, filename: this.filename, - data: fs.readFileSync(this.filePath), + data, }); $.export("$summary", `Successfully uploaded file from ${this.filePath}`); diff --git a/components/happy_scribe/package.json b/components/happy_scribe/package.json index d0dbabb592d74..fdfebb559a42f 100644 --- a/components/happy_scribe/package.json +++ b/components/happy_scribe/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/happy_scribe", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Happy Scribe Components", "main": "happy_scribe.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.5.1" + "@pipedream/platform": "^3.1.0" } } diff --git a/components/ilovepdf/actions/process-files/process-files.mjs b/components/ilovepdf/actions/process-files/process-files.mjs index cbc05d5418638..7b14a1edfb029 100644 --- a/components/ilovepdf/actions/process-files/process-files.mjs +++ b/components/ilovepdf/actions/process-files/process-files.mjs @@ -1,22 +1,16 @@ import ilovepdf from "../../ilovepdf.app.mjs"; import FormData from "form-data"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import fs from "fs"; -import { ConfigurationError } from "@pipedream/platform"; export default { key: "ilovepdf-process-files", name: "Process Files", description: "Process one or more files with the desired tool. [See the documentation](https://developer.ilovepdf.com/docs/api-reference)", - version: "0.0.1", + version: "1.0.0", type: "action", props: { ilovepdf, - fileUrls: { - propDefinition: [ - ilovepdf, - "fileUrls", - ], - }, filePaths: { propDefinition: [ ilovepdf, @@ -44,7 +38,7 @@ export default { }, async run({ $ }) { const { - fileUrls, filePaths, tool, extraParameters, outputFilename, + filePaths, tool, extraParameters, outputFilename, } = this; const { token } = await this.ilovepdf.getAuthToken({ @@ -60,55 +54,32 @@ export default { tool, }); - if (!filePaths?.length && !fileUrls.length) { - throw new ConfigurationError("You must provide either a File Path or File URL"); - } - - const fileNames = [ - ...filePaths ?? [], - ...fileUrls ?? [], - ].map((f) => f.split("/").pop()); + const fileNames = filePaths.map((f) => f.split("/").pop()); // Upload the files const pathUploads = (filePaths ?? []).map(async (filePath) => { const formData = new FormData(); formData.append("task", task); - const content = fs.createReadStream(filePath.includes("tmp/") - ? filePath - : `/tmp/${filePath}`); - formData.append("file", content); - const headers = { - "Content-Type": `multipart/form-data; boundary=${formData._boundary}`, - }; - return this.ilovepdf.uploadFile({ - $, - token, - server, - data: formData, - headers, + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); + formData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, }); - }); - const urlUploads = (fileUrls ?? []).map(async (fileUrl) => { return this.ilovepdf.uploadFile({ $, token, server, - data: { - task, - cloud_file: fileUrl, - }, - headers: { - "Content-Type": "application/json", - }, + data: formData, + headers: formData.getHeaders(), }); }); - const uploadResponses = await Promise.all([ - ...pathUploads, - ...urlUploads, - ]); + const uploadResponses = await Promise.all(pathUploads); const serverFilenames = uploadResponses.map((response) => response?.server_filename); diff --git a/components/ilovepdf/ilovepdf.app.mjs b/components/ilovepdf/ilovepdf.app.mjs index 0654024542a07..85351a85af3b0 100644 --- a/components/ilovepdf/ilovepdf.app.mjs +++ b/components/ilovepdf/ilovepdf.app.mjs @@ -4,17 +4,10 @@ export default { type: "app", app: "ilovepdf", propDefinitions: { - fileUrls: { - type: "string[]", - label: "File URLs", - description: "The URL(s) of the file(s) to be processed.", - optional: true, - }, filePaths: { type: "string[]", - label: "File Paths", - description: "The path(s) to file(s) in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp).", - optional: true, + label: "File Paths or URLs", + description: "The files to process. For each entry, provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`)", }, tool: { type: "string", diff --git a/components/ilovepdf/package.json b/components/ilovepdf/package.json index 829dbb173396e..25a3c92da547c 100644 --- a/components/ilovepdf/package.json +++ b/components/ilovepdf/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/ilovepdf", - "version": "0.1.0", + "version": "1.0.0", "description": "Pipedream iLovePDF Components", "main": "ilovepdf.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "1.5.1", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } From 6106cd9489fa8342f66801678aec719a349c6507 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Tue, 10 Jun 2025 23:48:46 -0300 Subject: [PATCH 30/47] add: lokalise, podio, zoho_crm --- .../actions/upload-file/upload-file.mjs | 20 +++++---- components/lokalise/package.json | 4 +- .../podio/actions/upload-file/upload-file.mjs | 25 ++++++----- components/podio/package.json | 4 +- .../upload-attachment/upload-attachment.mjs | 44 ++++++------------- components/zoho_crm/package.json | 4 +- 6 files changed, 43 insertions(+), 58 deletions(-) diff --git a/components/lokalise/actions/upload-file/upload-file.mjs b/components/lokalise/actions/upload-file/upload-file.mjs index 510bf37d2734a..8e2d2dd887c0c 100644 --- a/components/lokalise/actions/upload-file/upload-file.mjs +++ b/components/lokalise/actions/upload-file/upload-file.mjs @@ -1,11 +1,11 @@ import lokalise from "../../lokalise.app.mjs"; -import fs from "fs"; +import { getFileStream } from "@pipedream/platform"; export default { key: "lokalise-upload-file", name: "Upload File", description: "Uploads a specified file to a Lokalise project. [See the documentation](https://developers.lokalise.com/reference/upload-a-file)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { lokalise, @@ -17,8 +17,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to a file of a [supported file format](https://docs.lokalise.com/en/collections/2909229-supported-file-formats) in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp).", + label: "File Path or URL", + description: "The file to upload (see the [Lokalise documentation for supported file formats](https://docs.lokalise.com/en/collections/2909229-supported-file-formats)). Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`). Must be a [supported file format](https://docs.lokalise.com/en/collections/2909229-supported-file-formats).", }, language: { propDefinition: [ @@ -33,11 +33,13 @@ export default { }, }, async run({ $ }) { - const fileData = fs.readFileSync(this.filePath.startsWith("/tmp") - ? this.filePath - : `/tmp/${this.filePath}`, { - encoding: "base64", - }); + const stream = await getFileStream(this.filePath); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const fileData = Buffer.concat(chunks).toString("base64"); + const response = await this.lokalise.uploadFile({ $, projectId: this.projectId, diff --git a/components/lokalise/package.json b/components/lokalise/package.json index c8853ae704b26..50cbe0eabdd43 100644 --- a/components/lokalise/package.json +++ b/components/lokalise/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/lokalise", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Lokalise Components", "main": "lokalise.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3" + "@pipedream/platform": "^3.1.0" } } diff --git a/components/podio/actions/upload-file/upload-file.mjs b/components/podio/actions/upload-file/upload-file.mjs index 565866d3f769b..c374df4b9e662 100644 --- a/components/podio/actions/upload-file/upload-file.mjs +++ b/components/podio/actions/upload-file/upload-file.mjs @@ -1,19 +1,19 @@ import app from "../../podio.app.mjs"; import FormData from "form-data"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "podio-upload-file", name: "Upload File", description: "Uploads a new file to Podio. [See the documentation](https://developers.podio.com/doc/files/upload-file-1004361)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { app, filePath: { type: "string", - label: "File Path", - description: "The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp).", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, fileName: { type: "string", @@ -35,15 +35,16 @@ export default { const data = new FormData(); const { - filePath, fileName, - } = this; + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + const fileName = this.fileName || metadata.name; - const content = fs.createReadStream(filePath.includes("tmp/") - ? filePath - : `/tmp/${filePath}`); - - data.append("source", content); - data.append("filename", fileName ?? this.filePath.split("/").pop()); + data.append("source", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: fileName, + }); + data.append("filename", fileName); const response = await this.uploadFile({ $, diff --git a/components/podio/package.json b/components/podio/package.json index 1fce817801485..5b2d986513ff9 100644 --- a/components/podio/package.json +++ b/components/podio/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/podio", - "version": "0.2.0", + "version": "0.3.0", "description": "Pipedream Podio Components", "main": "podio.app.mjs", "keywords": [ @@ -10,7 +10,7 @@ "homepage": "https://pipedream.com/apps/podio", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^1.5.1", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" }, "publishConfig": { diff --git a/components/zoho_crm/actions/upload-attachment/upload-attachment.mjs b/components/zoho_crm/actions/upload-attachment/upload-attachment.mjs index e050b229d6875..50567d01b0ce8 100644 --- a/components/zoho_crm/actions/upload-attachment/upload-attachment.mjs +++ b/components/zoho_crm/actions/upload-attachment/upload-attachment.mjs @@ -1,13 +1,12 @@ import zohoCrm from "../../zoho_crm.app.mjs"; -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "zoho_crm-upload-attachment", name: "Upload Attachment", description: "Uploads an attachment file to Zoho CRM from a URL or file path. [See the documentation](https://www.zoho.com/crm/developer/docs/api/v3/upload-attachment.html)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { zohoCrm, @@ -26,40 +25,23 @@ export default { }), ], }, - url: { - type: "string", - label: "File URL", - description: "Source URL of the file to fetch and upload", - optional: true, - }, filePath: { type: "string", - label: "File Path", - description: "File path of a file previously downloaded in Pipedream E.g. (`/tmp/my-file.txt`). [Download a file to the `/tmp` directory](https://pipedream.com/docs/code/nodejs/http-requests/#download-a-file-to-the-tmp-directory)", - optional: true, - }, - }, - methods: { - formatFilePath(path) { - if (path.includes("/tmp/")) { - return path; - } - return `/tmp/${path}`; + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, async run({ $ }) { - if (!this.url && !this.filePath) { - throw new ConfigurationError("Either File URL or File Path must be entered."); - } - - let data = new FormData(); + const data = new FormData(); - if (this.filePath) { - const file = fs.createReadStream(this.formatFilePath(this.filePath)); - data.append("file", file); - } else { - data.append("attachmentUrl", this.url); - } + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + data.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); const response = await this.zohoCrm.uploadAttachment(this.module, this.recordId, data, $); diff --git a/components/zoho_crm/package.json b/components/zoho_crm/package.json index 5354b3dc2d8c2..1750c0532faa0 100644 --- a/components/zoho_crm/package.json +++ b/components/zoho_crm/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/zoho_crm", - "version": "0.5.3", + "version": "0.6.0", "description": "Pipedream Zoho CRM Components", "main": "zoho_crm.app.mjs", "keywords": [ @@ -10,7 +10,7 @@ "homepage": "https://pipedream.com/apps/zoho_crm", "author": "Pipedream (https://pipedream.com/)", "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "crypto": "^1.0.1", "lodash.sortby": "^4.7.0" }, From 9aa479e4c19e714ba85fbaa1aae4dbf18f59de94 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 11 Jun 2025 00:35:01 -0300 Subject: [PATCH 31/47] add: openai, imagga --- .../actions/analyze-image/analyze-image.mjs | 12 +-- .../actions/process-batch/process-batch.mjs | 15 +-- components/imagga/common/utils.mjs | 93 +++++++------------ components/imagga/imagga.app.mjs | 9 +- components/imagga/package.json | 2 +- components/openai/actions/chat/chat.mjs | 4 +- components/openai/actions/common/common.mjs | 16 ++-- components/openai/package.json | 2 +- 8 files changed, 55 insertions(+), 98 deletions(-) diff --git a/components/imagga/actions/analyze-image/analyze-image.mjs b/components/imagga/actions/analyze-image/analyze-image.mjs index 4409eaf7f852a..bb1f034c8668c 100644 --- a/components/imagga/actions/analyze-image/analyze-image.mjs +++ b/components/imagga/actions/analyze-image/analyze-image.mjs @@ -6,23 +6,15 @@ export default { key: "imagga-analyze-image", name: "Analyze Image", description: "Assign a category to a single image based on its visual content. [See the documentation](https://docs.imagga.com/?shell#categories-categorizer_id)", - version: "0.0.1", + version: "1.0.0", type: "action", props: { imagga, - imageUrl: { - propDefinition: [ - imagga, - "imageUrl", - ], - optional: true, - }, imageFile: { propDefinition: [ imagga, "imageFile", ], - optional: true, }, imageProcessType: { propDefinition: [ @@ -41,7 +33,6 @@ export default { async run({ $ }) { const { imagga, - imageUrl, imageFile, language = [], saveIndex, @@ -61,7 +52,6 @@ export default { } = this; const file = await prepareFile({ - imageUrl, imageFile, imagga, }); diff --git a/components/imagga/actions/process-batch/process-batch.mjs b/components/imagga/actions/process-batch/process-batch.mjs index 15ef2909507c9..e3138d47bfa05 100644 --- a/components/imagga/actions/process-batch/process-batch.mjs +++ b/components/imagga/actions/process-batch/process-batch.mjs @@ -9,22 +9,15 @@ export default { key: "imagga-process-batch", name: "Process Batch of Images", description: "Analyzes a batch of images for categorization, tagging, or color extraction. [See the documentation](https://docs.imagga.com/)", - version: "0.0.1", + version: "1.0.0", type: "action", props: { imagga, - imageUrl: { - label: "Image URLs", - description: "The array of Image URLs to analyze.", - type: "string[]", - optional: true, - }, imageFile: { - label: "Image File Paths", + label: "File Paths or URLs", description: - "The array of Image file path from [`/tmp` dir](https://pipedream.com/docs/code/nodejs/working-with-files/#the-tmp-directory) to analyze.", + "The image files to process. For each entry, provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", type: "string[]", - optional: true, }, callbackUrl: { propDefinition: [ @@ -51,7 +44,6 @@ export default { async run({ $ }) { const { imagga, - imageUrl, imageFile, language = [], saveIndex, @@ -71,7 +63,6 @@ export default { } = this; const file = await prepareBatchFile({ - imageUrl, imageFile, imagga, }); diff --git a/components/imagga/common/utils.mjs b/components/imagga/common/utils.mjs index 841a6863534d0..4ea705f8ecd81 100644 --- a/components/imagga/common/utils.mjs +++ b/components/imagga/common/utils.mjs @@ -1,75 +1,52 @@ import FormData from "form-data"; -import fs from "fs"; - -const checkTmp = (filename) => { - if (!filename.startsWith("/tmp")) { - return `/tmp/${filename}`; - } - return filename; -}; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export const prepareBatchFile = async ({ - imageUrl, imageFile, imagga, + imageFile, imagga, }) => { - if ((!imageUrl && !imageFile) || (imageUrl && imageFile)) { - throw new Error("You must provide either an Image URL or an Image File."); - } - - if (imageFile) { - const imageList = await imageFile.reduce(async (acc, cur) => { - await delay(1000); - const uploadId = await uploadFile({ - imageFile: cur, - imagga, - }); - - return [ - ...await acc, - uploadId, - ]; - }, []); - - return imageList.map((item) => ({ - params: { - image_upload_id: item, - }, - })); - } else { - return imageUrl.map((item) => ({ - params: { - image_url: item, - }, - })); - } + const imageList = await imageFile.reduce(async (acc, cur) => { + await delay(1000); + const uploadId = await uploadFile({ + imageFile: cur, + imagga, + }); + + return [ + ...await acc, + uploadId, + ]; + }, []); + + return imageList.map((item) => ({ + params: { + image_upload_id: item, + }, + })); }; export const prepareFile = async ({ - imageUrl, imageFile, imagga, + imageFile, imagga, }) => { - if ((!imageUrl && !imageFile) || (imageUrl && imageFile)) { - throw new Error("You must provide either an Image URL or an Image File."); - } - - if (imageFile) { - return { - image_upload_id: await uploadFile({ - imageFile, - imagga, - }), - }; - } else { - return { - image_url: imageUrl, - }; - } + return { + image_upload_id: await uploadFile({ + imageFile, + imagga, + }), + }; }; const uploadFile = async ({ imageFile, imagga, }) => { const formData = new FormData(); - const content = fs.createReadStream(checkTmp(imageFile)); - formData.append("image", content); + const { + stream, metadata, + } = await getFileStreamAndMetadata(imageFile); + formData.append("image", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); const { result } = await imagga.uploadImage({ data: formData, diff --git a/components/imagga/imagga.app.mjs b/components/imagga/imagga.app.mjs index 1a882c983c964..94b1f9c9a76db 100644 --- a/components/imagga/imagga.app.mjs +++ b/components/imagga/imagga.app.mjs @@ -29,15 +29,10 @@ export default { }, ], }, - imageUrl: { - type: "string", - label: "Image URL", - description: "The URL of the image to analyze.", - }, imageFile: { type: "string", - label: "Image File", - description: "The file path from [`/tmp` dir](https://pipedream.com/docs/code/nodejs/working-with-files/#the-tmp-directory) of the image to analyze.", + label: "File Path or URL", + description: "The image to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, callbackUrl: { type: "string", diff --git a/components/imagga/package.json b/components/imagga/package.json index 2e21b3f760848..f220574baa0a1 100644 --- a/components/imagga/package.json +++ b/components/imagga/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/imagga", - "version": "0.1.0", + "version": "1.0.0", "description": "Pipedream Imagga Components", "main": "imagga.app.mjs", "keywords": [ diff --git a/components/openai/actions/chat/chat.mjs b/components/openai/actions/chat/chat.mjs index c91d7a0461a8d..36fc0fffe6c78 100644 --- a/components/openai/actions/chat/chat.mjs +++ b/components/openai/actions/chat/chat.mjs @@ -6,7 +6,7 @@ import { ConfigurationError } from "@pipedream/platform"; export default { ...common, name: "Chat", - version: "0.2.8", + version: "0.3.0", key: "openai-chat", description: "The Chat API, using the `gpt-3.5-turbo` or `gpt-4` model. [See the documentation](https://platform.openai.com/docs/api-reference/chat)", type: "action", @@ -50,7 +50,7 @@ export default { audio: { type: "string", label: "Audio", - description: "Provide the file path to an audio file in the `/tmp` directory. For use with the `gpt-4o-audio-preview` model. Currently supports `wav` and `mp3` files.", + description: "The audio file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.mp3`). For use with the `gpt-4o-audio-preview` model. Currently supports `wav` and `mp3` files.", optional: true, }, responseFormat: { diff --git a/components/openai/actions/common/common.mjs b/components/openai/actions/common/common.mjs index 41474b809d1d6..88f85e77a8fa1 100644 --- a/components/openai/actions/common/common.mjs +++ b/components/openai/actions/common/common.mjs @@ -1,7 +1,8 @@ -import { ConfigurationError } from "@pipedream/platform"; +import { + ConfigurationError, getFileStream, +} from "@pipedream/platform"; import constants from "../../common/constants.mjs"; import { parse } from "../../common/helpers.mjs"; -import fs from "fs"; const CHAT_DOCS_MESSAGE_FORMAT_URL = "https://platform.openai.com/docs/guides/chat/introduction"; @@ -80,7 +81,7 @@ export default { user: this.user, }; }, - _getUserMessageContent() { + async _getUserMessageContent() { let content = []; if (this.images) { for (const image of this.images) { @@ -94,9 +95,12 @@ export default { } if (this.audio) { - const fileContent = fs.readFileSync(this.audio.includes("tmp/") - ? this.audio - : `/tmp/${this.audio}`).toString("base64"); + const stream = await getFileStream(this.audio); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const fileContent = Buffer.concat(chunks).toString("base64"); const extension = this.audio.match(/\.(\w+)$/)?.[1]; content.push({ type: "input_audio", diff --git a/components/openai/package.json b/components/openai/package.json index ae27551143732..3b5ce66d7f634 100644 --- a/components/openai/package.json +++ b/components/openai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/openai", - "version": "0.10.0", + "version": "1.0.0", "description": "Pipedream OpenAI Components", "main": "openai.app.mjs", "keywords": [ From 70082f33d1bdaaaa0bb26e013fbed3d3866929d4 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 11 Jun 2025 02:07:12 -0300 Subject: [PATCH 32/47] add: google_drive --- .../actions/update-file/update-file.mjs | 65 ++----------------- .../actions/upload-file/upload-file.mjs | 38 ++--------- .../google_drive/common/filePathOrUrl.mjs | 27 -------- components/google_drive/common/utils.mjs | 31 --------- components/google_drive/google_drive.app.mjs | 18 +---- components/google_drive/package.json | 2 +- 6 files changed, 16 insertions(+), 165 deletions(-) delete mode 100644 components/google_drive/common/filePathOrUrl.mjs diff --git a/components/google_drive/actions/update-file/update-file.mjs b/components/google_drive/actions/update-file/update-file.mjs index 58ad55506e22f..146585b2e4114 100644 --- a/components/google_drive/actions/update-file/update-file.mjs +++ b/components/google_drive/actions/update-file/update-file.mjs @@ -1,39 +1,15 @@ import googleDrive from "../../google_drive.app.mjs"; -import { - toSingleLineString, - getFileStream, -} from "../../common/utils.mjs"; -import { additionalProps } from "../../common/filePathOrUrl.mjs"; +import { toSingleLineString } from "../../common/utils.mjs"; +import { getFileStream } from "@pipedream/platform"; export default { key: "google_drive-update-file", name: "Update File", description: "Update a file's metadata and/or content. [See the documentation](https://developers.google.com/drive/api/v3/reference/files/update) for more information", - version: "1.0.2", + version: "2.0.0", type: "action", - additionalProps, props: { googleDrive, - updateType: { - type: "string", - label: "Update Type", - description: "Whether to update content or metadata only", - options: [ - { - label: "Upload content from File URL", - value: "File URL", - }, - { - label: "Upload content from File Path", - value: "File Path", - }, - { - label: "Update file metadata only", - value: "File Metadata", - }, - ], - reloadProps: true, - }, drive: { propDefinition: [ googleDrive, @@ -51,27 +27,12 @@ export default { ], description: "The file to update", }, - fileUrl: { - propDefinition: [ - googleDrive, - "fileUrl", - ], - description: "The URL of the file to use to update content", - optional: false, - hidden: true, - }, filePath: { propDefinition: [ googleDrive, "filePath", ], - description: toSingleLineString(` - The path to the file saved to the [\`/tmp\` - directory](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory) - (e.g., \`/tmp/myFile.csv\`) with which to update content - `), - optional: false, - hidden: true, + optional: true, }, name: { propDefinition: [ @@ -139,7 +100,6 @@ export default { async run({ $ }) { const { fileId, - fileUrl, filePath, name, mimeType, @@ -149,26 +109,13 @@ export default { ocrLanguage, useContentAsIndexableText, advanced, - updateType, } = this; - let fileStream; - if (updateType === "File URL") { - fileStream = await getFileStream({ - $, - fileUrl, - }); - } else if (updateType === "File Path") { - fileStream = await getFileStream({ - $, - filePath, - }); - } - // Update file content, if set, separately from metadata to prevent // multipart upload, which `google-apis-nodejs-client` doesn't seem to // support for [files.update](https://bit.ly/3lP5sWn) - if (fileStream) { + if (filePath) { + const fileStream = await getFileStream(filePath); await this.googleDrive.updateFileMedia(fileId, fileStream, { mimeType, }); diff --git a/components/google_drive/actions/upload-file/upload-file.mjs b/components/google_drive/actions/upload-file/upload-file.mjs index b6f05be40638b..b9597497a1f19 100644 --- a/components/google_drive/actions/upload-file/upload-file.mjs +++ b/components/google_drive/actions/upload-file/upload-file.mjs @@ -1,24 +1,16 @@ import googleDrive from "../../google_drive.app.mjs"; -import path from "path"; -import { - getFileStream, - omitEmptyStringValues, -} from "../../common/utils.mjs"; +import { omitEmptyStringValues } from "../../common/utils.mjs"; import { GOOGLE_DRIVE_UPLOAD_TYPE_MULTIPART } from "../../common/constants.mjs"; -import { - additionalProps, updateType, -} from "../../common/filePathOrUrl.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "google_drive-upload-file", name: "Upload File", description: "Upload a file to Google Drive. [See the documentation](https://developers.google.com/drive/api/v3/manage-uploads) for more information", - version: "1.0.3", + version: "2.0.0", type: "action", - additionalProps, props: { googleDrive, - updateType, drive: { propDefinition: [ googleDrive, @@ -38,21 +30,11 @@ export default { "The folder you want to upload the file to. If not specified, the file will be placed directly in the drive's top-level folder.", optional: true, }, - fileUrl: { - propDefinition: [ - googleDrive, - "fileUrl", - ], - optional: true, - hidden: true, - }, filePath: { propDefinition: [ googleDrive, "filePath", ], - optional: true, - hidden: true, }, name: { propDefinition: [ @@ -91,7 +73,6 @@ export default { async run({ $ }) { const { parentId, - fileUrl, filePath, name, mimeType, @@ -99,16 +80,11 @@ export default { let { uploadType } = this; const driveId = this.googleDrive.getDriveId(this.drive); - const filename = name || path.basename(fileUrl || filePath); + const { + stream: file, metadata, + } = await getFileStreamAndMetadata(filePath); - const file = await getFileStream({ - $, - fileUrl, - filePath: filePath?.startsWith("/tmp/") - ? filePath - : `/tmp/${filePath}`, - }); - console.log(`Upload type: ${uploadType}.`); + const filename = name || metadata.name; let result = null; if (this.fileId) { diff --git a/components/google_drive/common/filePathOrUrl.mjs b/components/google_drive/common/filePathOrUrl.mjs deleted file mode 100644 index 964f5580d917e..0000000000000 --- a/components/google_drive/common/filePathOrUrl.mjs +++ /dev/null @@ -1,27 +0,0 @@ -export const updateType = { - type: "string", - label: "Use File URL or File Path", - description: "Whether to upload a file from a URL or from the `/tmp` folder", - options: [ - "File URL", - "File Path", - ], - reloadProps: true, -}; - -export async function additionalProps(previousProps) { - const { updateType } = this; - - if (updateType === "File URL") { - previousProps.fileUrl.hidden = false; - previousProps.filePath.hidden = true; - } else if (updateType === "File Path") { - previousProps.fileUrl.hidden = true; - previousProps.filePath.hidden = false; - } else { - previousProps.fileUrl.hidden = true; - previousProps.filePath.hidden = true; - } - - return {}; -} diff --git a/components/google_drive/common/utils.mjs b/components/google_drive/common/utils.mjs index 9ff1989b5737f..31fbe96904704 100644 --- a/components/google_drive/common/utils.mjs +++ b/components/google_drive/common/utils.mjs @@ -1,5 +1,3 @@ -import fs from "fs"; -import { axios } from "@pipedream/platform"; import { MY_DRIVE_VALUE, LEGACY_MY_DRIVE_VALUE, @@ -58,34 +56,6 @@ function getListFilesOpts(drive, baseOpts = {}) { return opts; } -/** - * Returns a file stream from a file URL or file path to be used in Google Drive - * API calls - * - * @param {Object} opts - an object containing options for getting a file stream - * @param {String} opts.fileUrl - the url of a file to download to a Readable - * stream - * @param {String} opts.filePath - the path to a file from which to create a - * Readable stream - * @returns {stream.Readable} a Readable stream from the file URL or file path - */ -async function getFileStream({ - fileUrl, filePath, -}) { - if (fileUrl) { - try { - return await axios(this, { - url: fileUrl, - method: "GET", - responseType: "stream", - }); - } catch (e) { - throw new Error(`Status ${e.response.status} ${e.response.statusText}. ${e.message}`); - } - } - return fs.createReadStream(filePath); -} - function streamToBuffer(stream) { return new Promise((resolve, reject) => { const chunks = []; @@ -262,7 +232,6 @@ export { isMyDrive, getDriveId, getListFilesOpts, - getFileStream, omitEmptyStringValues, toSingleLineString, buildFilePaths, diff --git a/components/google_drive/google_drive.app.mjs b/components/google_drive/google_drive.app.mjs index 59d23e9eaaa6a..e49f8922bee24 100644 --- a/components/google_drive/google_drive.app.mjs +++ b/components/google_drive/google_drive.app.mjs @@ -155,24 +155,10 @@ export default { optional: true, default: false, }, - fileUrl: { - type: "string", - label: "File URL", - description: toSingleLineString(` - The URL of the file you want to upload to Google Drive. Must specify either **File URL** - or **File Path**. - `), - optional: true, - }, filePath: { type: "string", - label: "File Path", - description: toSingleLineString(` - The path to the file saved to the [\`/tmp\` - directory](https://pipedream.com/docs/workflows/steps/code/nodejs/working-with-files/#the-tmp-directory) - (e.g. \`/tmp/myFile.csv\`). Must specify either **File URL** or **File Path**. - `), - optional: true, + label: "File Path or URL", + description: "The file content to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, fileName: { type: "string", diff --git a/components/google_drive/package.json b/components/google_drive/package.json index d1374804cdda1..429b512d6af8c 100644 --- a/components/google_drive/package.json +++ b/components/google_drive/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/google_drive", - "version": "0.9.2", + "version": "1.0.0", "description": "Pipedream Google_drive Components", "main": "google_drive.app.mjs", "keywords": [ From 0ea7b35947ddbc9f71eecdcf5eaff5fe691eed12 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 11 Jun 2025 16:12:36 -0300 Subject: [PATCH 33/47] add: openai (remaining actions) --- .../analyze-image-content.mjs | 26 +++++++++---------- .../actions/create-batch/create-batch.mjs | 23 +++++++++------- .../create-transcription.mjs | 16 +++++++----- .../actions/upload-file/upload-file.mjs | 16 +++++++----- components/openai/openai.app.mjs | 4 +-- 5 files changed, 48 insertions(+), 37 deletions(-) diff --git a/components/openai/actions/analyze-image-content/analyze-image-content.mjs b/components/openai/actions/analyze-image-content/analyze-image-content.mjs index 9af99e3a03c76..3d42ab64147bd 100644 --- a/components/openai/actions/analyze-image-content/analyze-image-content.mjs +++ b/components/openai/actions/analyze-image-content/analyze-image-content.mjs @@ -1,14 +1,14 @@ import openai from "../../openai.app.mjs"; import common from "../common/common-assistants.mjs"; import FormData from "form-data"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { ...common, key: "openai-analyze-image-content", name: "Analyze Image Content", description: "Send a message or question about an image and receive a response. [See the documentation](https://platform.openai.com/docs/api-reference/runs/createThreadAndRun)", - version: "0.1.6", + version: "1.0.0", type: "action", props: { openai, @@ -17,12 +17,6 @@ export default { label: "Message", description: "The message or question to send", }, - imageUrl: { - type: "string", - label: "Image URL", - description: "The URL of the image to analyze. Must be a supported image types: jpeg, jpg, png, gif, webp", - optional: true, - }, imageFileId: { propDefinition: [ openai, @@ -35,8 +29,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The image to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`). Supported image types: jpeg, jpg, png, gif, webp", optional: true, }, }, @@ -83,11 +77,15 @@ export default { } if (this.filePath) { const fileData = new FormData(); - const content = fs.createReadStream(this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); fileData.append("purpose", "vision"); - fileData.append("file", content); + fileData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); const { id } = await this.openai.uploadFile({ $, diff --git a/components/openai/actions/create-batch/create-batch.mjs b/components/openai/actions/create-batch/create-batch.mjs index a74d0a722e99f..5e1a7be352435 100644 --- a/components/openai/actions/create-batch/create-batch.mjs +++ b/components/openai/actions/create-batch/create-batch.mjs @@ -1,14 +1,15 @@ import openai from "../../openai.app.mjs"; import constants from "../../common/constants.mjs"; -import { ConfigurationError } from "@pipedream/platform"; +import { + ConfigurationError, getFileStreamAndMetadata, +} from "@pipedream/platform"; import FormData from "form-data"; -import fs from "fs"; export default { key: "openai-create-batch", name: "Create Batch", description: "Creates and executes a batch from an uploaded file of requests. [See the documentation](https://platform.openai.com/docs/api-reference/batch/create)", - version: "0.0.9", + version: "0.1.0", type: "action", props: { openai, @@ -30,8 +31,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to a .jsonl file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The .jsonl file to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`)", optional: true, }, metadata: { @@ -49,11 +50,15 @@ export default { let fileId = this.fileId; if (this.filePath) { const fileData = new FormData(); - const content = fs.createReadStream(this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); fileData.append("purpose", "batch"); - fileData.append("file", content); + fileData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); const { id } = await this.openai.uploadFile({ $, diff --git a/components/openai/actions/create-transcription/create-transcription.mjs b/components/openai/actions/create-transcription/create-transcription.mjs index d3a574696f97b..fcfb9062495dc 100644 --- a/components/openai/actions/create-transcription/create-transcription.mjs +++ b/components/openai/actions/create-transcription/create-transcription.mjs @@ -1,12 +1,12 @@ +import { getFileStreamAndMetadata } from "@pipedream/platform"; import openai from "../../openai.app.mjs"; import FormData from "form-data"; -import fs from "fs"; export default { key: "openai-create-transcription", name: "Create Transcription", description: "Transcribes audio into the input language. [See the documentation](https://platform.openai.com/docs/api-reference/audio/createTranscription)", - version: "0.2.0", + version: "0.3.0", type: "action", props: { openai, @@ -90,11 +90,15 @@ export default { } = this; const data = new FormData(); - const content = fs.createReadStream(file.includes("tmp/") - ? file - : `/tmp/${file}`); + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); - data.append("file", content); + data.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); for (const [ key, diff --git a/components/openai/actions/upload-file/upload-file.mjs b/components/openai/actions/upload-file/upload-file.mjs index 331f441d92acd..70f1ba42009ff 100644 --- a/components/openai/actions/upload-file/upload-file.mjs +++ b/components/openai/actions/upload-file/upload-file.mjs @@ -1,12 +1,12 @@ import FormData from "form-data"; -import fs from "fs"; import openai from "../../openai.app.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "openai-upload-file", name: "Upload File", description: "Upload a file that can be used across various endpoints/features. The size of individual files can be a maximum of 512mb. [See the documentation](https://platform.openai.com/docs/api-reference/files/create)", - version: "0.0.18", + version: "0.1.0", type: "action", props: { openai, @@ -28,11 +28,15 @@ export default { file, purpose, } = this; const data = new FormData(); - const content = fs.createReadStream(file.includes("tmp/") - ? file - : `/tmp/${file}`); + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); data.append("purpose", purpose); - data.append("file", content); + data.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); const response = await this.openai.uploadFile({ $, diff --git a/components/openai/openai.app.mjs b/components/openai/openai.app.mjs index a3bc1841ee3b6..cc9fcb7df4c5f 100644 --- a/components/openai/openai.app.mjs +++ b/components/openai/openai.app.mjs @@ -255,8 +255,8 @@ export default { }, file: { type: "string", - label: "File", - description: "The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp). See the [Assistants Tools guide](https://platform.openai.com/docs/assistants/tools) to learn more about the types of files supported. The Fine-tuning API only supports `.jsonl` files.", + label: "File Path or URL", + description: "The file to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`). See the [Assistants Tools guide](https://platform.openai.com/docs/assistants/tools) to learn more about the types of files supported. The Fine-tuning API only supports `.jsonl` files.", }, purpose: { type: "string", From 5d5eb5fd49dc58a6af62eeba2645b67ed70d1937 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 11 Jun 2025 22:55:21 -0300 Subject: [PATCH 34/47] add: airparser, alttext_ai --- .../upload-document-parse.mjs | 20 +++--- components/airparser/package.json | 4 +- .../generate-alt-text/generate-alt-text.mjs | 61 +++++++++---------- components/alttext_ai/package.json | 4 +- pnpm-lock.yaml | 27 ++++---- 5 files changed, 57 insertions(+), 59 deletions(-) diff --git a/components/airparser/actions/upload-document-parse/upload-document-parse.mjs b/components/airparser/actions/upload-document-parse/upload-document-parse.mjs index 3cd2b362d44ab..03c045b78643a 100644 --- a/components/airparser/actions/upload-document-parse/upload-document-parse.mjs +++ b/components/airparser/actions/upload-document-parse/upload-document-parse.mjs @@ -1,12 +1,12 @@ import airparser from "../../airparser.app.mjs"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; import FormData from "form-data"; export default { key: "airparser-upload-document-parse", name: "Upload Document and Parse", description: "Uploads a document into the inbox for data extraction. [See the documentation](https://help.airparser.com/public-api/public-api)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { airparser, @@ -18,8 +18,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp).", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, metadata: { type: "object", @@ -29,11 +29,15 @@ export default { }, }, async run({ $ }) { - const fileStream = fs.createReadStream(this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); const data = new FormData(); - data.append("file", fileStream); + data.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); if (this.metadata) { data.append("meta", JSON.stringify(this.metadata)); } diff --git a/components/airparser/package.json b/components/airparser/package.json index 59eb36cd291b0..0adc56889271c 100644 --- a/components/airparser/package.json +++ b/components/airparser/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/airparser", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Airparser Components", "main": "airparser.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.6.0", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/components/alttext_ai/actions/generate-alt-text/generate-alt-text.mjs b/components/alttext_ai/actions/generate-alt-text/generate-alt-text.mjs index 33201179ce195..f7fb20eb9d69e 100644 --- a/components/alttext_ai/actions/generate-alt-text/generate-alt-text.mjs +++ b/components/alttext_ai/actions/generate-alt-text/generate-alt-text.mjs @@ -1,36 +1,35 @@ import alttextAi from "../../alttext_ai.app.mjs"; -import fs from "fs"; +import { + ConfigurationError, getFileStream, +} from "@pipedream/platform"; import { LANGUAGE_OPTIONS } from "../../commons/constants.mjs"; -import { ConfigurationError } from "@pipedream/platform"; export default { key: "alttext_ai-generate-alt-text", name: "Generate Alt Text", description: "Generates a descriptive alt text for a given image. [See the documentation](https://alttext.ai/apidocs#tag/Images/operation/create-image)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { alttextAi, + fileInfo: { + type: "alert", + alertType: "warning", + content: "Either `Image Data` or `Image File Path or URL` should be provided. If both are provided, `Image Data` will be used.", + }, imageData: { type: "string", label: "Image Data", description: - "The image data in base64 format. Only one of `Image Data`, `Image File Path` or `Image URL` should be specified.", - optional: true, - }, - imageUrl: { - type: "string", - label: "Image URL", - description: - "The public URL to an image. Only one of `Image URL`, `Image Data` or `Image File Path` should be specified.", + "The image data in base64 format", optional: true, }, imageFilePath: { type: "string", - label: "Image File Path", + label: "Image File Path or URL", description: - "The path to an image file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#the-tmp-directory). Only one of `Image File Path`, `Image URL` or `Image Data` should be specified.", + "The image to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myImage.jpg`)", optional: true, }, keywords: { @@ -64,31 +63,29 @@ export default { }, }, async run({ $ }) { - if ( - (!this.imageData && !this.imageFilePath && !this.imageUrl) - || (this.imageData && this.imageFilePath) - || (this.imageData && this.imageUrl) - || (this.imageFilePath && this.imageUrl) - ) { - throw new ConfigurationError("Only one of `Image Data`, `Image File Path` or `Image URL` should be specified."); + const { + imageData, imageFilePath, + } = this; + if (!imageData && !imageFilePath) { + throw new ConfigurationError("Either `Image Data` or `Image File Path or URL` should be specified."); + } + + let rawData = imageData; + if (!rawData) { + const stream = await getFileStream(this.imageInput); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const buffer = Buffer.concat(chunks); + rawData = buffer.toString("base64"); } const response = await this.alttextAi.generateAltText({ $, data: { image: { - url: this.imageUrl, - raw: - this.imageData ?? - (this.imageFilePath && - fs.readFileSync( - this.imageFilePath.includes("tmp/") - ? this.imageFilePath - : `/tmp/${this.imageFilePath}`, - { - encoding: "base64", - }, - )), + raw: rawData, }, keywords: this.keywords, negative_keywords: this.negativeKeywords, diff --git a/components/alttext_ai/package.json b/components/alttext_ai/package.json index cef2eebaf792e..4f1f47fe80e0a 100644 --- a/components/alttext_ai/package.json +++ b/components/alttext_ai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/alttext_ai", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream AltText.ai Components", "main": "alttext_ai.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.5.1" + "@pipedream/platform": "^3.1.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bffadb7c6fdd0..c5999abbf5661 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -548,8 +548,8 @@ importers: components/airparser: dependencies: '@pipedream/platform': - specifier: ^1.6.0 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -718,8 +718,8 @@ importers: components/alttext_ai: dependencies: '@pipedream/platform': - specifier: ^1.5.1 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 components/alttextify: {} @@ -1272,8 +1272,8 @@ importers: components/azure_storage: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 mime-types: specifier: ^2.1.35 version: 2.1.35 @@ -1654,8 +1654,8 @@ importers: components/boldsign: dependencies: '@pipedream/platform': - specifier: ^3.0.3 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 components/boloforms: dependencies: @@ -2079,8 +2079,8 @@ importers: components/canva: dependencies: '@pipedream/platform': - specifier: ^3.0.0 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 components/canva_enterprise: dependencies: @@ -2420,14 +2420,11 @@ importers: components/claid_ai: dependencies: '@pipedream/platform': - specifier: ^1.5.1 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 - fs: - specifier: ^0.0.1-security - version: 0.0.1-security url-exist: specifier: ^3.0.1 version: 3.0.1(web-streams-polyfill@3.3.3) From e09ce7e14c6be314208bc76333820613ea9c8d11 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 11 Jun 2025 23:31:06 -0300 Subject: [PATCH 35/47] add: azure_storage, boldsign --- .../actions/upload-blob/upload-blob.mjs | 25 +++++++++------- components/azure_storage/common/utils.mjs | 30 ------------------- components/azure_storage/package.json | 4 +-- .../send-document-template.mjs | 27 ++++++++++------- components/boldsign/package.json | 4 +-- 5 files changed, 35 insertions(+), 55 deletions(-) delete mode 100644 components/azure_storage/common/utils.mjs diff --git a/components/azure_storage/actions/upload-blob/upload-blob.mjs b/components/azure_storage/actions/upload-blob/upload-blob.mjs index 8b7b4c1dd9342..bf7e26d1f88be 100644 --- a/components/azure_storage/actions/upload-blob/upload-blob.mjs +++ b/components/azure_storage/actions/upload-blob/upload-blob.mjs @@ -1,12 +1,12 @@ import mime from "mime-types"; import app from "../../azure_storage.app.mjs"; -import utils from "../../common/utils.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "azure_storage-upload-blob", name: "Upload Blob", description: "Uploads a new blob to a specified container in Azure Storage. [See the documentation](https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob?tabs=microsoft-entra-id).", - version: "0.0.1", + version: "0.1.0", type: "action", props: { app, @@ -29,8 +29,8 @@ export default { }, filePath: { type: "string", - label: "File", - description: "The file to be uploaded, please provide a file from `/tmp` Eg. `/tmp/my-file.txt`. To upload a file to `/tmp` folder, please follow the doc [here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, methods: { @@ -51,9 +51,15 @@ export default { filePath, } = this; - const data = utils.getDataFromFile(filePath); - const fileName = utils.getFilenameFromPath(filePath); - const contentType = mime.lookup(fileName) || "application/octet-stream"; + const { + stream, metadata, + } = await getFileStreamAndMetadata(filePath); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const data = Buffer.concat(chunks); + const contentType = metadata.contentType || mime.lookup(metadata.name) || "application/octet-stream"; await uploadBlob({ $, @@ -62,12 +68,11 @@ export default { data, headers: { "x-ms-blob-type": "BlockBlob", - // "Content-Type": "text/plain; charset=UTF-8", "Content-Type": contentType, - "x-ms-blob-content-disposition": `attachment; filename=${fileName}`, + "x-ms-blob-content-disposition": `attachment; filename=${metadata.name}`, "x-ms-meta-m1": "v1", "x-ms-meta-m2": "v2", - "Content-Length": data?.length, + "Content-Length": data.length, }, }); diff --git a/components/azure_storage/common/utils.mjs b/components/azure_storage/common/utils.mjs deleted file mode 100644 index d769a6ea15ea5..0000000000000 --- a/components/azure_storage/common/utils.mjs +++ /dev/null @@ -1,30 +0,0 @@ -import fs from "fs"; -import { ConfigurationError } from "@pipedream/platform"; - -function checkTmp(filePath) { - const adjustedPath = filePath.startsWith("/tmp") - ? filePath - : `/tmp/${filePath}`; - - if (!fs.existsSync(adjustedPath)) { - throw new ConfigurationError("File does not exist!"); - } - - return adjustedPath; -} - -function getDataFromFile(filePath) { - const path = checkTmp(filePath); - const file = fs.readFileSync(path); - return file; -} - -function getFilenameFromPath(filePath) { - const pathParts = filePath.split("/"); - return pathParts[pathParts.length - 1]; -} - -export default { - getDataFromFile, - getFilenameFromPath, -}; diff --git a/components/azure_storage/package.json b/components/azure_storage/package.json index d605ccdadfa5d..58bbc792f28f1 100644 --- a/components/azure_storage/package.json +++ b/components/azure_storage/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/azure_storage", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Azure Storage Components", "main": "azure_storage.app.mjs", "keywords": [ @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "mime-types": "^2.1.35" } } diff --git a/components/boldsign/actions/send-document-template/send-document-template.mjs b/components/boldsign/actions/send-document-template/send-document-template.mjs index 6aae2961da60f..dde60f2e88afd 100644 --- a/components/boldsign/actions/send-document-template/send-document-template.mjs +++ b/components/boldsign/actions/send-document-template/send-document-template.mjs @@ -1,17 +1,15 @@ -import { ConfigurationError } from "@pipedream/platform"; -import fs from "fs"; +import { + ConfigurationError, getFileStreamAndMetadata, +} from "@pipedream/platform"; import boldsign from "../../boldsign.app.mjs"; import { DOCUMENT_DOWNLOAD_OPTIONS } from "../../common/constants.mjs"; -import { - checkTmp, - parseObject, -} from "../../common/utils.mjs"; +import { parseObject } from "../../common/utils.mjs"; export default { key: "boldsign-send-document-template", name: "Send Document Using Template", description: "Send documents for e-signature using a BoldSign template. [See the documentation](https://developers.boldsign.com/documents/send-document-from-template/?region=us)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { boldsign, @@ -155,8 +153,8 @@ export default { }, files: { type: "string[]", - label: "Files", - description: "A list of paths to files in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp).", + label: "File Paths or URLs", + description: "The files to upload. For each entry, provide either a file URL or path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", optional: true, }, fileUrls: { @@ -194,8 +192,15 @@ export default { const files = []; if (this.files) { for (const file of parseObject(this.files)) { - const filePath = fs.readFileSync(checkTmp(file), "base64"); - files.push(`data:application/${file.substr(file.length - 3)};base64,${filePath}`); + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const buffer = Buffer.concat(chunks); + files.push(`data:${metadata.contentType};base64,${buffer.toString("base64")}`); } } diff --git a/components/boldsign/package.json b/components/boldsign/package.json index 214d46128b487..589c157b7d5be 100644 --- a/components/boldsign/package.json +++ b/components/boldsign/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/boldsign", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream BoldSign Components", "main": "boldsign.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3" + "@pipedream/platform": "^3.1.0" } } From d8b5d3caf93956db7acbfab21f6e4d6997eeafdb Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 11 Jun 2025 23:41:26 -0300 Subject: [PATCH 36/47] add: canva --- .../create-design-import-job.mjs | 15 ++++++++------- .../canva/actions/upload-asset/upload-asset.mjs | 15 ++++++++------- components/canva/canva.app.mjs | 4 ++-- components/canva/package.json | 4 ++-- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/components/canva/actions/create-design-import-job/create-design-import-job.mjs b/components/canva/actions/create-design-import-job/create-design-import-job.mjs index 42c2beb4200f1..87dd36dc86485 100644 --- a/components/canva/actions/create-design-import-job/create-design-import-job.mjs +++ b/components/canva/actions/create-design-import-job/create-design-import-job.mjs @@ -1,11 +1,11 @@ import canva from "../../canva.app.mjs"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "canva-create-design-import-job", name: "Create Design Import Job", description: "Starts a new job to import an external file as a new design in Canva. [See the documentation](https://www.canva.dev/docs/connect/api-reference/design-imports/create-design-import-job/)", - version: "0.0.5", + version: "0.1.0", type: "action", props: { canva, @@ -30,19 +30,20 @@ export default { }, async run({ $ }) { const titleBase64 = Buffer.from(this.title).toString("base64"); - const filePath = this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`; + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + let response = await this.canva.importDesign({ $, headers: { "Import-Metadata": JSON.stringify({ "title_base64": titleBase64, }), - "Content-Length": fs.statSync(filePath).size, + "Content-Length": metadata.size, "Content-Type": "application/octet-stream", }, - data: fs.createReadStream(filePath), + data: stream, }); if (this.waitForCompletion) { diff --git a/components/canva/actions/upload-asset/upload-asset.mjs b/components/canva/actions/upload-asset/upload-asset.mjs index a7b5d9f448fec..5c3011037c211 100644 --- a/components/canva/actions/upload-asset/upload-asset.mjs +++ b/components/canva/actions/upload-asset/upload-asset.mjs @@ -1,11 +1,11 @@ import canva from "../../canva.app.mjs"; -import fs from "fs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "canva-upload-asset", name: "Upload Asset", description: "Uploads an asset to Canva. [See the documentation](https://www.canva.dev/docs/connect/api-reference/assets/create-asset-upload-job/)", - version: "0.0.5", + version: "0.1.0", type: "action", props: { canva, @@ -29,19 +29,20 @@ export default { }, async run({ $ }) { const nameBase64 = Buffer.from(this.name).toString("base64"); - const filePath = this.filePath.includes("tmp/") - ? this.filePath - : `/tmp/${this.filePath}`; + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.filePath); + let response = await this.canva.uploadAsset({ $, headers: { "Asset-Upload-Metadata": JSON.stringify({ "name_base64": nameBase64, }), - "Content-Length": fs.statSync(filePath).size, + "Content-Length": metadata.size, "Content-Type": "application/octet-stream", }, - data: fs.createReadStream(filePath), + data: stream, }); if (this.waitForCompletion) { diff --git a/components/canva/canva.app.mjs b/components/canva/canva.app.mjs index 00c7d6df5c3ea..aa2af2913d142 100644 --- a/components/canva/canva.app.mjs +++ b/components/canva/canva.app.mjs @@ -39,8 +39,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)", + label: "File Path or URL", + description: "The file to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, waitForCompletion: { type: "boolean", diff --git a/components/canva/package.json b/components/canva/package.json index 2fe7e79402317..a558c3aa2a1c1 100644 --- a/components/canva/package.json +++ b/components/canva/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/canva", - "version": "0.2.3", + "version": "0.3.0", "description": "Pipedream Canva Components", "main": "canva.app.mjs", "keywords": [ @@ -13,6 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.0" + "@pipedream/platform": "^3.1.0" } } From c90462e6df6ce3c329e7c57baafd37ecce700776 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Wed, 11 Jun 2025 23:48:58 -0300 Subject: [PATCH 37/47] add: claid_ai --- .../correct-color-lighting.mjs | 14 ++++++++++---- .../remove-background/remove-background.mjs | 14 ++++++++++---- .../actions/upscale-image/upscale-image.mjs | 14 ++++++++++---- components/claid_ai/claid_ai.app.mjs | 4 ++-- components/claid_ai/package.json | 5 ++--- 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/components/claid_ai/actions/correct-color-lighting/correct-color-lighting.mjs b/components/claid_ai/actions/correct-color-lighting/correct-color-lighting.mjs index ec3cca196d72f..9c0e1c0fbafc5 100644 --- a/components/claid_ai/actions/correct-color-lighting/correct-color-lighting.mjs +++ b/components/claid_ai/actions/correct-color-lighting/correct-color-lighting.mjs @@ -1,14 +1,13 @@ import FormData from "form-data"; -import fs from "fs"; import urlExists from "url-exist"; import claidAi from "../../claid_ai.app.mjs"; -import { checkTmp } from "../../common/utils.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "claid_ai-correct-color-lighting", name: "Correct Color & Lighting", description: "Automatically adjusts the color and lighting of an image by applying HDR. The result is an enhancement of the dynamic range in dark or overexposed images. [See the documentation](https://docs.claid.ai/image-editing-api/image-operations/color-adjustments)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { claidAi, @@ -100,8 +99,15 @@ export default { } if (!await urlExists(this.image)) { + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.image); const formData = new FormData(); - formData.append("file", fs.createReadStream(checkTmp(this.image))); + formData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); formData.append("data", JSON.stringify({ operations, })); diff --git a/components/claid_ai/actions/remove-background/remove-background.mjs b/components/claid_ai/actions/remove-background/remove-background.mjs index 25fde6de72782..034d5e80316eb 100644 --- a/components/claid_ai/actions/remove-background/remove-background.mjs +++ b/components/claid_ai/actions/remove-background/remove-background.mjs @@ -1,14 +1,13 @@ import FormData from "form-data"; -import fs from "fs"; import urlExists from "url-exist"; import claidAi from "../../claid_ai.app.mjs"; -import { checkTmp } from "../../common/utils.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "claid_ai-remove-background", name: "Remove Background", description: "Easily erases the image's background, effectively isolating the main subject. [See the documentation](https://docs.claid.ai/image-editing-api/image-operations/background)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { claidAi, @@ -98,8 +97,15 @@ export default { }; if (!await urlExists(this.image)) { + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.image); const formData = new FormData(); - formData.append("file", fs.createReadStream(checkTmp(this.image))); + formData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); formData.append("data", JSON.stringify({ operations, })); diff --git a/components/claid_ai/actions/upscale-image/upscale-image.mjs b/components/claid_ai/actions/upscale-image/upscale-image.mjs index d3625d6ddae2d..9c46d0753a3d4 100644 --- a/components/claid_ai/actions/upscale-image/upscale-image.mjs +++ b/components/claid_ai/actions/upscale-image/upscale-image.mjs @@ -1,14 +1,13 @@ import FormData from "form-data"; -import fs from "fs"; import urlExists from "url-exist"; import claidAi from "../../claid_ai.app.mjs"; -import { checkTmp } from "../../common/utils.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; export default { key: "claid_ai-upscale-image", name: "Upscale Image", description: "Enlarges the selected image in order to improve its resolution. By running this action, users can obtain clearer and sharper images. [See the documentation](https://docs.claid.ai/image-editing-api/image-operations/resizing)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { claidAi, @@ -57,8 +56,15 @@ export default { }; if (!await urlExists(this.image)) { + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.image); const formData = new FormData(); - formData.append("file", fs.createReadStream(checkTmp(this.image))); + formData.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); formData.append("data", JSON.stringify({ operations, })); diff --git a/components/claid_ai/claid_ai.app.mjs b/components/claid_ai/claid_ai.app.mjs index eeca62585ffed..3c864f4f42f80 100644 --- a/components/claid_ai/claid_ai.app.mjs +++ b/components/claid_ai/claid_ai.app.mjs @@ -6,8 +6,8 @@ export default { propDefinitions: { image: { type: "string", - label: "Image URL", - description: "The image URL or the path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp).", + label: "File Path or URL", + description: "The image to process. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myImage.jpg`)", }, }, methods: { diff --git a/components/claid_ai/package.json b/components/claid_ai/package.json index fe9e51013cfca..feef85c52a629 100644 --- a/components/claid_ai/package.json +++ b/components/claid_ai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/claid_ai", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Claid AI Components", "main": "claid_ai.app.mjs", "keywords": [ @@ -13,9 +13,8 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.5.1", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0", - "fs": "^0.0.1-security", "url-exist": "^3.0.1" } } From ade6f851e6e4a154a6967375c339f53ef793fd33 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Thu, 12 Jun 2025 17:47:04 -0300 Subject: [PATCH 38/47] update package.json --- components/imagga/package.json | 2 +- components/openai/package.json | 2 +- pnpm-lock.yaml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/imagga/package.json b/components/imagga/package.json index f220574baa0a1..fa1a300c640c8 100644 --- a/components/imagga/package.json +++ b/components/imagga/package.json @@ -13,7 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.5.1", + "@pipedream/platform": "^3.1.0", "form-data": "^4.0.0" } } diff --git a/components/openai/package.json b/components/openai/package.json index 3b5ce66d7f634..2f001194200d3 100644 --- a/components/openai/package.json +++ b/components/openai/package.json @@ -14,7 +14,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", + "@pipedream/platform": "^3.1.0", "@pipedream/types": "^0.1.4", "axios": "^1.6.2", "bottleneck": "^2.19.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b87f57c3b2a05..d76ba6d07752a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6518,8 +6518,8 @@ importers: components/imagga: dependencies: '@pipedream/platform': - specifier: ^1.5.1 - version: 1.6.6 + specifier: ^3.1.0 + version: 3.1.0 form-data: specifier: ^4.0.0 version: 4.0.1 @@ -9254,7 +9254,7 @@ importers: components/openai: dependencies: '@pipedream/platform': - specifier: ^3.0.3 + specifier: ^3.1.0 version: 3.1.0 '@pipedream/types': specifier: ^0.1.4 From d8e872a6957d49360e6fde222e7c54016c0bcec0 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Thu, 12 Jun 2025 19:47:23 -0300 Subject: [PATCH 39/47] pdf4me: readding std filename check to action --- components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs b/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs index b7010ed2166b0..d8880bfeb3219 100644 --- a/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs +++ b/components/pdf4me/actions/convert-to-pdf/convert-to-pdf.mjs @@ -23,13 +23,14 @@ export default { }, }, async run({ $ }) { + const filename = utils.checkForExtension(this.filename, "pdf"); const fileContent = await utils.getBase64File(this.filePath); const response = await this.pdf4me.convertToPdf({ $, data: { docContent: fileContent, - docName: this.filename, + docName: filename, }, responseType: "arraybuffer", }); From 5b7ca8ca3f146b43c29f2dd62c464003268a0fb7 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Thu, 12 Jun 2025 22:48:10 -0300 Subject: [PATCH 40/47] Adjusting zoho_desk formdata building --- .../add-ticket-attachment.mjs | 26 ++++++++++++++----- components/zoho_desk/zoho_desk.app.mjs | 19 ++------------ 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs b/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs index 8db712defab74..9ed510dca8b21 100644 --- a/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs +++ b/components/zoho_desk/actions/add-ticket-attachment/add-ticket-attachment.mjs @@ -1,5 +1,6 @@ import zohoDesk from "../../zoho_desk.app.mjs"; -import constants from "../../common/constants.mjs"; +import { getFileStreamAndMetadata } from "@pipedream/platform"; +import FormData from "form-data"; export default { key: "zoho_desk-add-ticket-attachment", @@ -44,19 +45,30 @@ export default { file, } = this; + const data = new FormData(); + const { + stream, metadata, + } = await getFileStreamAndMetadata(file); + data.append("file", stream, { + contentType: metadata.contentType, + knownLength: metadata.size, + filename: metadata.name, + }); + const response = await this.zohoDesk.createTicketAttachment({ + $, ticketId, + params: { + isPublic, + }, headers: { orgId, - ...constants.MULTIPART_FORM_DATA_HEADERS, - }, - data: { - file, - isPublic, + ...data.getHeaders(), }, + data, }); - $.export("$summary", `Successfully created a new ticket attachment with ID ${response.id}`); + $.export("$summary", `Successfully created a new ticket attachment with ID ${response?.id}`); return response; }, diff --git a/components/zoho_desk/zoho_desk.app.mjs b/components/zoho_desk/zoho_desk.app.mjs index de2840cfae167..5df21a3aa84aa 100644 --- a/components/zoho_desk/zoho_desk.app.mjs +++ b/components/zoho_desk/zoho_desk.app.mjs @@ -124,31 +124,16 @@ export default { $ = this, url, path, - headers: preHeaders, params, - data: preData, + headers, versionPath = constants.VERSION_PATH, withRetries = true, ...args } = {}) { - const contentType = constants.CONTENT_TYPE_KEY_HEADER; - - const hasMultipartHeader = utils.hasMultipartHeader(preHeaders); - const data = hasMultipartHeader && await utils.getFormData(preData) || preData; - - const currentHeaders = this.getHeaders(preHeaders); - const headers = hasMultipartHeader - ? { - ...currentHeaders, - [contentType]: data.getHeaders()[contentType.toLowerCase()], - } - : currentHeaders; - const config = { - headers, url: this.getUrl(url, path, versionPath), params: this.getParams(url, params), - data, + headers: this.getHeaders(headers), ...args, }; try { From 648cc4208d86dc3686abd3bc1fa4291968c9d6d9 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Fri, 13 Jun 2025 23:31:22 -0300 Subject: [PATCH 41/47] Merge branch 'master' into 16977-applying-fileurl-paths-improvement --- pnpm-lock.yaml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cf823f8344c48..607c8ee027d8a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -990,8 +990,7 @@ importers: specifier: ^3.0.0 version: 3.0.3 - components/arpoone: - specifiers: {} + components/arpoone: {} components/arxiv: {} @@ -10354,8 +10353,7 @@ importers: specifier: ^1.5.1 version: 1.6.6 - components/postmaster: - specifiers: {} + components/postmaster: {} components/power_automate: {} @@ -15679,14 +15677,6 @@ importers: specifier: ^6.0.0 version: 6.2.0 - modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/cjs: {} - - modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/esm: {} - - modelcontextprotocol/node_modules2/zod-to-json-schema/dist/cjs: {} - - modelcontextprotocol/node_modules2/zod-to-json-schema/dist/esm: {} - packages/ai: dependencies: '@pipedream/sdk': From c30b8b924d074fb0253e4b0903ee3f159b5bbf0f Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Fri, 13 Jun 2025 23:34:02 -0300 Subject: [PATCH 42/47] pnpm --- pnpm-lock.yaml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c680bddb6fe3b..06bea340bb689 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -990,8 +990,7 @@ importers: specifier: ^3.0.0 version: 3.0.3 - components/arpoone: - specifiers: {} + components/arpoone: {} components/arxiv: {} @@ -10348,8 +10347,7 @@ importers: specifier: ^1.5.1 version: 1.6.6 - components/postmaster: - specifiers: {} + components/postmaster: {} components/power_automate: {} @@ -15673,14 +15671,6 @@ importers: specifier: ^6.0.0 version: 6.2.0 - modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/cjs: {} - - modelcontextprotocol/node_modules2/@modelcontextprotocol/sdk/dist/esm: {} - - modelcontextprotocol/node_modules2/zod-to-json-schema/dist/cjs: {} - - modelcontextprotocol/node_modules2/zod-to-json-schema/dist/esm: {} - packages/ai: dependencies: '@pipedream/sdk': From 7f2bceb1f1a711eca0fd45a25f43748c57f9038d Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 16 Jun 2025 20:40:27 -0300 Subject: [PATCH 43/47] fix: alttext_ai, box --- .../actions/generate-alt-text/generate-alt-text.mjs | 2 +- components/box/common/common-file-upload.mjs | 2 ++ components/box/common/utils.mjs | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/alttext_ai/actions/generate-alt-text/generate-alt-text.mjs b/components/alttext_ai/actions/generate-alt-text/generate-alt-text.mjs index f7fb20eb9d69e..27d9150ba043a 100644 --- a/components/alttext_ai/actions/generate-alt-text/generate-alt-text.mjs +++ b/components/alttext_ai/actions/generate-alt-text/generate-alt-text.mjs @@ -72,7 +72,7 @@ export default { let rawData = imageData; if (!rawData) { - const stream = await getFileStream(this.imageInput); + const stream = await getFileStream(imageFilePath); const chunks = []; for await (const chunk of stream) { chunks.push(chunk); diff --git a/components/box/common/common-file-upload.mjs b/components/box/common/common-file-upload.mjs index a5e4434af54bc..0f5af9cbaec8e 100644 --- a/components/box/common/common-file-upload.mjs +++ b/components/box/common/common-file-upload.mjs @@ -27,7 +27,9 @@ export async function getFileUploadBody({ const data = new FormData(); data.append("attributes", JSON.stringify(attributes)); data.append("file", fileContent, { + contentType: fileMeta.contentType, knownLength: fileMeta.size, + filename: attributes.name, }); return data; diff --git a/components/box/common/utils.mjs b/components/box/common/utils.mjs index 2897f8d946c48..aaec30611aab9 100644 --- a/components/box/common/utils.mjs +++ b/components/box/common/utils.mjs @@ -1,4 +1,3 @@ -import path from "path"; import constants from "./constants.mjs"; import { getFileStreamAndMetadata } from "@pipedream/platform"; @@ -15,12 +14,13 @@ export default { .split(".")[0] + "Z", content_modified_at: new Date(metadata.lastModified).toISOString() .split(".")[0] + "Z", - name: path.basename(filePath), + name: metadata.name, parent: { id: 0, }, }, size: metadata.size, + contentType: metadata.contentType, }, }; }, From fab27d8b7f0d05e805f80fadef334eb50236d480 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 16 Jun 2025 21:06:48 -0300 Subject: [PATCH 44/47] description improvements: docsbot_ai, lokalise --- components/docsbot_ai/docsbot_ai.app.mjs | 4 ++-- components/lokalise/actions/upload-file/upload-file.mjs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/docsbot_ai/docsbot_ai.app.mjs b/components/docsbot_ai/docsbot_ai.app.mjs index 1480f0d94cc55..b0d03488623f8 100644 --- a/components/docsbot_ai/docsbot_ai.app.mjs +++ b/components/docsbot_ai/docsbot_ai.app.mjs @@ -122,8 +122,8 @@ export default { }, filePath: { type: "string", - label: "File Path", - description: "A file path in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/)", + label: "File Path or URL", + description: "The file to upload. Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.txt`)", }, }, methods: { diff --git a/components/lokalise/actions/upload-file/upload-file.mjs b/components/lokalise/actions/upload-file/upload-file.mjs index 8e2d2dd887c0c..1c7584d8a8db5 100644 --- a/components/lokalise/actions/upload-file/upload-file.mjs +++ b/components/lokalise/actions/upload-file/upload-file.mjs @@ -29,7 +29,7 @@ export default { filename: { type: "string", label: "Filename", - description: "Set the filename. You may optionally use a relative path in the filename", + description: "Set the filename. You may optionally use a relative path in the filename (e.g `admin/main.json`)", }, }, async run({ $ }) { From a0bc8d7fba9ae0d900b44dad1b9b6dd1b135fc7b Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Mon, 16 Jun 2025 21:53:33 -0300 Subject: [PATCH 45/47] fix: openai --- components/openai/actions/chat/chat.mjs | 7 +------ components/openai/actions/common/common.mjs | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/components/openai/actions/chat/chat.mjs b/components/openai/actions/chat/chat.mjs index caa79a09c5da2..c2a4fa315d534 100644 --- a/components/openai/actions/chat/chat.mjs +++ b/components/openai/actions/chat/chat.mjs @@ -1,7 +1,6 @@ import openai from "../../openai.app.mjs"; import common from "../common/common.mjs"; import constants from "../../common/constants.mjs"; -import { ConfigurationError } from "@pipedream/platform"; export default { ...common, @@ -145,11 +144,7 @@ export default { }, }, async run({ $ }) { - if (this.audio && !this.modelId.includes("gpt-4o-audio-preview")) { - throw new ConfigurationError("Use of audio files requires using the `gpt-4o-audio-preview` model."); - } - - const args = this._getChatArgs(); + const args = await this._getChatArgs(); const response = await this.openai.createChatCompletion({ $, diff --git a/components/openai/actions/common/common.mjs b/components/openai/actions/common/common.mjs index 88f85e77a8fa1..9d7f156413fcf 100644 --- a/components/openai/actions/common/common.mjs +++ b/components/openai/actions/common/common.mjs @@ -118,7 +118,7 @@ export default { return content; }, - _getChatArgs() { + async _getChatArgs() { if (this.messages && this.messages.length && !this.userMessage) { throw new ConfigurationError( `When you provide previous messages, you must provide the next User Message for the assistant to answer. See the OpenAI Chat format docs here: ${CHAT_DOCS_MESSAGE_FORMAT_URL}`, @@ -164,7 +164,7 @@ export default { messages.push({ "role": "user", - "content": this._getUserMessageContent(), + "content": await this._getUserMessageContent(), }); const responseFormat = {}; From b1a6f9112c20805012cde44fff7904fb5517dcd9 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Tue, 17 Jun 2025 00:09:45 -0300 Subject: [PATCH 46/47] Improving OpenAI image handling --- .../actions/extract-text/extract-text.mjs | 4 +++- components/openai/actions/chat/chat.mjs | 2 +- components/openai/actions/common/common.mjs | 12 ++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/components/llmwhisperer/actions/extract-text/extract-text.mjs b/components/llmwhisperer/actions/extract-text/extract-text.mjs index 1b16c0efb32b6..41af8acbf212e 100644 --- a/components/llmwhisperer/actions/extract-text/extract-text.mjs +++ b/components/llmwhisperer/actions/extract-text/extract-text.mjs @@ -99,7 +99,9 @@ export default { }, methods: { getHeaders() { - return "application/octet-stream"; + return { + "Content-Type": "application/octet-stream", + }; }, async getData(data) { return getFileStream(data); diff --git a/components/openai/actions/chat/chat.mjs b/components/openai/actions/chat/chat.mjs index c2a4fa315d534..558c9e0ceeb01 100644 --- a/components/openai/actions/chat/chat.mjs +++ b/components/openai/actions/chat/chat.mjs @@ -43,7 +43,7 @@ export default { images: { label: "Images", type: "string[]", - description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Accepts URLs or base64 encoded strings. Compatible with the `gpt4-vision-preview` model", + description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Each entry should be either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`) Compatible with the `gpt4-vision-preview` model", optional: true, }, audio: { diff --git a/components/openai/actions/common/common.mjs b/components/openai/actions/common/common.mjs index 9d7f156413fcf..9e44d2c616049 100644 --- a/components/openai/actions/common/common.mjs +++ b/components/openai/actions/common/common.mjs @@ -85,11 +85,15 @@ export default { let content = []; if (this.images) { for (const image of this.images) { + const stream = await getFileStream(image); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const base64Image = Buffer.concat(chunks).toString("base64"); content.push({ - "type": "image_url", - "image_url": { - "url": image, - }, + "type": "input_image", + "image_url": `data:image/jpeg;base64,${base64Image}`, }); } } From d92f62237ed695fdac049294db8cc2b82b732787 Mon Sep 17 00:00:00 2001 From: GTFalcao Date: Tue, 17 Jun 2025 03:38:40 -0300 Subject: [PATCH 47/47] openai-chat fixes --- components/openai/actions/chat/chat.mjs | 2 +- components/openai/actions/common/common.mjs | 33 ++++++++++++++------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/components/openai/actions/chat/chat.mjs b/components/openai/actions/chat/chat.mjs index 558c9e0ceeb01..872646e188989 100644 --- a/components/openai/actions/chat/chat.mjs +++ b/components/openai/actions/chat/chat.mjs @@ -43,7 +43,7 @@ export default { images: { label: "Images", type: "string[]", - description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Each entry should be either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`) Compatible with the `gpt4-vision-preview` model", + description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Each entry should be either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.jpg`), or raw base64-encoded image data. Compatible with the `gpt4-vision-preview` model", optional: true, }, audio: { diff --git a/components/openai/actions/common/common.mjs b/components/openai/actions/common/common.mjs index 9e44d2c616049..c8a81c2ac5997 100644 --- a/components/openai/actions/common/common.mjs +++ b/components/openai/actions/common/common.mjs @@ -1,5 +1,5 @@ import { - ConfigurationError, getFileStream, + ConfigurationError, getFileStreamAndMetadata, } from "@pipedream/platform"; import constants from "../../common/constants.mjs"; import { parse } from "../../common/helpers.mjs"; @@ -85,27 +85,40 @@ export default { let content = []; if (this.images) { for (const image of this.images) { - const stream = await getFileStream(image); - const chunks = []; - for await (const chunk of stream) { - chunks.push(chunk); + let base64Image = image; + let imageType = "image/jpeg"; + if (image.startsWith("http") || image.includes("tmp/")) { + const { + stream, metadata, + } = await getFileStreamAndMetadata(image); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + base64Image = Buffer.concat(chunks).toString("base64"); + if (metadata.contentType) imageType = metadata.contentType; } - const base64Image = Buffer.concat(chunks).toString("base64"); content.push({ - "type": "input_image", - "image_url": `data:image/jpeg;base64,${base64Image}`, + "type": "image_url", + "image_url": { + "url": base64Image.startsWith("data:") + ? base64Image + : `data:${imageType};base64,${base64Image}`, + }, }); } } if (this.audio) { - const stream = await getFileStream(this.audio); + const { + stream, metadata, + } = await getFileStreamAndMetadata(this.audio); const chunks = []; for await (const chunk of stream) { chunks.push(chunk); } const fileContent = Buffer.concat(chunks).toString("base64"); - const extension = this.audio.match(/\.(\w+)$/)?.[1]; + const extension = metadata.name.split(".").pop(); content.push({ type: "input_audio", input_audio: {