From dd7bb9b017ac4c2f2329939e57f65f42d960930d Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Tue, 3 Jun 2025 18:38:06 -0400 Subject: [PATCH 01/11] update package-lock and pnpm-lock --- platform/package-lock.json | 4 ++-- pnpm-lock.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/platform/package-lock.json b/platform/package-lock.json index e89c54a9a5608..93c3e858f0363 100644 --- a/platform/package-lock.json +++ b/platform/package-lock.json @@ -1,12 +1,12 @@ { "name": "@pipedream/platform", - "version": "3.0.1", + "version": "3.0.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@pipedream/platform", - "version": "3.0.1", + "version": "3.0.3", "license": "MIT", "dependencies": { "axios": "^1.7.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e11efdcd7858..604557836aaad 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29125,22 +29125,22 @@ packages: superagent@3.8.1: resolution: {integrity: sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==} engines: {node: '>= 4.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v7.0.2+ of superagent. We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing. See the releases tab for more information at . superagent@4.1.0: resolution: {integrity: sha512-FT3QLMasz0YyCd4uIi5HNe+3t/onxMyEho7C3PSqmti3Twgy2rXT4fmkTz6wRL6bTF4uzPcfkUCa8u4JWHw8Ag==} engines: {node: '>= 6.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v7.0.2+ of superagent. We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing. See the releases tab for more information at . superagent@5.3.1: resolution: {integrity: sha512-wjJ/MoTid2/RuGCOFtlacyGNxN9QLMgcpYLDQlWFIhhdJ93kNscFonGvrpAHSCVjRVj++DGCglocF7Aej1KHvQ==} engines: {node: '>= 7.0.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v7.0.2+ of superagent. We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing. See the releases tab for more information at . superagent@7.1.6: resolution: {integrity: sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==} engines: {node: '>=6.4.0 <13 || >=14'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please downgrade to v7.1.5 if you need IE/ActiveXObject support OR upgrade to v8.0.0 as we no longer support IE and published an incorrect patch version (see https://github.com/visionmedia/superagent/issues/1731) supports-color@2.0.0: resolution: {integrity: sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==} From 9e8053b9d8f015fe95ea7b00cab4ac659f4dfd76 Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Tue, 3 Jun 2025 18:38:06 -0400 Subject: [PATCH 02/11] feat(platform): add file-stream utility This commit introduces a new helper for working with file streams, providing a unified interface for reading local and remote files. --- platform/__tests__/file-stream.js | 196 ++++++++++++++++++++++++++++++ platform/dist/file-stream.d.ts | 22 ++++ platform/dist/file-stream.js | 132 ++++++++++++++++++++ platform/dist/index.d.ts | 2 + platform/dist/index.js | 3 + platform/lib/file-stream.ts | 148 ++++++++++++++++++++++ platform/lib/index.ts | 8 ++ 7 files changed, 511 insertions(+) create mode 100644 platform/__tests__/file-stream.js create mode 100644 platform/dist/file-stream.d.ts create mode 100644 platform/dist/file-stream.js create mode 100644 platform/lib/file-stream.ts diff --git a/platform/__tests__/file-stream.js b/platform/__tests__/file-stream.js new file mode 100644 index 0000000000000..2d593083da012 --- /dev/null +++ b/platform/__tests__/file-stream.js @@ -0,0 +1,196 @@ +const { + getFileStream, getFileStreamAndMetadata, +} = require("../dist"); +const fs = require("fs"); +const path = require("path"); +const http = require("http"); +const os = require("os"); + +// Helper function to read content from a readable stream +async function readStreamContent(stream) { + let content = ""; + stream.on("data", (chunk) => { + content += chunk.toString(); + }); + + await new Promise((resolve) => { + stream.on("end", resolve); + }); + + return content; +} + +// Helper function to wait for stream cleanup by listening to close event +async function waitForStreamCleanup(stream) { + return new Promise((resolve) => { + stream.on("close", resolve); + }); +} + +describe("file-stream", () => { + let testFilePath; + let server; + const testPort = 3892; + + beforeAll(() => { + // Create a test file + testFilePath = path.join(__dirname, "test-file.txt"); + fs.writeFileSync(testFilePath, "test content for file stream"); + + // Create a simple HTTP server for testing remote files + server = http.createServer((req, res) => { + if (req.url === "/test-file.txt") { + res.writeHead(200, { + "Content-Type": "text/plain", + "Content-Length": "28", + "Last-Modified": new Date().toUTCString(), + "ETag": "\"test-etag\"", + }); + res.end("test content for file stream"); + } else if (req.url === "/no-content-length") { + res.writeHead(200, { + "Content-Type": "application/json", + "Last-Modified": new Date().toUTCString(), + }); + res.end("{\"test\": \"data\"}"); + } else if (req.url === "/error") { + res.writeHead(404, "Not Found"); + res.end(); + } else { + res.writeHead(404, "Not Found"); + res.end(); + } + }); + + return new Promise((resolve) => + server.listen(testPort, resolve)); + }); + + afterAll(() => { + // Clean up test file + if (fs.existsSync(testFilePath)) { + fs.unlinkSync(testFilePath); + } + + if (server) { + return new Promise((resolve) => + server.close(resolve)); + } + }); + + describe("getFileStream", () => { + it("should return readable stream for local file", async () => { + const stream = await getFileStream(testFilePath); + expect(stream).toBeDefined(); + expect(typeof stream.read).toBe("function"); + + const content = await readStreamContent(stream); + expect(content).toBe("test content for file stream"); + }); + + it("should return readable stream for remote URL", async () => { + const stream = await getFileStream(`http://localhost:${testPort}/test-file.txt`); + expect(stream).toBeDefined(); + expect(typeof stream.read).toBe("function"); + + const content = await readStreamContent(stream); + expect(content).toBe("test content for file stream"); + }); + + it("should throw error for invalid URL", async () => { + await expect(getFileStream(`http://localhost:${testPort}/error`)) + .rejects.toThrow("Failed to fetch"); + }); + + it("should throw error for non-existent local file", async () => { + await expect(getFileStream("/non/existent/file.txt")) + .rejects.toThrow(); + }); + }); + + describe("getFileStreamAndMetadata", () => { + it("should return stream and metadata for local file", async () => { + const result = await getFileStreamAndMetadata(testFilePath); + + expect(result.stream).toBeDefined(); + expect(typeof result.stream.read).toBe("function"); + expect(result.metadata).toMatchObject({ + size: 28, + name: "test-file.txt", + }); + expect(result.metadata.lastModified.constructor.name).toBe("Date"); + const content = await readStreamContent(result.stream); + expect(content).toBe("test content for file stream"); + }); + + it("should return stream and metadata for remote file with content-length", async () => { + const result = await getFileStreamAndMetadata(`http://localhost:${testPort}/test-file.txt`); + + expect(result.stream).toBeDefined(); + expect(typeof result.stream.read).toBe("function"); + expect(result.metadata).toMatchObject({ + size: 28, + contentType: "text/plain", + name: "test-file.txt", + etag: "\"test-etag\"", + }); + expect(result.metadata.lastModified).toBeInstanceOf(Date); + const content = await readStreamContent(result.stream); + expect(content).toBe("test content for file stream"); + }); + + it("should handle remote file without content-length", async () => { + const result = await getFileStreamAndMetadata(`http://localhost:${testPort}/no-content-length`); + + expect(result.stream).toBeDefined(); + expect(typeof result.stream.read).toBe("function"); + + expect(result.metadata).toMatchObject({ + size: 16, // Size determined after download + contentType: "application/json", + }); + expect(result.metadata.lastModified).toBeInstanceOf(Date); + + const content = await readStreamContent(result.stream); + expect(content).toBe("{\"test\": \"data\"}"); + }); + + it("should throw error for invalid remote URL", async () => { + await expect(getFileStreamAndMetadata(`http://localhost:${testPort}/error`)) + .rejects.toThrow("Failed to fetch"); + }); + }); + + describe("temporary file cleanup", () => { + it("should clean up temporary files after stream ends", async () => { + const tmpDir = os.tmpdir(); + const tempFilesBefore = fs.readdirSync(tmpDir); + const result = await getFileStreamAndMetadata(`http://localhost:${testPort}/no-content-length`); + + const content = await readStreamContent(result.stream); + // Wait for cleanup to complete by listening to close event + await waitForStreamCleanup(result.stream); + + // Check that temp files were cleaned up + const tempFilesAfter = fs.readdirSync(tmpDir); + expect(tempFilesAfter.length).toEqual(tempFilesBefore.length); + expect(content).toBe("{\"test\": \"data\"}"); + }); + + it("should clean up temporary files on stream error", async () => { + // Check temp files before + const tmpDir = os.tmpdir(); + const tempFilesBefore = fs.readdirSync(tmpDir); + + const result = await getFileStreamAndMetadata(`http://localhost:${testPort}/no-content-length`); + + // Trigger an error and wait for cleanup + result.stream.destroy(new Error("Test error")); + await waitForStreamCleanup(result.stream); + + // Check that temp files were cleaned up + const tempFilesAfter = fs.readdirSync(tmpDir); + expect(tempFilesAfter.length).toEqual(tempFilesBefore.length); + }); + }); +}); diff --git a/platform/dist/file-stream.d.ts b/platform/dist/file-stream.d.ts new file mode 100644 index 0000000000000..53088006cfc38 --- /dev/null +++ b/platform/dist/file-stream.d.ts @@ -0,0 +1,22 @@ +/// +import { Readable } from "stream"; +export interface FileMetadata { + size?: number; + contentType?: string; + lastModified?: Date; + name?: string; + etag?: string; +} +/** + * @param pathOrUrl - a file path or a URL + * @returns a Readable stream of the file content + */ +export declare function getFileStream(pathOrUrl: string): Promise; +/** + * @param pathOrUrl - a file path or a URL + * @returns a Readable stream of the file content and its metadata + */ +export declare function getFileStreamAndMetadata(pathOrUrl: string): Promise<{ + stream: Readable; + metadata: FileMetadata; +}>; diff --git a/platform/dist/file-stream.js b/platform/dist/file-stream.js new file mode 100644 index 0000000000000..1a90c64f71f92 --- /dev/null +++ b/platform/dist/file-stream.js @@ -0,0 +1,132 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.getFileStreamAndMetadata = exports.getFileStream = void 0; +const stream_1 = require("stream"); +const fs_1 = require("fs"); +const os_1 = require("os"); +const path_1 = require("path"); +const fs_2 = require("fs"); +const promises_1 = require("stream/promises"); +/** + * @param pathOrUrl - a file path or a URL + * @returns a Readable stream of the file content + */ +async function getFileStream(pathOrUrl) { + if (isUrl(pathOrUrl)) { + const response = await fetch(pathOrUrl); + if (!response.ok) { + throw new Error(`Failed to fetch ${pathOrUrl}: ${response.status} ${response.statusText}`); + } + return stream_1.Readable.fromWeb(response.body); + } + else { + // Check if file exists first (this will throw if file doesn't exist) + await fs_1.promises.stat(pathOrUrl); + return fs_1.createReadStream(pathOrUrl); + } +} +exports.getFileStream = getFileStream; +/** + * @param pathOrUrl - a file path or a URL + * @returns a Readable stream of the file content and its metadata + */ +async function getFileStreamAndMetadata(pathOrUrl) { + if (isUrl(pathOrUrl)) { + return await getRemoteFileStreamAndMetadata(pathOrUrl); + } + else { + return await getLocalFileStreamAndMetadata(pathOrUrl); + } +} +exports.getFileStreamAndMetadata = getFileStreamAndMetadata; +function isUrl(pathOrUrl) { + try { + new URL(pathOrUrl); + return true; + } + catch (_a) { + return false; + } +} +async function getLocalFileStreamAndMetadata(filePath) { + const stats = await fs_1.promises.stat(filePath); + const metadata = { + size: stats.size, + lastModified: stats.mtime, + name: filePath.split("/").pop() || filePath.split("\\").pop(), + }; + const stream = fs_1.createReadStream(filePath); + return { + stream, + metadata, + }; +} +async function getRemoteFileStreamAndMetadata(url) { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); + } + const headers = response.headers; + const contentLength = headers.get("content-length"); + const contentType = headers.get("content-type") || undefined; + const lastModified = headers.get("last-modified") + ? new Date(headers.get("last-modified")) + : undefined; + const etag = headers.get("etag") || undefined; + const urlObj = new URL(url); + const name = urlObj.pathname.split("/").pop() || undefined; + const baseMetadata = { + contentType, + lastModified, + name, + etag, + }; + // If we have content-length, we can stream directly + if (contentLength) { + const metadata = { + ...baseMetadata, + size: parseInt(contentLength, 10), + }; + const stream = stream_1.Readable.fromWeb(response.body); + return { + stream, + metadata, + }; + } + // No content-length header - need to download to temporary file to get size + return await downloadToTemporaryFile(response, baseMetadata); +} +async function downloadToTemporaryFile(response, baseMetadata) { + // Generate unique temporary file path + const tempFileName = `file-stream-${Date.now()}-${Math.random().toString(36) + .substring(2)}`; + const tempFilePath = path_1.join(os_1.tmpdir(), tempFileName); + // Download to temporary file + const fileStream = fs_2.createWriteStream(tempFilePath); + const webStream = stream_1.Readable.fromWeb(response.body); + await promises_1.pipeline(webStream, fileStream); + // Get file stats + const stats = await fs_1.promises.stat(tempFilePath); + const metadata = { + ...baseMetadata, + size: stats.size, + }; + // Create a readable stream that cleans up the temp file when done + const stream = fs_1.createReadStream(tempFilePath); + // Clean up temp file when stream is closed or ends + const cleanup = async () => { + try { + await fs_1.promises.unlink(tempFilePath); + } + catch (_a) { + // Ignore cleanup errors (file might already be deleted) + } + }; + stream.on("close", cleanup); + stream.on("end", cleanup); + stream.on("error", cleanup); + return { + stream, + metadata, + }; +} diff --git a/platform/dist/index.d.ts b/platform/dist/index.d.ts index a68f663ae5e1a..8cd2d6364bc0c 100644 --- a/platform/dist/index.d.ts +++ b/platform/dist/index.d.ts @@ -3,6 +3,8 @@ import axios, { transformConfigForOauth } from "./axios"; import { AxiosRequestConfig as AxiosConfig } from "axios"; export { axios, transformConfigForOauth, }; export { cloneSafe, jsonStringifySafe, } from "./utils"; +export { getFileStreamAndMetadata, getFileStream, } from "./file-stream"; +export type { FileMetadata, } from "./file-stream"; export { ConfigurationError, } from "./errors"; export { default as sqlProp, } from "./sql-prop"; export type { ColumnSchema, DbInfo, TableInfo, TableMetadata, TableSchema, } from "./sql-prop"; diff --git a/platform/dist/index.js b/platform/dist/index.js index b34b761357888..aad48e6f1f11c 100644 --- a/platform/dist/index.js +++ b/platform/dist/index.js @@ -8,6 +8,9 @@ Object.defineProperty(exports, "transformConfigForOauth", { enumerable: true, ge var utils_1 = require("./utils"); Object.defineProperty(exports, "cloneSafe", { enumerable: true, get: function () { return utils_1.cloneSafe; } }); Object.defineProperty(exports, "jsonStringifySafe", { enumerable: true, get: function () { return utils_1.jsonStringifySafe; } }); +var file_stream_1 = require("./file-stream"); +Object.defineProperty(exports, "getFileStreamAndMetadata", { enumerable: true, get: function () { return file_stream_1.getFileStreamAndMetadata; } }); +Object.defineProperty(exports, "getFileStream", { enumerable: true, get: function () { return file_stream_1.getFileStream; } }); var errors_1 = require("./errors"); Object.defineProperty(exports, "ConfigurationError", { enumerable: true, get: function () { return errors_1.ConfigurationError; } }); var sql_prop_1 = require("./sql-prop"); diff --git a/platform/lib/file-stream.ts b/platform/lib/file-stream.ts new file mode 100644 index 0000000000000..e6a62808002c0 --- /dev/null +++ b/platform/lib/file-stream.ts @@ -0,0 +1,148 @@ +import { Readable } from "stream"; +import { ReadableStream } from "stream/web"; +import { + createReadStream, promises as fs, +} from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { createWriteStream } from "fs"; +import { pipeline } from "stream/promises"; + +export interface FileMetadata { + size?: number; + contentType?: string; + lastModified?: Date; + name?: string; + etag?: string; +} + +/** + * @param pathOrUrl - a file path or a URL + * @returns a Readable stream of the file content + */ +export async function getFileStream(pathOrUrl: string): Promise { + if (isUrl(pathOrUrl)) { + const response = await fetch(pathOrUrl); + if (!response.ok) { + throw new Error(`Failed to fetch ${pathOrUrl}: ${response.status} ${response.statusText}`); + } + return Readable.fromWeb(response.body as ReadableStream); + } else { + // Check if file exists first (this will throw if file doesn't exist) + await fs.stat(pathOrUrl); + return createReadStream(pathOrUrl); + } +} + +/** + * @param pathOrUrl - a file path or a URL + * @returns a Readable stream of the file content and its metadata + */ +export async function getFileStreamAndMetadata(pathOrUrl: string): Promise<{ stream: Readable; metadata: FileMetadata }> { + if (isUrl(pathOrUrl)) { + return await getRemoteFileStreamAndMetadata(pathOrUrl); + } else { + return await getLocalFileStreamAndMetadata(pathOrUrl); + } +} + +function isUrl(pathOrUrl: string): boolean { + try { + new URL(pathOrUrl); + return true; + } catch { + return false; + } +} + +async function getLocalFileStreamAndMetadata(filePath: string): Promise<{ stream: Readable; metadata: FileMetadata }> { + const stats = await fs.stat(filePath); + const metadata: FileMetadata = { + size: stats.size, + lastModified: stats.mtime, + name: filePath.split("/").pop() || filePath.split("\\").pop(), + }; + const stream = createReadStream(filePath); + return { + stream, + metadata, + }; +} + +async function getRemoteFileStreamAndMetadata(url: string): Promise<{ stream: Readable; metadata: FileMetadata }> { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); + } + + const headers = response.headers; + const contentLength = headers.get("content-length"); + const contentType = headers.get("content-type") || undefined; + const lastModified = headers.get("last-modified") + ? new Date(headers.get("last-modified")!) + : undefined; + const etag = headers.get("etag") || undefined; + const urlObj = new URL(url); + const name = urlObj.pathname.split("/").pop() || undefined; + + const baseMetadata: FileMetadata = { + contentType, + lastModified, + name, + etag, + }; + + // If we have content-length, we can stream directly + if (contentLength) { + const metadata: FileMetadata = { + ...baseMetadata, + size: parseInt(contentLength, 10), + }; + const stream = Readable.fromWeb(response.body as ReadableStream); + return { + stream, + metadata, + }; + } + + // No content-length header - need to download to temporary file to get size + return await downloadToTemporaryFile(response, baseMetadata); +} + +async function downloadToTemporaryFile(response: Response, baseMetadata: FileMetadata): Promise<{ stream: Readable; metadata: FileMetadata }> { + // Generate unique temporary file path + const tempFileName = `file-stream-${Date.now()}-${Math.random().toString(36) + .substring(2)}`; + const tempFilePath = join(tmpdir(), tempFileName); + // Download to temporary file + const fileStream = createWriteStream(tempFilePath); + const webStream = Readable.fromWeb(response.body as ReadableStream); + await pipeline(webStream, fileStream); + // Get file stats + const stats = await fs.stat(tempFilePath); + const metadata: FileMetadata = { + ...baseMetadata, + size: stats.size, + }; + + // Create a readable stream that cleans up the temp file when done + const stream = createReadStream(tempFilePath); + + // Clean up temp file when stream is closed or ends + const cleanup = async () => { + try { + await fs.unlink(tempFilePath); + } catch { + // Ignore cleanup errors (file might already be deleted) + } + }; + + stream.on("close", cleanup); + stream.on("end", cleanup); + stream.on("error", cleanup); + + return { + stream, + metadata, + }; +} diff --git a/platform/lib/index.ts b/platform/lib/index.ts index 55fa677cb020d..2399ee1d05b66 100644 --- a/platform/lib/index.ts +++ b/platform/lib/index.ts @@ -10,6 +10,14 @@ export { cloneSafe, jsonStringifySafe, } from "./utils"; +export { + getFileStreamAndMetadata, + getFileStream, +} from "./file-stream"; +export type { + FileMetadata, +} from "./file-stream"; + export { ConfigurationError, } from "./errors"; From 881246e14d53d2845401d3196a76599358652084 Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Tue, 3 Jun 2025 18:49:05 -0400 Subject: [PATCH 03/11] bump package minor version --- platform/package-lock.json | 4 ++-- platform/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/platform/package-lock.json b/platform/package-lock.json index 93c3e858f0363..cef42d76da817 100644 --- a/platform/package-lock.json +++ b/platform/package-lock.json @@ -1,12 +1,12 @@ { "name": "@pipedream/platform", - "version": "3.0.3", + "version": "3.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@pipedream/platform", - "version": "3.0.3", + "version": "3.1.0", "license": "MIT", "dependencies": { "axios": "^1.7.4", diff --git a/platform/package.json b/platform/package.json index 498ecd1c48e72..ae97307ce7422 100644 --- a/platform/package.json +++ b/platform/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/platform", - "version": "3.0.3", + "version": "3.1.0", "description": "Pipedream platform globals (typing and runtime type checking)", "homepage": "https://pipedream.com", "main": "dist/index.js", From fdd695912a67c4fd3c73adc5822fefdf068431ea Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Tue, 3 Jun 2025 18:50:13 -0400 Subject: [PATCH 04/11] update pnpm-lock.yaml --- pnpm-lock.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 604557836aaad..d8e155f1bbfe2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32644,7 +32644,7 @@ snapshots: '@babel/helper-validator-option': 8.0.0-alpha.13 browserslist: 4.24.2 lru-cache: 7.18.3 - semver: 7.7.1 + semver: 7.7.2 '@babel/helper-create-class-features-plugin@7.25.9(@babel/core@7.26.0)': dependencies: @@ -35984,6 +35984,8 @@ snapshots: '@putout/operator-filesystem': 5.0.0(putout@36.13.1(eslint@8.57.1)(typescript@5.6.3)) '@putout/operator-json': 2.2.0 putout: 36.13.1(eslint@8.57.1)(typescript@5.6.3) + transitivePeerDependencies: + - supports-color '@putout/operator-regexp@1.0.0(putout@36.13.1(eslint@8.57.1)(typescript@5.6.3))': dependencies: @@ -43260,7 +43262,7 @@ snapshots: is-bun-module@1.2.1: dependencies: - semver: 7.7.1 + semver: 7.7.2 is-callable@1.2.7: {} From 4d2678d552bcdfa6aefecab3810ce7e9105e38bd Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Tue, 3 Jun 2025 19:26:16 -0400 Subject: [PATCH 05/11] suppress linter error by allowing 'any' type for event and request body --- platform/lib/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platform/lib/index.ts b/platform/lib/index.ts index 2399ee1d05b66..4f16ceded9ad4 100644 --- a/platform/lib/index.ts +++ b/platform/lib/index.ts @@ -167,6 +167,7 @@ export const sendTypeMap = { }; // Event object that persists throughout worfklow with observability after each step. +// eslint-disable-next-line @typescript-eslint/no-explicit-any export let $event: any; export const END_NEEDLE = "__pd_end"; @@ -215,6 +216,7 @@ export const $sendConfigRuntimeTypeChecker = (function () { export interface AxiosRequestConfig extends AxiosConfig { debug?: boolean; + // eslint-disable-next-line @typescript-eslint/no-explicit-any body?: any; returnFullResponse?: boolean; } From 1ec900cdb92d93590085473faf7bff23e9f47269 Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Wed, 4 Jun 2025 14:22:39 -0400 Subject: [PATCH 06/11] Apply suggestions from code review * Create and use safeStat function * Check for response.body when fetching file * Use uuid package to generate temp file name * Use mime-types package to lookup content type * Cleanup temp file on error Co-authored-by: Jorge Cortes --- platform/lib/file-stream.ts | 91 +++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/platform/lib/file-stream.ts b/platform/lib/file-stream.ts index e6a62808002c0..012b365a809f5 100644 --- a/platform/lib/file-stream.ts +++ b/platform/lib/file-stream.ts @@ -1,12 +1,11 @@ import { Readable } from "stream"; import { ReadableStream } from "stream/web"; -import { - createReadStream, promises as fs, -} from "fs"; +import { createReadStream, createWriteStream, promises as fs } from "fs"; import { tmpdir } from "os"; -import { join } from "path"; -import { createWriteStream } from "fs"; +import { join, basename } from "path"; import { pipeline } from "stream/promises"; +import { v4 as uuidv4 } from "uuid"; +import mime from "mime-types"; export interface FileMetadata { size?: number; @@ -23,13 +22,12 @@ export interface FileMetadata { export async function getFileStream(pathOrUrl: string): Promise { if (isUrl(pathOrUrl)) { const response = await fetch(pathOrUrl); - if (!response.ok) { + if (!response.ok || !response.body) { throw new Error(`Failed to fetch ${pathOrUrl}: ${response.status} ${response.statusText}`); } return Readable.fromWeb(response.body as ReadableStream); } else { - // Check if file exists first (this will throw if file doesn't exist) - await fs.stat(pathOrUrl); + await safeStat(pathOrUrl); return createReadStream(pathOrUrl); } } @@ -55,23 +53,32 @@ function isUrl(pathOrUrl: string): boolean { } } -async function getLocalFileStreamAndMetadata(filePath: string): Promise<{ stream: Readable; metadata: FileMetadata }> { - const stats = await fs.stat(filePath); +async function safeStat(path: string): Promise { + try { + return await fs.stat(path); + } catch { + throw new Error(`File not found: ${path}`); + } +} + +async function getLocalFileStreamAndMetadata( + filePath: string +): Promise<{ stream: Readable; metadata: FileMetadata }> { + const stats = await safeStat(filePath); + const contentType = mime.lookup(filePath) || undefined; const metadata: FileMetadata = { size: stats.size, lastModified: stats.mtime, - name: filePath.split("/").pop() || filePath.split("\\").pop(), + name: basename(filePath), + contentType }; const stream = createReadStream(filePath); - return { - stream, - metadata, - }; + return { stream, metadata }; } async function getRemoteFileStreamAndMetadata(url: string): Promise<{ stream: Readable; metadata: FileMetadata }> { const response = await fetch(url); - if (!response.ok) { + if (!response.ok || !response.body) { throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); } @@ -111,38 +118,36 @@ async function getRemoteFileStreamAndMetadata(url: string): Promise<{ stream: Re async function downloadToTemporaryFile(response: Response, baseMetadata: FileMetadata): Promise<{ stream: Readable; metadata: FileMetadata }> { // Generate unique temporary file path - const tempFileName = `file-stream-${Date.now()}-${Math.random().toString(36) - .substring(2)}`; + const tempFileName = `file-stream-${uuidv4()}`; const tempFilePath = join(tmpdir(), tempFileName); // Download to temporary file const fileStream = createWriteStream(tempFilePath); const webStream = Readable.fromWeb(response.body as ReadableStream); - await pipeline(webStream, fileStream); - // Get file stats - const stats = await fs.stat(tempFilePath); - const metadata: FileMetadata = { - ...baseMetadata, - size: stats.size, - }; - - // Create a readable stream that cleans up the temp file when done - const stream = createReadStream(tempFilePath); + try { + await pipeline(webStream, fileStream); + const stats = await fs.stat(tempFilePath); + const metadata: FileMetadata = { + ...baseMetadata, + size: stats.size + }; + const stream = createReadStream(tempFilePath); - // Clean up temp file when stream is closed or ends - const cleanup = async () => { - try { - await fs.unlink(tempFilePath); - } catch { - // Ignore cleanup errors (file might already be deleted) - } - }; + const cleanup = async () => { + try { + await fs.unlink(tempFilePath); + } catch { + // Ignore cleanup errors + } + }; - stream.on("close", cleanup); - stream.on("end", cleanup); - stream.on("error", cleanup); + stream.on("close", cleanup); + stream.on("end", cleanup); + stream.on("error", cleanup); - return { - stream, - metadata, - }; + return { stream, metadata }; + } catch (err) { + // Cleanup on error + try { await fs.unlink(tempFilePath); } catch {} + throw err; + } } From d6eddb90a117a31d52f32f3e83a231852943d048 Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Wed, 4 Jun 2025 14:25:12 -0400 Subject: [PATCH 07/11] add uuid and mime-types packages --- platform/package-lock.json | 83 +++++++++++++++++++++++++++++++------- platform/package.json | 6 ++- 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/platform/package-lock.json b/platform/package-lock.json index cef42d76da817..ed9af9fc05a25 100644 --- a/platform/package-lock.json +++ b/platform/package-lock.json @@ -12,7 +12,9 @@ "axios": "^1.7.4", "fp-ts": "^2.0.2", "io-ts": "^2.0.0", - "querystring": "^0.2.1" + "mime-types": "^3.0.1", + "querystring": "^0.2.1", + "uuid": "^11.1.0" }, "devDependencies": { "@octokit/core": "^3.6.0", @@ -2320,6 +2322,25 @@ "node": ">= 6" } }, + "node_modules/form-data/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/form-data/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/fp-ts": { "version": "2.12.1", "resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.12.1.tgz", @@ -4603,19 +4624,19 @@ } }, "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz", + "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==", "dependencies": { - "mime-db": "1.52.0" + "mime-db": "^1.54.0" }, "engines": { "node": ">= 0.6" @@ -5516,6 +5537,18 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/v8-to-istanbul": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.0.1.tgz", @@ -7488,6 +7521,21 @@ "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "mime-types": "^2.1.12" + }, + "dependencies": { + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } + } } }, "fp-ts": { @@ -9180,16 +9228,16 @@ } }, "mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==" }, "mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz", + "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==", "requires": { - "mime-db": "1.52.0" + "mime-db": "^1.54.0" } }, "mimic-fn": { @@ -9859,6 +9907,11 @@ "picocolors": "^1.0.0" } }, + "uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==" + }, "v8-to-istanbul": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.0.1.tgz", diff --git a/platform/package.json b/platform/package.json index ae97307ce7422..7ddc7be58a2b8 100644 --- a/platform/package.json +++ b/platform/package.json @@ -20,13 +20,15 @@ "axios": "^1.7.4", "fp-ts": "^2.0.2", "io-ts": "^2.0.0", - "querystring": "^0.2.1" + "mime-types": "^3.0.1", + "querystring": "^0.2.1", + "uuid": "^11.1.0" }, "devDependencies": { + "@octokit/core": "^3.6.0", "husky": "^3.0.0", "jest": "^29.1.2", "type-fest": "^4.15.0", - "@octokit/core": "^3.6.0", "typescript": "^3.5.3" } } From 4cd5b5893262764162751c3e4d511c90f6800b45 Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Wed, 4 Jun 2025 14:33:37 -0400 Subject: [PATCH 08/11] apply linter fixes and build --- platform/dist/file-stream.js | 78 ++++++++++++++++++++++-------------- platform/lib/file-stream.ts | 30 +++++++++----- 2 files changed, 69 insertions(+), 39 deletions(-) diff --git a/platform/dist/file-stream.js b/platform/dist/file-stream.js index 1a90c64f71f92..7d3aff85103b9 100644 --- a/platform/dist/file-stream.js +++ b/platform/dist/file-stream.js @@ -5,8 +5,9 @@ const stream_1 = require("stream"); const fs_1 = require("fs"); const os_1 = require("os"); const path_1 = require("path"); -const fs_2 = require("fs"); const promises_1 = require("stream/promises"); +const uuid_1 = require("uuid"); +const mime = require("mime-types"); /** * @param pathOrUrl - a file path or a URL * @returns a Readable stream of the file content @@ -14,14 +15,13 @@ const promises_1 = require("stream/promises"); async function getFileStream(pathOrUrl) { if (isUrl(pathOrUrl)) { const response = await fetch(pathOrUrl); - if (!response.ok) { + if (!response.ok || !response.body) { throw new Error(`Failed to fetch ${pathOrUrl}: ${response.status} ${response.statusText}`); } return stream_1.Readable.fromWeb(response.body); } else { - // Check if file exists first (this will throw if file doesn't exist) - await fs_1.promises.stat(pathOrUrl); + await safeStat(pathOrUrl); return fs_1.createReadStream(pathOrUrl); } } @@ -48,12 +48,22 @@ function isUrl(pathOrUrl) { return false; } } +async function safeStat(path) { + try { + return await fs_1.promises.stat(path); + } + catch (_a) { + throw new Error(`File not found: ${path}`); + } +} async function getLocalFileStreamAndMetadata(filePath) { - const stats = await fs_1.promises.stat(filePath); + const stats = await safeStat(filePath); + const contentType = mime.lookup(filePath) || undefined; const metadata = { size: stats.size, lastModified: stats.mtime, - name: filePath.split("/").pop() || filePath.split("\\").pop(), + name: path_1.basename(filePath), + contentType, }; const stream = fs_1.createReadStream(filePath); return { @@ -63,7 +73,7 @@ async function getLocalFileStreamAndMetadata(filePath) { } async function getRemoteFileStreamAndMetadata(url) { const response = await fetch(url); - if (!response.ok) { + if (!response.ok || !response.body) { throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); } const headers = response.headers; @@ -98,35 +108,43 @@ async function getRemoteFileStreamAndMetadata(url) { } async function downloadToTemporaryFile(response, baseMetadata) { // Generate unique temporary file path - const tempFileName = `file-stream-${Date.now()}-${Math.random().toString(36) - .substring(2)}`; + const tempFileName = `file-stream-${uuid_1.v4()}`; const tempFilePath = path_1.join(os_1.tmpdir(), tempFileName); // Download to temporary file - const fileStream = fs_2.createWriteStream(tempFilePath); + const fileStream = fs_1.createWriteStream(tempFilePath); const webStream = stream_1.Readable.fromWeb(response.body); - await promises_1.pipeline(webStream, fileStream); - // Get file stats - const stats = await fs_1.promises.stat(tempFilePath); - const metadata = { - ...baseMetadata, - size: stats.size, - }; - // Create a readable stream that cleans up the temp file when done - const stream = fs_1.createReadStream(tempFilePath); - // Clean up temp file when stream is closed or ends - const cleanup = async () => { + try { + await promises_1.pipeline(webStream, fileStream); + const stats = await fs_1.promises.stat(tempFilePath); + const metadata = { + ...baseMetadata, + size: stats.size, + }; + const stream = fs_1.createReadStream(tempFilePath); + const cleanup = async () => { + try { + await fs_1.promises.unlink(tempFilePath); + } + catch (_a) { + // Ignore cleanup errors + } + }; + stream.on("close", cleanup); + stream.on("end", cleanup); + stream.on("error", cleanup); + return { + stream, + metadata, + }; + } + catch (err) { + // Cleanup on error try { await fs_1.promises.unlink(tempFilePath); } catch (_a) { - // Ignore cleanup errors (file might already be deleted) + // Ignore cleanup errors } - }; - stream.on("close", cleanup); - stream.on("end", cleanup); - stream.on("error", cleanup); - return { - stream, - metadata, - }; + throw err; + } } diff --git a/platform/lib/file-stream.ts b/platform/lib/file-stream.ts index 012b365a809f5..ffa47d2628e53 100644 --- a/platform/lib/file-stream.ts +++ b/platform/lib/file-stream.ts @@ -1,11 +1,15 @@ import { Readable } from "stream"; import { ReadableStream } from "stream/web"; -import { createReadStream, createWriteStream, promises as fs } from "fs"; +import { + createReadStream, createWriteStream, promises as fs, Stats, +} from "fs"; import { tmpdir } from "os"; -import { join, basename } from "path"; +import { + join, basename, +} from "path"; import { pipeline } from "stream/promises"; import { v4 as uuidv4 } from "uuid"; -import mime from "mime-types"; +import * as mime from "mime-types"; export interface FileMetadata { size?: number; @@ -62,7 +66,7 @@ async function safeStat(path: string): Promise { } async function getLocalFileStreamAndMetadata( - filePath: string + filePath: string, ): Promise<{ stream: Readable; metadata: FileMetadata }> { const stats = await safeStat(filePath); const contentType = mime.lookup(filePath) || undefined; @@ -70,10 +74,13 @@ async function getLocalFileStreamAndMetadata( size: stats.size, lastModified: stats.mtime, name: basename(filePath), - contentType + contentType, }; const stream = createReadStream(filePath); - return { stream, metadata }; + return { + stream, + metadata, + }; } async function getRemoteFileStreamAndMetadata(url: string): Promise<{ stream: Readable; metadata: FileMetadata }> { @@ -128,7 +135,7 @@ async function downloadToTemporaryFile(response: Response, baseMetadata: FileMet const stats = await fs.stat(tempFilePath); const metadata: FileMetadata = { ...baseMetadata, - size: stats.size + size: stats.size, }; const stream = createReadStream(tempFilePath); @@ -144,10 +151,15 @@ async function downloadToTemporaryFile(response: Response, baseMetadata: FileMet stream.on("end", cleanup); stream.on("error", cleanup); - return { stream, metadata }; + return { + stream, + metadata, + }; } catch (err) { // Cleanup on error - try { await fs.unlink(tempFilePath); } catch {} + try { await fs.unlink(tempFilePath); } catch { + // Ignore cleanup errors + } throw err; } } From 59c3aad6c0f8f959f0ded4f28ba17013475f152f Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Wed, 4 Jun 2025 14:44:49 -0400 Subject: [PATCH 09/11] * make size required in file metadata * use basename to get file name from file url * lookup content type from file name if no content-type header --- platform/dist/file-stream.d.ts | 2 +- platform/dist/file-stream.js | 4 ++-- platform/lib/file-stream.ts | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/platform/dist/file-stream.d.ts b/platform/dist/file-stream.d.ts index 53088006cfc38..4cff2e1f7abaf 100644 --- a/platform/dist/file-stream.d.ts +++ b/platform/dist/file-stream.d.ts @@ -1,7 +1,7 @@ /// import { Readable } from "stream"; export interface FileMetadata { - size?: number; + size: number; contentType?: string; lastModified?: Date; name?: string; diff --git a/platform/dist/file-stream.js b/platform/dist/file-stream.js index 7d3aff85103b9..de4c9b6a80864 100644 --- a/platform/dist/file-stream.js +++ b/platform/dist/file-stream.js @@ -78,13 +78,13 @@ async function getRemoteFileStreamAndMetadata(url) { } const headers = response.headers; const contentLength = headers.get("content-length"); - const contentType = headers.get("content-type") || undefined; const lastModified = headers.get("last-modified") ? new Date(headers.get("last-modified")) : undefined; const etag = headers.get("etag") || undefined; const urlObj = new URL(url); - const name = urlObj.pathname.split("/").pop() || undefined; + const name = path_1.basename(urlObj.pathname); + const contentType = headers.get("content-type") || mime.lookup(urlObj.pathname) || undefined; const baseMetadata = { contentType, lastModified, diff --git a/platform/lib/file-stream.ts b/platform/lib/file-stream.ts index ffa47d2628e53..aa3dfc59d0e44 100644 --- a/platform/lib/file-stream.ts +++ b/platform/lib/file-stream.ts @@ -12,7 +12,7 @@ import { v4 as uuidv4 } from "uuid"; import * as mime from "mime-types"; export interface FileMetadata { - size?: number; + size: number; contentType?: string; lastModified?: Date; name?: string; @@ -91,15 +91,15 @@ async function getRemoteFileStreamAndMetadata(url: string): Promise<{ stream: Re const headers = response.headers; const contentLength = headers.get("content-length"); - const contentType = headers.get("content-type") || undefined; const lastModified = headers.get("last-modified") ? new Date(headers.get("last-modified")!) : undefined; const etag = headers.get("etag") || undefined; const urlObj = new URL(url); - const name = urlObj.pathname.split("/").pop() || undefined; + const name = basename(urlObj.pathname); + const contentType = headers.get("content-type") || mime.lookup(urlObj.pathname) || undefined; - const baseMetadata: FileMetadata = { + const baseMetadata = { contentType, lastModified, name, @@ -123,7 +123,7 @@ async function getRemoteFileStreamAndMetadata(url: string): Promise<{ stream: Re return await downloadToTemporaryFile(response, baseMetadata); } -async function downloadToTemporaryFile(response: Response, baseMetadata: FileMetadata): Promise<{ stream: Readable; metadata: FileMetadata }> { +async function downloadToTemporaryFile(response: Response, baseMetadata: Partial): Promise<{ stream: Readable; metadata: FileMetadata }> { // Generate unique temporary file path const tempFileName = `file-stream-${uuidv4()}`; const tempFilePath = join(tmpdir(), tempFileName); From 65653d840fe4811368fbbb9478aafe01699cbef6 Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Wed, 4 Jun 2025 14:47:47 -0400 Subject: [PATCH 10/11] update pnpm-lock.yaml --- pnpm-lock.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d8e155f1bbfe2..f107bd247ed8b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15765,9 +15765,15 @@ importers: io-ts: specifier: ^2.0.0 version: 2.2.21(fp-ts@2.16.9) + mime-types: + specifier: ^3.0.1 + version: 3.0.1 querystring: specifier: ^0.2.1 version: 0.2.1 + uuid: + specifier: ^11.1.0 + version: 11.1.0 devDependencies: '@octokit/core': specifier: ^3.6.0 From edc6b54f6ce34e4963e372bf4d15e258071dee71 Mon Sep 17 00:00:00 2001 From: js07 <19861096+js07@users.noreply.github.com> Date: Wed, 4 Jun 2025 14:56:03 -0400 Subject: [PATCH 11/11] prevent multiple cleanup executions --- platform/dist/file-stream.js | 6 +++--- platform/lib/file-stream.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/platform/dist/file-stream.js b/platform/dist/file-stream.js index de4c9b6a80864..e8c8a639549a2 100644 --- a/platform/dist/file-stream.js +++ b/platform/dist/file-stream.js @@ -129,9 +129,9 @@ async function downloadToTemporaryFile(response, baseMetadata) { // Ignore cleanup errors } }; - stream.on("close", cleanup); - stream.on("end", cleanup); - stream.on("error", cleanup); + stream.once("close", cleanup); + stream.once("end", cleanup); + stream.once("error", cleanup); return { stream, metadata, diff --git a/platform/lib/file-stream.ts b/platform/lib/file-stream.ts index aa3dfc59d0e44..bafa9b194478e 100644 --- a/platform/lib/file-stream.ts +++ b/platform/lib/file-stream.ts @@ -147,9 +147,9 @@ async function downloadToTemporaryFile(response: Response, baseMetadata: Partial } }; - stream.on("close", cleanup); - stream.on("end", cleanup); - stream.on("error", cleanup); + stream.once("close", cleanup); + stream.once("end", cleanup); + stream.once("error", cleanup); return { stream,