-
Notifications
You must be signed in to change notification settings - Fork 5.6k
Platform: adding base64 support #19878
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
5346363
d6e69a9
5fea385
ea4aa81
29eb0d1
af13e78
dbc224b
1d1291f
57751d7
aa1dfb2
3e68127
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| export declare type ExecuteQueryArgs = object | string; | ||
| export type ExecuteQueryArgs = object | string; |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| import axios from "axios"; | ||
| import axios, { AxiosHeaders } from "axios"; | ||
|
||
| import { AxiosRequestConfig } from "./index"; | ||
| import * as querystring from "querystring"; | ||
| import { cloneSafe } from "./utils"; | ||
|
|
@@ -164,7 +164,7 @@ function create(config?: AxiosRequestConfig, signConfig?: any) { | |
| axiosInstance.interceptors.request.use(async (config) => { | ||
| if (signConfig) { | ||
| const oauthSignature = await getOauthSignature(config, signConfig); | ||
| if (!config.headers) config.headers = {}; | ||
| if (!config.headers) config.headers = new AxiosHeaders(); | ||
| config.headers.Authorization = oauthSignature; | ||
| } | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,7 +24,9 @@ export interface FileMetadata { | |
| * @returns a Readable stream of the file content | ||
| */ | ||
| export async function getFileStream(pathOrUrl: string): Promise<Readable> { | ||
| if (isUrl(pathOrUrl)) { | ||
| if (isDataUrl(pathOrUrl)) { | ||
| return getDataUrlStream(pathOrUrl); | ||
| } else if (isUrl(pathOrUrl)) { | ||
| const response = await fetch(pathOrUrl); | ||
| if (!response.ok || !response.body) { | ||
| throw new Error(`Failed to fetch ${pathOrUrl}: ${response.status} ${response.statusText}`); | ||
|
|
@@ -41,7 +43,9 @@ export async function getFileStream(pathOrUrl: string): Promise<Readable> { | |
| * @returns a Readable stream of the file content and its metadata | ||
| */ | ||
| export async function getFileStreamAndMetadata(pathOrUrl: string): Promise<{ stream: Readable; metadata: FileMetadata }> { | ||
| if (isUrl(pathOrUrl)) { | ||
| if (isDataUrl(pathOrUrl)) { | ||
| return getDataUrlStreamAndMetadata(pathOrUrl); | ||
| } else if (isUrl(pathOrUrl)) { | ||
| return await getRemoteFileStreamAndMetadata(pathOrUrl); | ||
| } else { | ||
| return await getLocalFileStreamAndMetadata(pathOrUrl); | ||
|
|
@@ -57,6 +61,66 @@ function isUrl(pathOrUrl: string): boolean { | |
| } | ||
| } | ||
|
|
||
| function isDataUrl(pathOrUrl: string): boolean { | ||
| return pathOrUrl.startsWith("data:"); | ||
| } | ||
|
|
||
| interface ParsedDataUrl { | ||
| mediaType: string; | ||
| isBase64: boolean; | ||
| data: string; | ||
| } | ||
|
|
||
| function parseDataUrl(dataUrl: string): ParsedDataUrl { | ||
| // Format: data:[<mediatype>][;base64],<data> | ||
| const match = dataUrl.match(/^data:([^;,]*)?(?:;(base64))?,(.*)$/); | ||
| if (!match) { | ||
| throw new Error("Invalid data URL format"); | ||
| } | ||
| const [ | ||
| , | ||
| mediaType = "text/plain;charset=US-ASCII", | ||
| base64Flag, | ||
| data, | ||
| ] = match; | ||
| return { | ||
| mediaType, | ||
| isBase64: base64Flag === "base64", | ||
| data, | ||
| }; | ||
| } | ||
|
|
||
| function getDataUrlStream(dataUrl: string): Readable { | ||
| const parsed = parseDataUrl(dataUrl); | ||
| const buffer = parsed.isBase64 | ||
| ? Buffer.from(parsed.data, "base64") | ||
| : Buffer.from(decodeURIComponent(parsed.data), "utf-8"); | ||
| return Readable.from(buffer); | ||
| } | ||
|
|
||
| function getDataUrlStreamAndMetadata(dataUrl: string): { stream: Readable; metadata: FileMetadata } { | ||
| const parsed = parseDataUrl(dataUrl); | ||
| const buffer = parsed.isBase64 | ||
| ? Buffer.from(parsed.data, "base64") | ||
| : Buffer.from(decodeURIComponent(parsed.data), "utf-8"); | ||
|
|
||
| const ext = mime.extension(parsed.mediaType); | ||
| const name = ext | ||
| ? `file.${ext}` | ||
| : "file"; | ||
|
|
||
| const metadata: FileMetadata = { | ||
| size: buffer.length, | ||
| contentType: parsed.mediaType || undefined, | ||
| name, | ||
| }; | ||
|
Comment on lines
+74
to
+116
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Data URL parser rejects valid media-type parameters and loses default type. The current regex only allows 🐛 Proposed fix (more spec‑compliant parsing) function parseDataUrl(dataUrl: string): ParsedDataUrl {
// Format: data:[<mediatype>][;base64],<data>
- const match = dataUrl.match(/^data:([^;,]*)?(?:;(base64))?,(.*)$/);
- if (!match) {
- throw new Error("Invalid data URL format");
- }
- const [
- ,
- mediaType = "text/plain;charset=US-ASCII",
- base64Flag,
- data,
- ] = match;
- return {
- mediaType,
- isBase64: base64Flag === "base64",
- data,
- };
+ if (!dataUrl.startsWith("data:")) {
+ throw new Error("Invalid data URL format");
+ }
+ const commaIndex = dataUrl.indexOf(",");
+ if (commaIndex === -1) {
+ throw new Error("Invalid data URL format");
+ }
+ const meta = dataUrl.slice(5, commaIndex); // after "data:"
+ const data = dataUrl.slice(commaIndex + 1);
+ const isBase64 = /;base64$/i.test(meta);
+ const mediaTypeRaw = isBase64 ? meta.replace(/;base64$/i, "") : meta;
+ const mediaType = mediaTypeRaw ? mediaTypeRaw : "text/plain;charset=US-ASCII";
+ return {
+ mediaType,
+ isBase64,
+ data,
+ };
}🤖 Prompt for AI Agents |
||
|
|
||
| return { | ||
| stream: Readable.from(buffer), | ||
| metadata, | ||
| }; | ||
| } | ||
|
|
||
| async function safeStat(path: string): Promise<Stats> { | ||
| try { | ||
| return await fs.stat(path); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| { | ||
| "name": "@pipedream/platform", | ||
| "version": "3.1.1", | ||
| "version": "3.2.0", | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧩 Analysis chain🏁 Script executed: cat -n platform/package.jsonRepository: PipedreamHQ/pipedream Length of output: 1128 🏁 Script executed: find platform -name "jest.config.*" -o -name "jest.setup.*" | head -20Repository: PipedreamHQ/pipedream Length of output: 47 🏁 Script executed: rg "ts-jest" platform/ 2>/dev/null || echo "No ts-jest references found"Repository: PipedreamHQ/pipedream Length of output: 92 🏁 Script executed: rg "preset" platform/ --type json 2>/dev/null || echo "No preset configuration found"Repository: PipedreamHQ/pipedream Length of output: 94 🏁 Script executed: ls -la | grep -E "jest|test"Repository: PipedreamHQ/pipedream Length of output: 134 🏁 Script executed: find platform -type f \( -name "*.test.ts" -o -name "*.spec.ts" -o -name "*.test.js" -o -name "*.spec.js" \) | head -20Repository: PipedreamHQ/pipedream Length of output: 47 🏁 Script executed: cat jest.config.js 2>/dev/null || cat jest.config.json 2>/dev/null || echo "No root jest config found"Repository: PipedreamHQ/pipedream Length of output: 556 🏁 Script executed: head -60 package.jsonRepository: PipedreamHQ/pipedream Length of output: 2235 🌐 Web query:
💡 Result:
Sources 🏁 Script executed: cat package.json | grep -A 100 '"devDependencies"' | grep -E "ts-jest|typescript" | head -10Repository: PipedreamHQ/pipedream Length of output: 246 Ensure ts-jest is listed in platform/package.json devDependencies TypeScript 4.9.5 is compatible with ts-jest 29.1.2 (ts-jest 29.x supports TypeScript >=4.3 <6). However, ts-jest is missing from platform's devDependencies—it's currently only in the root package.json. Add it to platform to ensure explicit dependency declaration and consistency in the monorepo structure. Also note the TypeScript version mismatch: root requires >=5.5.0 <5.7.0 while platform specifies ^4.9.5. Align these versions or document the intentional difference. 🤖 Prompt for AI Agents |
||
| "description": "Pipedream platform globals (typing and runtime type checking)", | ||
| "homepage": "https://pipedream.com", | ||
| "main": "dist/index.js", | ||
|
|
@@ -29,6 +29,6 @@ | |
| "husky": "^3.0.0", | ||
| "jest": "^29.1.2", | ||
| "type-fest": "^4.15.0", | ||
| "typescript": "^3.5.3" | ||
| "typescript": "^4.9.5" | ||
| } | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.