Skip to content

Commit 7c91646

Browse files
michelle0927michelle0927
committed
add includeLink
1 parent 68da7a3 commit 7c91646

File tree

36 files changed

+611
-71
lines changed

36 files changed

+611
-71
lines changed

‎components/aws/.upm/store.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"version":2,"languages":{"nodejs-npm":{"guessedImports":["aws-sdk","axios","shortid"],"guessedImportsHash":"08190b61ee0169fd35ea81dce6f4754d"}}}
1+
{"version":2, "languages":{"nodejs-npm":{"guessedImports":["aws-sdk", "axios", "shortid"], "guessedImportsHash":"08190b61ee0169fd35ea81dce6f4754d"}}}

‎components/aws/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@pipedream/aws",
3-
"version": "1.1.0",
3+
"version": "1.1.1",
44
"description": "Pipedream Aws Components",
55
"main": "aws.app.mjs",
66
"keywords": [
@@ -34,9 +34,11 @@
3434
"@pipedream/platform": "^3.1.0",
3535
"adm-zip": "^0.5.10",
3636
"dedent": "^1.5.1",
37+
"file-type": "^21.0.0",
3738
"mailparser": "^3.6.6",
3839
"mailparser-mit": "^1.0.0",
3940
"nanoid": "^5.0.4",
41+
"stream": "^0.0.3",
4042
"uuid": "^9.0.1"
4143
},
4244
"gitHead": "e12480b94cc03bed4808ebc6b13e7fdb3a1ba535",

‎components/aws/sources/common/include-link.mjs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { Readable } from "stream";
2+
import { fileTypeFromBuffer } from "file-type";
3+
4+
export default {
5+
props: {
6+
includeLink: {
7+
label: "Include Link",
8+
type: "boolean",
9+
description: "Upload file to your File Stash and emit temporary download link to the file. See [the docs](https://pipedream.com/docs/connect/components/files) to learn more about working with files in Pipedream.",
10+
default: false,
11+
optional: true,
12+
},
13+
dir: {
14+
type: "dir",
15+
accessMode: "write",
16+
optional: true,
17+
},
18+
},
19+
methods: {
20+
async streamToBuffer(stream) {
21+
const chunks = [];
22+
for await (const chunk of stream) {
23+
chunks.push(chunk);
24+
}
25+
return Buffer.concat(chunks);
26+
},
27+
async stashFile(item) {
28+
const { Body } = await this.getObject({
29+
Bucket: item.bucket.name,
30+
Key: item.object.key,
31+
});
32+
const filepath = `${item.bucket.name}/${item.object.key}`;
33+
const buffer = await this.streamToBuffer(Body);
34+
const type = await fileTypeFromBuffer(buffer);
35+
// Upload the attachment to the configured directory (File Stash) so it
36+
// can be accessed later.
37+
const file = await this.dir.open(filepath).fromReadableStream(
38+
Readable.from(buffer),
39+
type?.mime,
40+
buffer.length,
41+
);
42+
// Return file details and temporary download link:
43+
// { path, get_url, s3Key, type }
44+
return await file.withoutPutUrl().withGetUrl();
45+
},
46+
async processEvent(event) {
47+
const { Message: rawMessage } = event.body;
48+
const {
49+
Records: s3Events = [],
50+
Event: eventType,
51+
} = JSON.parse(rawMessage);
52+
53+
if (eventType === "s3:TestEvent") {
54+
console.log("Received initial test event. Skipping...");
55+
return;
56+
}
57+
58+
for (const s3Event of s3Events) {
59+
const meta = this.generateMeta(s3Event);
60+
let { s3: item } = s3Event;
61+
if (this.includeLink) {
62+
item.file = await this.stashFile(item);
63+
}
64+
this.$emit(item, meta);
65+
}
66+
},
67+
},
68+
};

‎components/aws/sources/s3-new-file/s3-new-file.mjs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,21 @@
11
import base from "../common/s3.mjs";
2+
import includeLink from "../common/include-link.mjs";
23

34
export default {
45
...base,
56
type: "source",
67
key: "aws-s3-new-file",
78
name: "New S3 File",
89
description: "Emit new event when a file is added to an S3 bucket",
9-
version: "0.1.4",
10+
version: "0.2.0",
1011
dedupe: "unique",
12+
props: {
13+
...base.props,
14+
...includeLink.props,
15+
},
1116
methods: {
1217
...base.methods,
18+
...includeLink.methods,
1319
getEvents() {
1420
return [
1521
"s3:ObjectCreated:*",

‎components/aws/sources/s3-restored-file/s3-restored-file.mjs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
import base from "../common/s3.mjs";
2+
import includeLink from "../common/include-link.mjs";
23

34
export default {
45
...base,
56
type: "source",
67
key: "aws-s3-restored-file",
78
name: "New Restored S3 File",
89
description: "Emit new event when an file is restored into a S3 bucket",
9-
version: "0.1.4",
10+
version: "0.2.0",
1011
dedupe: "unique",
1112
props: {
1213
...base.props,
@@ -16,9 +17,11 @@ export default {
1617
description: "When enabled, this event source will also emit events whenever a restore is initiated",
1718
default: false,
1819
},
20+
...includeLink.props,
1921
},
2022
methods: {
2123
...base.methods,
24+
...includeLink.methods,
2225
getEvents() {
2326
return [
2427
this.detectRestoreInitiation

‎components/box/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@pipedream/box",
3-
"version": "0.5.0",
3+
"version": "0.5.1",
44
"description": "Pipedream Box Components",
55
"main": "box.app.mjs",
66
"keywords": [
@@ -11,6 +11,7 @@
1111
"author": "Pipedream <[email protected]> (https://pipedream.com/)",
1212
"dependencies": {
1313
"@pipedream/platform": "^3.1.0",
14+
"file-type": "^21.0.0",
1415
"form-data": "^4.0.0",
1516
"path": "^0.12.7",
1617
"stream": "^0.0.2",

‎components/box/sources/new-file/new-file.mjs‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,30 @@
11
import common from "../common/common.mjs";
2+
import { Readable } from "stream";
3+
import { fileTypeFromBuffer } from "file-type";
24

35
export default {
46
key: "box-new-file",
57
name: "New File Event",
6-
description: "Emit new event when a new file uploaded on a target. [See the documentation](https://developer.box.com/reference/post-webhooks)",
7-
version: "0.0.4",
8+
description: "Emit new event when a new file is uploaded to a target. [See the documentation](https://developer.box.com/reference/post-webhooks)",
9+
version: "0.1.0",
810
type: "source",
911
dedupe: "unique",
1012
...common,
13+
props: {
14+
...common.props,
15+
includeLink: {
16+
label: "Include Link",
17+
type: "boolean",
18+
description: "Upload file to your File Stash and emit temporary download link to the file. See [the docs](https://pipedream.com/docs/connect/components/files) to learn more about working with files in Pipedream.",
19+
default: false,
20+
optional: true,
21+
},
22+
dir: {
23+
type: "dir",
24+
accessMode: "write",
25+
optional: true,
26+
},
27+
},
1128
methods: {
1229
...common.methods,
1330
getTriggers() {
@@ -18,5 +35,29 @@ export default {
1835
getSummary(event) {
1936
return `New file uploaded event with ID(${event.id})`;
2037
},
38+
async stashFile(event) {
39+
const response = await this.app.downloadFile({
40+
fileId: event.source.id,
41+
responseType: "arraybuffer",
42+
});
43+
const buffer = Buffer.from(response);
44+
const filepath = `${event.source.id}/${event.source.name}`;
45+
const type = await fileTypeFromBuffer(buffer);
46+
const file = await this.dir.open(filepath).fromReadableStream(
47+
Readable.from(buffer),
48+
type?.mime,
49+
buffer.length,
50+
);
51+
return await file.withoutPutUrl().withGetUrl();
52+
},
53+
},
54+
async run(event) {
55+
if (this.includeLink) {
56+
event.body.file = await this.stashFile(event.body);
57+
}
58+
this.$emit(
59+
event.body,
60+
this.getMetadata(event.body),
61+
);
2162
},
2263
};

‎components/egnyte/egnyte.app.mjs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,5 +55,14 @@ export default {
5555
...opts,
5656
});
5757
},
58+
downloadFile({
59+
folderPath, filename, ...opts
60+
}) {
61+
return this._makeRequest({
62+
path: `/fs-content/${folderPath}/${filename}`,
63+
responseType: "arraybuffer",
64+
...opts,
65+
});
66+
},
5867
},
5968
};

‎components/egnyte/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@pipedream/egnyte",
3-
"version": "0.2.1",
3+
"version": "0.2.2",
44
"description": "Pipedream Egnyte Components",
55
"main": "egnyte.app.mjs",
66
"keywords": [

‎components/egnyte/sources/common/base.mjs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@ export default {
3030
generateMeta() {
3131
throw new Error("generateMeta is not implemented");
3232
},
33+
emitEvents(items) {
34+
items.reverse().forEach((item) => {
35+
const meta = this.generateMeta(item);
36+
this.$emit(item, meta);
37+
});
38+
},
3339
},
3440
async run() {
3541
const lastTs = this._getLastTs();
@@ -67,10 +73,7 @@ export default {
6773
}
6874
}
6975

70-
newItems.reverse().forEach((item) => {
71-
const meta = this.generateMeta(item);
72-
this.$emit(item, meta);
73-
});
76+
await this.emitEvents(newItems);
7477
};
7578

7679
await processFolder(this.folderPath);

0 commit comments

Comments
 (0)