Skip to content

Commit 3c167c8

Browse files
michelle0927michelle0927
committed
new components
1 parent 666b7c4 commit 3c167c8

File tree

5 files changed

+233
-6
lines changed

5 files changed

+233
-6
lines changed

‎components/zenscrape/actions/get-credit-status/get-credit-status.mjs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import zenscrape from "../../zenscrape.app.mjs";
2+
3+
export default {
4+
key: "zenscrape-get-credit-status",
5+
name: "Get Credit Status",
6+
description: "Retrieve the number of remaining credits in Zenscrape. [See the documentation](https://app.zenscrape.com/documentation)",
7+
version: "0.0.1",
8+
type: "action",
9+
props: {
10+
zenscrape,
11+
},
12+
async run({ $ }) {
13+
const response = await this.zenscrape.getStatus({
14+
$,
15+
});
16+
$.export("$summary", "Successfully retrieved credit status.");
17+
return response;
18+
},
19+
};

‎components/zenscrape/actions/get-website-content/get-website-content.mjs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import zenscrape from "../../zenscrape.app.mjs";
2+
3+
export default {
4+
key: "zenscrape-get-website-content",
5+
name: "Get Website Content",
6+
description: "Retrieve the content of a website. [See the documentation](https://app.zenscrape.com/documentation)",
7+
version: "0.0.1",
8+
type: "action",
9+
props: {
10+
zenscrape,
11+
url: {
12+
propDefinition: [
13+
zenscrape,
14+
"url",
15+
],
16+
},
17+
premium: {
18+
propDefinition: [
19+
zenscrape,
20+
"premium",
21+
],
22+
},
23+
location: {
24+
propDefinition: [
25+
zenscrape,
26+
"location",
27+
],
28+
},
29+
keepHeaders: {
30+
propDefinition: [
31+
zenscrape,
32+
"keepHeaders",
33+
],
34+
},
35+
render: {
36+
propDefinition: [
37+
zenscrape,
38+
"render",
39+
],
40+
},
41+
},
42+
async run({ $ }) {
43+
const response = await this.zenscrape.getContent({
44+
$,
45+
params: {
46+
url: this.url,
47+
premium: this.premium,
48+
location: this.location,
49+
keep_headers: this.keepHeaders,
50+
render: this.render,
51+
},
52+
});
53+
$.export("$summary", `Successfully scraped website \`${this.url}.\``);
54+
return response;
55+
},
56+
};

‎components/zenscrape/package.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@pipedream/zenscrape",
3-
"version": "0.0.1",
3+
"version": "0.1.0",
44
"description": "Pipedream Zenscrape Components",
55
"main": "zenscrape.app.mjs",
66
"keywords": [
@@ -11,5 +11,9 @@
1111
"author": "Pipedream <[email protected]> (https://pipedream.com/)",
1212
"publishConfig": {
1313
"access": "public"
14+
},
15+
"dependencies": {
16+
"@pipedream/platform": "^3.0.3",
17+
"md5": "^2.3.0"
1418
}
15-
}
19+
}

‎components/zenscrape/sources/website-content-updated/website-content-updated.mjs‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import zenscrape from "../../zenscrape.app.mjs";
2+
import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
3+
import md5 from "md5";
4+
5+
export default {
6+
key: "zenscrape-website-content-updated",
7+
name: "Website Content Updated",
8+
description: "Emit new event when the content of a URL has updated. [See the documentation](https://app.zenscrape.com/documentation)",
9+
version: "0.0.1",
10+
type: "source",
11+
dedupe: "unique",
12+
props: {
13+
zenscrape,
14+
db: "$.service.db",
15+
timer: {
16+
type: "$.interface.timer",
17+
default: {
18+
intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
19+
},
20+
},
21+
url: {
22+
propDefinition: [
23+
zenscrape,
24+
"url",
25+
],
26+
},
27+
premium: {
28+
propDefinition: [
29+
zenscrape,
30+
"premium",
31+
],
32+
},
33+
location: {
34+
propDefinition: [
35+
zenscrape,
36+
"location",
37+
],
38+
},
39+
keepHeaders: {
40+
propDefinition: [
41+
zenscrape,
42+
"keepHeaders",
43+
],
44+
},
45+
render: {
46+
propDefinition: [
47+
zenscrape,
48+
"render",
49+
],
50+
},
51+
},
52+
methods: {
53+
_getContentHash() {
54+
return this.db.get("contentHash");
55+
},
56+
_setContentHash(contentHash) {
57+
this.db.set("contentHash", contentHash);
58+
},
59+
generateMeta() {
60+
const ts = Date.now();
61+
return {
62+
id: ts,
63+
summary: "Website Content Updated",
64+
ts,
65+
};
66+
},
67+
},
68+
async run() {
69+
const contentHash = this._getContentHash();
70+
71+
const content = await this.zenscrape.getContent({
72+
params: {
73+
url: this.url,
74+
premium: this.premium,
75+
location: this.location,
76+
keep_headers: this.keepHeaders,
77+
render: this.render,
78+
},
79+
});
80+
81+
const newContentHash = md5(JSON.stringify(content));
82+
83+
if (newContentHash === contentHash) {
84+
return;
85+
}
86+
87+
this._setContentHash(newContentHash);
88+
89+
const meta = this.generateMeta();
90+
this.$emit(content, meta);
91+
},
92+
};

‎components/zenscrape/zenscrape.app.mjs‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,67 @@
1+
import { axios } from "@pipedream/platform";
2+
13
export default {
24
type: "app",
35
app: "zenscrape",
4-
propDefinitions: {},
6+
propDefinitions: {
7+
url: {
8+
type: "string",
9+
label: "URL",
10+
description: "The target site you want to scrape",
11+
},
12+
premium: {
13+
type: "boolean",
14+
label: "Premium",
15+
description: "Uses residential proxies, unlocks sites that are hard to scrape. Counts as 20 credits towards your quota.",
16+
optional: true,
17+
},
18+
location: {
19+
type: "string",
20+
label: "Location",
21+
description: "If premium=`false` possible locations are 'na' (North America) and 'eu' (Europe). If premium=`true` you can choose a location from Zenscrape's [list of 230+ countries](https://app.zenscrape.com/documentation#proxyLocationList)",
22+
optional: true,
23+
},
24+
keepHeaders: {
25+
type: "boolean",
26+
label: "Keep Headers",
27+
description: "Allow to pass through forward headers (e.g. user agents, cookies)",
28+
optional: true,
29+
},
30+
render: {
31+
type: "boolean",
32+
label: "Render",
33+
description: "Use a headless browser to fetch content that relies on javascript. Counts as 5 credits towards your quota.",
34+
optional: true,
35+
},
36+
},
537
methods: {
6-
// this.$auth contains connected account data
7-
authKeys() {
8-
console.log(Object.keys(this.$auth));
38+
_baseUrl() {
39+
return "https://app.zenscrape.com/api/v1";
40+
},
41+
_makeRequest({
42+
$ = this,
43+
path,
44+
...opts
45+
}) {
46+
return axios($, {
47+
url: `${this._baseUrl()}${path}`,
48+
headers: {
49+
apikey: this.$auth.api_key,
50+
},
51+
...opts,
52+
});
53+
},
54+
getContent(opts = {}) {
55+
return this._makeRequest({
56+
path: "/get",
57+
...opts,
58+
});
59+
},
60+
getStatus(opts = {}) {
61+
return this._makeRequest({
62+
path: "/status",
63+
...opts,
64+
});
965
},
1066
},
1167
};

0 commit comments

Comments
 (0)