Skip to content

Commit 87adde7

Browse files
lcaresialcaresia
committed
Added actions
1 parent 29c0273 commit 87adde7

File tree

4 files changed

+338
-5
lines changed

4 files changed

+338
-5
lines changed

‎components/scrapingant/actions/general-extraction/general-extraction.mjs‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import app from "../../scrapingant.app.mjs";
2+
3+
export default {
4+
key: "scrapingant-general-extraction",
5+
name: "General Extraction",
6+
description: "Send a request using the standard extraction method of ScrapingAnt.",
7+
version: "0.0.1",
8+
type: "action",
9+
props: {
10+
app,
11+
url: {
12+
propDefinition: [
13+
app,
14+
"url",
15+
],
16+
},
17+
browser: {
18+
propDefinition: [
19+
app,
20+
"browser",
21+
],
22+
},
23+
returnPageSource: {
24+
propDefinition: [
25+
app,
26+
"returnPageSource",
27+
],
28+
},
29+
cookies: {
30+
propDefinition: [
31+
app,
32+
"cookies",
33+
],
34+
},
35+
jsSnippet: {
36+
propDefinition: [
37+
app,
38+
"jsSnippet",
39+
],
40+
},
41+
proxyType: {
42+
propDefinition: [
43+
app,
44+
"proxyType",
45+
],
46+
},
47+
proxyCountry: {
48+
propDefinition: [
49+
app,
50+
"proxyCountry",
51+
],
52+
},
53+
waitForSelector: {
54+
propDefinition: [
55+
app,
56+
"waitForSelector",
57+
],
58+
},
59+
blockResource: {
60+
propDefinition: [
61+
app,
62+
"blockResource",
63+
],
64+
},
65+
},
66+
async run({ $ }) {
67+
const response = await this.app.generalExtraction({
68+
$,
69+
params: {
70+
url: this.url,
71+
browser: this.browser,
72+
return_page_source: this.returnPageSource,
73+
cookies: this.cookies,
74+
js_snippet: this.jsSnippet,
75+
proxy_type: this.proxyType,
76+
proxy_country: this.proxyCountry,
77+
wait_for_selector: this.waitForSelector,
78+
block_resource: this.blockResource,
79+
},
80+
});
81+
82+
$.export("$summary", "Successfully sent the request to ScrapingAnt");
83+
84+
return response;
85+
},
86+
};

‎components/scrapingant/common/constants.mjs‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
export default {
2+
PROXY_COUNTRIES: [
3+
{
4+
label: "World",
5+
value: "",
6+
},
7+
{
8+
label: "Brazil",
9+
value: "BR",
10+
},
11+
{
12+
label: "Canada",
13+
value: "CA",
14+
},
15+
{
16+
label: "China",
17+
value: "CN",
18+
},
19+
{
20+
label: "Czech Republic",
21+
value: "CZ",
22+
},
23+
{
24+
label: "France",
25+
value: "FR",
26+
},
27+
{
28+
label: "Germany",
29+
value: "DE",
30+
},
31+
{
32+
label: "Hong Kong",
33+
value: "HK",
34+
},
35+
{
36+
label: "India",
37+
value: "IN",
38+
},
39+
{
40+
label: "Indonesia",
41+
value: "ID",
42+
},
43+
{
44+
label: "Italy",
45+
value: "IT",
46+
},
47+
{
48+
label: "Israel",
49+
value: "IL",
50+
},
51+
{
52+
label: "Japan",
53+
value: "JP",
54+
},
55+
{
56+
label: "Netherlands",
57+
value: "NL",
58+
},
59+
{
60+
label: "Poland",
61+
value: "PL",
62+
},
63+
{
64+
label: "Russia",
65+
value: "RU",
66+
},
67+
{
68+
label: "Saudi Arabia",
69+
value: "SA",
70+
},
71+
{
72+
label: "Singapore",
73+
value: "SG",
74+
},
75+
{
76+
label: "South Korea",
77+
value: "KR",
78+
},
79+
{
80+
label: "Spain",
81+
value: "ES",
82+
},
83+
{
84+
label: "United Kingdom",
85+
value: "GB",
86+
},
87+
{
88+
label: "United Arab Emirates",
89+
value: "AE",
90+
},
91+
{
92+
label: "USA",
93+
value: "US",
94+
},
95+
{
96+
label: "Vietnam",
97+
value: "VN",
98+
},
99+
],
100+
PROXY_TYPES: [
101+
{
102+
label: "Residential",
103+
value: "residential",
104+
},
105+
{
106+
label: "Datacenter",
107+
value: "datacenter",
108+
},
109+
],
110+
RESOURCE_TYPES: [
111+
{
112+
label: "Document",
113+
value: "document",
114+
},
115+
{
116+
label: "Stylesheet",
117+
value: "stylesheet",
118+
},
119+
{
120+
label: "Image",
121+
value: "image",
122+
},
123+
{
124+
label: "Media",
125+
value: "media",
126+
},
127+
{
128+
label: "Font",
129+
value: "font",
130+
},
131+
{
132+
label: "Script",
133+
value: "script",
134+
},
135+
{
136+
label: "Texttrack",
137+
value: "texttrack",
138+
},
139+
{
140+
label: "XHR",
141+
value: "xhr",
142+
},
143+
{
144+
label: "Fetch",
145+
value: "fetch",
146+
},
147+
{
148+
label: "Eventsource",
149+
value: "eventsource",
150+
},
151+
{
152+
label: "Websocket",
153+
value: "websocket",
154+
},
155+
{
156+
label: "Manifest",
157+
value: "manifest",
158+
},
159+
{
160+
label: "Other",
161+
value: "other",
162+
},
163+
],
164+
};

‎components/scrapingant/package.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,8 @@
1111
"author": "Pipedream <[email protected]> (https://pipedream.com/)",
1212
"publishConfig": {
1313
"access": "public"
14+
},
15+
"dependencies": {
16+
"@pipedream/platform": "^3.0.3"
1417
}
15-
}
18+
}

‎components/scrapingant/scrapingant.app.mjs‎

Lines changed: 84 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,91 @@
1+
import { axios } from "@pipedream/platform";
2+
import constants from "./common/constants.mjs";
3+
14
export default {
25
type: "app",
36
app: "scrapingant",
4-
propDefinitions: {},
7+
propDefinitions: {
8+
url: {
9+
type: "string",
10+
label: "URL",
11+
description: "URL to scrape. This is a required parameter",
12+
},
13+
browser: {
14+
type: "boolean",
15+
label: "Browser",
16+
description: "Enables using a headless browser for scraping",
17+
optional: true,
18+
},
19+
returnPageSource: {
20+
type: "boolean",
21+
label: "Return Page Source",
22+
description: "Enables returning data returned by the server and unaltered by the browser. Default: false. When true - JS won't be rendered. This feature works only with `browser=true`",
23+
optional: true,
24+
},
25+
cookies: {
26+
type: "string",
27+
label: "Cookies",
28+
description: "Cookies to pass with a scraping request to the target site, i.e.: `cookie_name1=cookie_value1;cookie_name2=cookie_value2`",
29+
optional: true,
30+
},
31+
jsSnippet: {
32+
type: "string",
33+
label: "JS Snippet",
34+
description: "Base64 encoded JS snippet to run once page being loaded in the ScrapingAnt browser. This feature works only with `browser=true`",
35+
optional: true,
36+
},
37+
proxyType: {
38+
type: "string",
39+
label: "Proxy Type",
40+
description: "Specifies proxy type to make request from",
41+
options: constants.PROXY_TYPES,
42+
optional: true,
43+
},
44+
proxyCountry: {
45+
type: "string",
46+
label: "Proxy Country",
47+
description: "Specifies the proxy country to make the request from",
48+
options: constants.PROXY_COUNTRIES,
49+
optional: true,
50+
},
51+
waitForSelector: {
52+
type: "string",
53+
label: "Wait for Selector",
54+
description: "The CSS selector of the element our service will wait for before returning result. This feature works only with `browser=true`",
55+
optional: true,
56+
},
57+
blockResource: {
58+
type: "string[]",
59+
label: "Block Resource",
60+
description: "Prevents cloud browser from loading specified resource types. Available resource types: `document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`, `eventsource`, `websocket`, `manifest`, `other`. This feature works only with `browser=true`",
61+
optional: true,
62+
},
63+
},
564
methods: {
6-
// this.$auth contains connected account data
7-
authKeys() {
8-
console.log(Object.keys(this.$auth));
65+
_baseUrl() {
66+
return "https://api.scrapingant.com/v2";
67+
},
68+
async _makeRequest(opts = {}) {
69+
const {
70+
$ = this,
71+
path,
72+
headers,
73+
...otherOpts
74+
} = opts;
75+
return axios($, {
76+
...otherOpts,
77+
url: this._baseUrl() + path,
78+
headers: {
79+
...headers,
80+
"x-api-key": `${this.$auth.api_token}`,
81+
},
82+
});
83+
},
84+
async generalExtraction(args = {}) {
85+
return this._makeRequest({
86+
path: "/general",
87+
...args,
88+
});
989
},
1090
},
1191
};

0 commit comments

Comments
 (0)