Skip to content

Commit baa56ab

Browse files
luancazarineluancazarine
committed
scrapeninja init
1 parent 915a70a commit baa56ab

File tree

4 files changed

+548
-1
lines changed

4 files changed

+548
-1
lines changed

‎components/scrapeninja/actions/non-js-scraping/non-js-scraping.mjs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import scrapeninja from "../../scrapeninja.app.mjs";
2+
import { axios } from "@pipedream/platform";
3+
4+
export default {
5+
key: "scrapeninja-non-js-scraping",
6+
name: "ScrapeNinja Non-JS Scraping",
7+
description: "Use ScrapeNinja's high-performance non-JS scraping endpoint. [See the documentation]()",
8+
version: "0.0.{{ts}}",
9+
type: "action",
10+
props: {
11+
scrapeninja: {
12+
type: "app",
13+
app: "scrapeninja",
14+
},
15+
url: {
16+
propDefinition: [
17+
scrapeninja,
18+
"url",
19+
],
20+
},
21+
headers: {
22+
propDefinition: [
23+
scrapeninja,
24+
"headers",
25+
],
26+
optional: true,
27+
},
28+
retrynum: {
29+
propDefinition: [
30+
scrapeninja,
31+
"retrynum",
32+
],
33+
optional: true,
34+
},
35+
geo: {
36+
propDefinition: [
37+
scrapeninja,
38+
"geo",
39+
],
40+
optional: true,
41+
},
42+
proxy: {
43+
propDefinition: [
44+
scrapeninja,
45+
"proxy",
46+
],
47+
optional: true,
48+
},
49+
followredirects: {
50+
propDefinition: [
51+
scrapeninja,
52+
"followredirects",
53+
],
54+
optional: true,
55+
},
56+
timeout: {
57+
propDefinition: [
58+
scrapeninja,
59+
"timeout",
60+
],
61+
optional: true,
62+
},
63+
textnotexpected: {
64+
propDefinition: [
65+
scrapeninja,
66+
"textnotexpected",
67+
],
68+
optional: true,
69+
},
70+
statusnotexpected: {
71+
propDefinition: [
72+
scrapeninja,
73+
"statusnotexpected",
74+
],
75+
optional: true,
76+
},
77+
extractor: {
78+
propDefinition: [
79+
scrapeninja,
80+
"extractor",
81+
],
82+
optional: true,
83+
},
84+
},
85+
async run({ $ }) {
86+
const response = await this.scrapeninja.scrapeNonJs();
87+
$.export("$summary", "Successfully scraped the URL");
88+
return response;
89+
},
90+
};

‎components/scrapeninja/actions/scraping-with-js-rendering/scraping-with-js-rendering.mjs‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
import scrapeninja from "../../scrapeninja.app.mjs";
2+
import { axios } from "@pipedream/platform";
3+
4+
export default {
5+
key: "scrapeninja-scraping-with-js-rendering",
6+
name: "ScrapeNinja Scraping with JS Rendering",
7+
description: "Uses the ScrapeNinja real Chrome browser engine to scrape pages that require JS rendering. [See the documentation](https://scrapeninja.net/docs/api-reference/scrape-js/)",
8+
version: "0.0.{{ts}}",
9+
type: "action",
10+
props: {
11+
scrapeninja,
12+
url: {
13+
propDefinition: [
14+
scrapeninja,
15+
"url",
16+
],
17+
},
18+
waitforselector: {
19+
propDefinition: [
20+
scrapeninja,
21+
"waitforselector",
22+
],
23+
optional: true,
24+
},
25+
postwaittime: {
26+
propDefinition: [
27+
scrapeninja,
28+
"postwaittime",
29+
],
30+
optional: true,
31+
},
32+
dumpiframe: {
33+
propDefinition: [
34+
scrapeninja,
35+
"dumpiframe",
36+
],
37+
optional: true,
38+
},
39+
waitforselectoriframe: {
40+
propDefinition: [
41+
scrapeninja,
42+
"waitforselectoriframe",
43+
],
44+
optional: true,
45+
},
46+
extractortargetiframe: {
47+
propDefinition: [
48+
scrapeninja,
49+
"extractortargetiframe",
50+
],
51+
optional: true,
52+
},
53+
headers: {
54+
propDefinition: [
55+
scrapeninja,
56+
"headers",
57+
],
58+
optional: true,
59+
},
60+
retrynum: {
61+
propDefinition: [
62+
scrapeninja,
63+
"retrynum",
64+
],
65+
optional: true,
66+
},
67+
geo: {
68+
propDefinition: [
69+
scrapeninja,
70+
"geo",
71+
],
72+
optional: true,
73+
},
74+
proxy: {
75+
propDefinition: [
76+
scrapeninja,
77+
"proxy",
78+
],
79+
optional: true,
80+
},
81+
timeout: {
82+
propDefinition: [
83+
scrapeninja,
84+
"timeout",
85+
],
86+
optional: true,
87+
},
88+
textnotexpected: {
89+
propDefinition: [
90+
scrapeninja,
91+
"textnotexpected",
92+
],
93+
optional: true,
94+
},
95+
statusnotexpected: {
96+
propDefinition: [
97+
scrapeninja,
98+
"statusnotexpected",
99+
],
100+
optional: true,
101+
},
102+
blockimages: {
103+
propDefinition: [
104+
scrapeninja,
105+
"blockimages",
106+
],
107+
optional: true,
108+
},
109+
blockmedia: {
110+
propDefinition: [
111+
scrapeninja,
112+
"blockmedia",
113+
],
114+
optional: true,
115+
},
116+
screenshot: {
117+
propDefinition: [
118+
scrapeninja,
119+
"screenshot",
120+
],
121+
optional: true,
122+
},
123+
catchajaxheadersurlmask: {
124+
propDefinition: [
125+
scrapeninja,
126+
"catchajaxheadersurlmask",
127+
],
128+
optional: true,
129+
},
130+
viewportWidth: {
131+
propDefinition: [
132+
scrapeninja,
133+
"viewportwitdh",
134+
],
135+
optional: true,
136+
},
137+
viewportHeight: {
138+
propDefinition: [
139+
scrapeninja,
140+
"viewportheight",
141+
],
142+
optional: true,
143+
},
144+
viewportDeviceScaleFactor: {
145+
propDefinition: [
146+
scrapeninja,
147+
"viewportdevicescalefactor",
148+
],
149+
optional: true,
150+
},
151+
viewportHasTouch: {
152+
propDefinition: [
153+
scrapeninja,
154+
"viewporthastouch",
155+
],
156+
optional: true,
157+
},
158+
viewportIsMobile: {
159+
propDefinition: [
160+
scrapeninja,
161+
"viewportismobile",
162+
],
163+
optional: true,
164+
},
165+
viewportIsLandscape: {
166+
propDefinition: [
167+
scrapeninja,
168+
"viewportislandscape",
169+
],
170+
optional: true,
171+
},
172+
extractor: {
173+
propDefinition: [
174+
scrapeninja,
175+
"extractor",
176+
],
177+
optional: true,
178+
},
179+
},
180+
async run({ $ }) {
181+
const viewport = {
182+
width: this.viewportWidth,
183+
height: this.viewportHeight,
184+
deviceScaleFactor: this.viewportDeviceScaleFactor,
185+
hasTouch: this.viewportHasTouch,
186+
isMobile: this.viewportIsMobile,
187+
isLandscape: this.viewportIsLandscape,
188+
};
189+
190+
const response = await this.scrapeninja.scrapeJs({
191+
url: this.url,
192+
waitForSelector: this.waitforselector,
193+
postWaitTime: this.postwaittime,
194+
dumpIframe: this.dumpiframe,
195+
waitForSelectorIframe: this.waitforselectoriframe,
196+
extractorTargetIframe: this.extractortargetiframe,
197+
headers: this.headers,
198+
retryNum: this.retrynum,
199+
geo: this.geo,
200+
proxy: this.proxy,
201+
timeout: this.timeout,
202+
textNotExpected: this.textnotexpected,
203+
statusNotExpected: this.statusnotexpected,
204+
blockImages: this.blockimages,
205+
blockMedia: this.blockmedia,
206+
screenshot: this.screenshot,
207+
catchAjaxHeadersUrlMask: this.catchajaxheadersurlmask,
208+
viewport,
209+
extractor: this.extractor,
210+
});
211+
212+
$.export("$summary", `Successfully scraped ${this.url} with JS rendering`);
213+
return response;
214+
},
215+
};

‎components/scrapeninja/app/scrapeninja.app.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ export default defineApp({
1010
console.log(Object.keys(this.$auth));
1111
},
1212
},
13-
});
13+
});

0 commit comments

Comments
 (0)