Skip to content

Commit 1c15185

Browse files
committed
[components] Scrapeless - update actions
- Updated @scrapeless-ai/sdk dependency to version 1.6.0 in package.json. - Updated Scraping API action to support Google Search and Google Trends with new parameters for better data retrieval. - Update the way to obtain scrapeless client.
1 parent 827dd7b commit 1c15185

File tree

7 files changed

+6909
-931
lines changed

7 files changed

+6909
-931
lines changed

components/scrapeless/actions/crawler/crawler.mjs

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ export default {
44
key: "scrapeless-crawler",
55
name: "Crawler",
66
description: "Crawl any website at scale and say goodbye to blocks. [See the documentation](https://apidocs.scrapeless.com/api-17509010).",
7-
version: "0.0.2",
7+
version: "0.0.3",
88
type: "action",
99
props: {
1010
scrapeless,
@@ -26,9 +26,34 @@ export default {
2626
reloadProps: true,
2727
},
2828
},
29+
additionalProps() {
30+
const props = {
31+
url: {
32+
type: "string",
33+
label: "URL to Crawl",
34+
description: "If you want to crawl in batches, please refer to the SDK of the document",
35+
},
36+
};
37+
38+
if (this.apiServer === "crawl") {
39+
return {
40+
...props,
41+
limitCrawlPages: {
42+
type: "integer",
43+
label: "Number Of Subpages",
44+
default: 5,
45+
description: "Max number of results to return",
46+
},
47+
};
48+
}
49+
50+
return props;
51+
},
2952
async run({ $ }) {
3053
const {
31-
scrapeless, apiServer, ...inputProps
54+
scrapeless,
55+
apiServer,
56+
...inputProps
3257
} = this;
3358

3459
const browserOptions = {
@@ -40,50 +65,28 @@ export default {
4065

4166
let response;
4267

68+
const client = await scrapeless._scrapelessClient();
69+
4370
if (apiServer === "crawl") {
4471
response =
45-
await scrapeless._scrapelessClient().scrapingCrawl.crawl.crawlUrl(inputProps.url, {
72+
await client.scrapingCrawl.crawl.crawlUrl(inputProps.url, {
4673
limit: inputProps.limitCrawlPages,
4774
browserOptions,
4875
});
4976
}
5077

5178
if (apiServer === "scrape") {
5279
response =
53-
await scrapeless._scrapelessClient().scrapingCrawl.scrape.scrapeUrl(inputProps.url, {
80+
await client.scrapingCrawl.scrape.scrapeUrl(inputProps.url, {
5481
browserOptions,
5582
});
5683
}
5784

5885
if (response?.status === "completed" && response?.data) {
59-
$.export("$summary", `Successfully retrieved crawling results for ${inputProps.url}`);
86+
$.export("$summary", `Successfully retrieved crawling results for \`${inputProps.url}\``);
6087
return response.data;
6188
} else {
6289
throw new Error(response?.error || "Failed to retrieve crawling results");
6390
}
6491
},
65-
additionalProps() {
66-
const { apiServer } = this;
67-
68-
const props = {};
69-
70-
if (apiServer === "crawl" || apiServer === "scrape") {
71-
props.url = {
72-
type: "string",
73-
label: "URL to Crawl",
74-
description: "If you want to crawl in batches, please refer to the SDK of the document",
75-
};
76-
}
77-
78-
if (apiServer === "crawl") {
79-
props.limitCrawlPages = {
80-
type: "integer",
81-
label: "Number Of Subpages",
82-
default: 5,
83-
description: "Max number of results to return",
84-
};
85-
}
86-
87-
return props;
88-
},
8992
};

0 commit comments

Comments
 (0)