|
| 1 | +import { ConfigurationError } from "@pipedream/platform"; |
1 | 2 | import { ACTOR_OPTIONS } from "../../common/constants.mjs"; |
2 | 3 | import { parseObject } from "../../common/utils.mjs"; |
3 | 4 | import scrapeless from "../../scrapeless.app.mjs"; |
@@ -34,29 +35,42 @@ export default { |
34 | 35 | description: "Additional input parameters if you need to pass a specific configuration based on the actor. [See the documentation](https://apidocs.scrapeless.com/) for further details.", |
35 | 36 | optional: true, |
36 | 37 | }, |
| 38 | + asyncMode: { |
| 39 | + type: "boolean", |
| 40 | + label: "Async Mode", |
| 41 | + description: "Whether to run the scrape job in asynchronous mode. If set to true, the job will be processed in the background.", |
| 42 | + }, |
37 | 43 | }, |
38 | 44 | async run({ $ }) { |
39 | | - const data = { |
40 | | - actor: this.actor, |
41 | | - input: parseObject(this.additionalInput), |
42 | | - async: true, |
43 | | - }; |
44 | | - |
45 | | - if (this.inputUrl) { |
46 | | - data.input.url = this.inputUrl; |
47 | | - } |
48 | | - if (this.proxyCountry) { |
49 | | - data.proxy = { |
50 | | - country: this.proxyCountry, |
| 45 | + try { |
| 46 | + const data = { |
| 47 | + actor: this.actor, |
| 48 | + input: parseObject(this.additionalInput), |
51 | 49 | }; |
52 | | - } |
53 | 50 |
|
54 | | - const response = await this.scrapeless.submitScrapeJob({ |
55 | | - $, |
56 | | - data, |
57 | | - }); |
| 51 | + if (this.asyncMode) { |
| 52 | + data.async = this.asyncMode; |
| 53 | + } |
| 54 | + if (this.inputUrl) { |
| 55 | + data.input.url = this.inputUrl; |
| 56 | + } |
| 57 | + if (this.proxyCountry) { |
| 58 | + data.proxy = { |
| 59 | + country: this.proxyCountry, |
| 60 | + }; |
| 61 | + } |
| 62 | + |
| 63 | + const response = await this.scrapeless.submitScrapeJob({ |
| 64 | + $, |
| 65 | + data, |
| 66 | + }); |
58 | 67 |
|
59 | | - $.export("$summary", `Successfully submitted scrape job with ID: ${response.taskId}`); |
60 | | - return response; |
| 68 | + $.export("$summary", this.asyncMode |
| 69 | + ? `Successfully submitted scrape job with ID: ${response.taskId}` |
| 70 | + : "Successfully scraped the target configuration."); |
| 71 | + return response; |
| 72 | + } catch ({ response }) { |
| 73 | + throw new ConfigurationError(response.data.message); |
| 74 | + } |
61 | 75 | }, |
62 | 76 | }; |
0 commit comments