Skip to content

Commit b70c366

Browse files
committed
some adjusts
1 parent d5fe429 commit b70c366

File tree

1 file changed

+33
-19
lines changed

1 file changed

+33
-19
lines changed

components/scrapeless/actions/submit-scrape-job/submit-scrape-job.mjs

Lines changed: 33 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { ConfigurationError } from "@pipedream/platform";
12
import { ACTOR_OPTIONS } from "../../common/constants.mjs";
23
import { parseObject } from "../../common/utils.mjs";
34
import scrapeless from "../../scrapeless.app.mjs";
@@ -34,29 +35,42 @@ export default {
3435
description: "Additional input parameters if you need to pass a specific configuration based on the actor. [See the documentation](https://apidocs.scrapeless.com/) for further details.",
3536
optional: true,
3637
},
38+
asyncMode: {
39+
type: "boolean",
40+
label: "Async Mode",
41+
description: "Whether to run the scrape job in asynchronous mode. If set to true, the job will be processed in the background.",
42+
},
3743
},
3844
async run({ $ }) {
39-
const data = {
40-
actor: this.actor,
41-
input: parseObject(this.additionalInput),
42-
async: true,
43-
};
44-
45-
if (this.inputUrl) {
46-
data.input.url = this.inputUrl;
47-
}
48-
if (this.proxyCountry) {
49-
data.proxy = {
50-
country: this.proxyCountry,
45+
try {
46+
const data = {
47+
actor: this.actor,
48+
input: parseObject(this.additionalInput),
5149
};
52-
}
5350

54-
const response = await this.scrapeless.submitScrapeJob({
55-
$,
56-
data,
57-
});
51+
if (this.asyncMode) {
52+
data.async = this.asyncMode;
53+
}
54+
if (this.inputUrl) {
55+
data.input.url = this.inputUrl;
56+
}
57+
if (this.proxyCountry) {
58+
data.proxy = {
59+
country: this.proxyCountry,
60+
};
61+
}
62+
63+
const response = await this.scrapeless.submitScrapeJob({
64+
$,
65+
data,
66+
});
5867

59-
$.export("$summary", `Successfully submitted scrape job with ID: ${response.taskId}`);
60-
return response;
68+
$.export("$summary", this.asyncMode
69+
? `Successfully submitted scrape job with ID: ${response.taskId}`
70+
: "Successfully scraped the target configuration.");
71+
return response;
72+
} catch ({ response }) {
73+
throw new ConfigurationError(response.data.message);
74+
}
6175
},
6276
};

0 commit comments

Comments
 (0)