diff --git a/components/apify/actions/get-dataset-items/get-dataset-items.mjs b/components/apify/actions/get-dataset-items/get-dataset-items.mjs new file mode 100644 index 0000000000000..54f483cf90d40 --- /dev/null +++ b/components/apify/actions/get-dataset-items/get-dataset-items.mjs @@ -0,0 +1,87 @@ +import apify from "../../apify.app.mjs"; +import { LIMIT } from "../../common/constants.mjs"; + +export default { + key: "apify-get-dataset-items", + name: "Get Dataset Items", + description: "Returns data stored in a dataset. [See the documentation](https://docs.apify.com/api/v2/dataset-items-get)", + version: "0.0.1", + type: "action", + props: { + apify, + datasetId: { + propDefinition: [ + apify, + "datasetId", + ], + }, + clean: { + propDefinition: [ + apify, + "clean", + ], + }, + fields: { + propDefinition: [ + apify, + "fields", + ], + }, + omit: { + propDefinition: [ + apify, + "omit", + ], + }, + flatten: { + propDefinition: [ + apify, + "flatten", + ], + }, + maxResults: { + propDefinition: [ + apify, + "maxResults", + ], + }, + }, + async run({ $ }) { + const params = { + limit: LIMIT, + offset: 0, + clean: this.clean, + fields: this.fields && this.fields.join(), + omit: this.omit && this.omit.join(), + flatten: this.flatten && this.flatten.join(), + }; + + const results = []; + let total; + + do { + const items = await this.apify.listDatasetItems({ + $, + datasetId: this.datasetId, + params, + }); + results.push(...items); + if (results.length >= this.maxResults) { + break; + } + total = items?.length; + params.offset += LIMIT; + } while (total); + + if (results.length > this.maxResults) { + results.length = this.maxResults; + } + + if (results.length > 0) { + $.export("$summary", `Successfully retrieved ${results.length} item${results.length === 1 + ? "" + : "s"}`); + } + return results; + }, +}; diff --git a/components/apify/actions/run-actor/run-actor.mjs b/components/apify/actions/run-actor/run-actor.mjs index c154add225786..ce776c00b1be7 100644 --- a/components/apify/actions/run-actor/run-actor.mjs +++ b/components/apify/actions/run-actor/run-actor.mjs @@ -5,7 +5,7 @@ export default { key: "apify-run-actor", name: "Run Actor", description: "Performs an execution of a selected actor in Apify. [See the documentation](https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { apify, diff --git a/components/apify/actions/run-task-synchronously/run-task-synchronously.mjs b/components/apify/actions/run-task-synchronously/run-task-synchronously.mjs new file mode 100644 index 0000000000000..52a3ef7a4fa4a --- /dev/null +++ b/components/apify/actions/run-task-synchronously/run-task-synchronously.mjs @@ -0,0 +1,87 @@ +import apify from "../../apify.app.mjs"; + +export default { + key: "apify-run-task-synchronously", + name: "Run Task Synchronously", + description: "Run a specific task and return its dataset items. [See the documentation](https://docs.apify.com/api/v2/actor-task-run-sync-get-dataset-items-get)", + version: "0.0.1", + type: "action", + props: { + apify, + taskId: { + propDefinition: [ + apify, + "taskId", + ], + description: "The ID of the task to run", + }, + timeout: { + type: "integer", + label: "Timeout", + description: "Optional timeout for the run, in seconds. By default, the run uses a timeout specified in the task settings.", + optional: true, + }, + memory: { + type: "integer", + label: "Memory", + description: "Memory limit for the run, in megabytes. The amount of memory can be set to a power of 2 with a minimum of 128. By default, the run uses a memory limit specified in the task settings.", + optional: true, + }, + build: { + type: "string", + label: "Build", + description: "Specifies the Actor build to run. It can be either a build tag or build number. By default, the run uses the build specified in the task settings (typically latest).", + optional: true, + }, + clean: { + propDefinition: [ + apify, + "clean", + ], + }, + fields: { + propDefinition: [ + apify, + "fields", + ], + }, + omit: { + propDefinition: [ + apify, + "omit", + ], + }, + flatten: { + propDefinition: [ + apify, + "flatten", + ], + }, + maxResults: { + propDefinition: [ + apify, + "maxResults", + ], + }, + }, + async run({ $ }) { + const response = await this.apify.runTaskSynchronously({ + $, + taskId: this.taskId, + params: { + timeout: this.timeout, + memory: this.memory, + build: this.build, + clean: this.clean, + fields: this.fields && this.fields.join(), + omit: this.omit && this.omit.join(), + flatten: this.flatten && this.flatten.join(), + maxItems: this.maxResults, + }, + }); + + $.export("$summary", `Successfully ran task with ID: ${this.taskId}`); + + return response; + }, +}; diff --git a/components/apify/actions/scrape-single-url/scrape-single-url.mjs b/components/apify/actions/scrape-single-url/scrape-single-url.mjs index 24c4f92c5b7c0..3cb7d3ab2bbbb 100644 --- a/components/apify/actions/scrape-single-url/scrape-single-url.mjs +++ b/components/apify/actions/scrape-single-url/scrape-single-url.mjs @@ -5,7 +5,7 @@ export default { key: "apify-scrape-single-url", name: "Scrape Single URL", description: "Executes a scraper on a specific website and returns its content as text. This action is perfect for extracting content from a single page.", - version: "0.0.1", + version: "0.0.2", type: "action", props: { apify, diff --git a/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs b/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs index a9318261b1927..fc3c4ddf6cbec 100644 --- a/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs +++ b/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs @@ -5,7 +5,7 @@ export default { key: "apify-set-key-value-store-record", name: "Set Key-Value Store Record", description: "Create or update a record in the key-value store of Apify. [See the documentation](https://docs.apify.com/api/v2#/reference/key-value-stores/record-collection/put-record)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { apify, diff --git a/components/apify/apify.app.mjs b/components/apify/apify.app.mjs index 82ffa8830bd48..5452ce12ca802 100644 --- a/components/apify/apify.app.mjs +++ b/components/apify/apify.app.mjs @@ -86,6 +86,56 @@ export default { })); }, }, + datasetId: { + type: "string", + label: "Dataset ID", + description: "The ID of the dataset to retrieve items within", + async options({ page }) { + const { data: { items } } = await this.listDatasets({ + params: { + offset: LIMIT * page, + limit: LIMIT, + }, + }); + return items?.map(({ + id: value, name: label, + }) => ({ + label, + value, + })) || []; + }, + }, + clean: { + type: "boolean", + label: "Clean", + description: "Return only non-empty items and skips hidden fields (i.e. fields starting with the # character)", + optional: true, + }, + fields: { + type: "string[]", + label: "Fields", + description: "An array of fields which should be picked from the items, only these fields will remain in the resulting record objects.", + optional: true, + }, + omit: { + type: "string[]", + label: "Omit", + description: "An array of fields which should be omitted from the items", + optional: true, + }, + flatten: { + type: "string[]", + label: "Flatten", + description: "An array of fields which should transform nested objects into flat structures. For example, with `flatten=\"foo\"` the object `{\"foo\":{\"bar\": \"hello\"}}` is turned into `{\"foo.bar\": \"hello\"}`", + optional: true, + }, + maxResults: { + type: "integer", + label: "Max Results", + description: "The maximum number of items to return", + default: LIMIT, + optional: true, + }, }, methods: { _baseUrl() { @@ -168,6 +218,28 @@ export default { ...opts, }); }, + listDatasets(opts = {}) { + return this._makeRequest({ + path: "/datasets", + ...opts, + }); + }, + listDatasetItems({ + datasetId, ...opts + }) { + return this._makeRequest({ + path: `/datasets/${datasetId}/items`, + ...opts, + }); + }, + runTaskSynchronously({ + taskId, ...opts + }) { + return this._makeRequest({ + path: `/actor-tasks/${taskId}/run-sync-get-dataset-items`, + ...opts, + }); + }, setKeyValueStoreRecord({ storeId, recordKey, ...opts }) { diff --git a/components/apify/package.json b/components/apify/package.json index 7e5ca56745aa6..93b7a54f2e40c 100644 --- a/components/apify/package.json +++ b/components/apify/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/apify", - "version": "0.1.0", + "version": "0.2.0", "description": "Pipedream Apify Components", "main": "apify.app.mjs", "keywords": [ @@ -13,7 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.6.5" + "@pipedream/platform": "^3.0.3" } } - diff --git a/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs b/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs index 00a355a91045e..0682efa6a181a 100644 --- a/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs +++ b/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs @@ -6,7 +6,7 @@ export default { key: "apify-new-finished-actor-run-instant", name: "New Finished Actor Run (Instant)", description: "Emit new event when a selected actor is run and finishes.", - version: "0.0.1", + version: "0.0.2", type: "source", dedupe: "unique", props: { diff --git a/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs b/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs index 0dc18ed997399..94146ecf19f2a 100644 --- a/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs +++ b/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs @@ -6,7 +6,7 @@ export default { key: "apify-new-finished-task-run-instant", name: "New Finished Task Run (Instant)", description: "Emit new event when a selected task is run and finishes.", - version: "0.0.1", + version: "0.0.2", type: "source", dedupe: "unique", props: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3b30391b04c80..2930725c302ce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -874,8 +874,8 @@ importers: components/apify: dependencies: '@pipedream/platform': - specifier: ^1.6.5 - version: 1.6.6 + specifier: ^3.0.3 + version: 3.0.3 components/apilio: {} @@ -2147,8 +2147,7 @@ importers: components/changenow: {} - components/changes_page: - specifiers: {} + components/changes_page: {} components/channeladvisor: dependencies: