From 10f75cff5818537c6303974650afe577d60eed64 Mon Sep 17 00:00:00 2001 From: Jorge Cortes Date: Tue, 1 Oct 2024 15:09:42 -0500 Subject: [PATCH] Parsera: added new action components --- .../parsera/actions/extract/extract.mjs | 56 +++++++++++++++++ components/parsera/actions/parse/parse.mjs | 50 +++++++++++++++ components/parsera/package.json | 7 ++- components/parsera/parsera.app.mjs | 62 +++++++++++++++++-- pnpm-lock.yaml | 5 +- 5 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 components/parsera/actions/extract/extract.mjs create mode 100644 components/parsera/actions/parse/parse.mjs diff --git a/components/parsera/actions/extract/extract.mjs b/components/parsera/actions/extract/extract.mjs new file mode 100644 index 0000000000000..38bc26825ad6d --- /dev/null +++ b/components/parsera/actions/extract/extract.mjs @@ -0,0 +1,56 @@ +import app from "../../parsera.app.mjs"; + +export default { + key: "parsera-extract", + name: "Extract", + description: "Extract data from a given URL. [See the documentation](https://docs.parsera.org/api/getting-started/)", + version: "0.0.1", + type: "action", + props: { + app, + url: { + type: "string", + label: "URL", + description: "The URL to extract information from.", + }, + attributes: { + propDefinition: [ + app, + "attributes", + ], + }, + proxyCountry: { + propDefinition: [ + app, + "proxyCountry", + ], + }, + }, + methods: { + extract(args = {}) { + return this.app.post({ + path: "/extract", + ...args, + }); + }, + }, + async run({ $ }) { + const { + extract, + url, + attributes, + proxyCountry, + } = this; + + const response = await extract({ + $, + data: { + url, + attributes: Array.isArray(attributes) && attributes.map(JSON.parse), + proxy_country: proxyCountry, + }, + }); + $.export("$summary", "Successfully extracted data from url."); + return response; + }, +}; diff --git a/components/parsera/actions/parse/parse.mjs b/components/parsera/actions/parse/parse.mjs new file mode 100644 index 0000000000000..eac4d2eebd9bd --- /dev/null +++ b/components/parsera/actions/parse/parse.mjs @@ -0,0 +1,50 @@ +import app from "../../parsera.app.mjs"; + +export default { + key: "parsera-parse", + name: "Parse", + description: "Parse data using pre-defined attributes. [See the documentation](https://docs.parsera.org/api/getting-started/)", + version: "0.0.1", + type: "action", + props: { + app, + content: { + type: "string", + label: "Content", + description: "The content to parse. HTML or text content. Eg. `

Hello World

`.", + }, + attributes: { + propDefinition: [ + app, + "attributes", + ], + }, + }, + methods: { + parse(args = {}) { + return this.app.post({ + path: "/parse", + ...args, + }); + }, + }, + async run({ $ }) { + const { + parse, + content, + attributes, + } = this; + + const response = await parse({ + $, + data: { + content, + attributes: Array.isArray(attributes) && attributes.map(JSON.parse), + }, + }); + + $.export("$summary", "Successfully parsed content."); + + return response; + }, +}; diff --git a/components/parsera/package.json b/components/parsera/package.json index 39d319f33c9e1..4689243199bd4 100644 --- a/components/parsera/package.json +++ b/components/parsera/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/parsera", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Parsera Components", "main": "parsera.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "3.0.3" } -} \ No newline at end of file +} diff --git a/components/parsera/parsera.app.mjs b/components/parsera/parsera.app.mjs index 9a1fe95072793..a75cdb2f0584a 100644 --- a/components/parsera/parsera.app.mjs +++ b/components/parsera/parsera.app.mjs @@ -1,11 +1,63 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "parsera", - propDefinitions: {}, + propDefinitions: { + proxyCountry: { + type: "string", + label: "Proxy Country", + description: "Set a specific proxy country for the request.", + optional: true, + async options() { + const response = await this.listProxyCountries(); + return Object.entries(response) + .map(([ + value, + label, + ]) => ({ + label, + value, + })); + }, + }, + attributes: { + type: "string[]", + label: "Attributes", + description: "List of attributes to extract or parse from the content. Format each attribute as a JSON string. Eg. `{\"name\": \"Title\",\"description\": \"News title\"}`.", + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + getUrl(path) { + return `https://api.parsera.org/v1${path}`; + }, + getHeaders(headers) { + return { + "Content-Type": "application/json", + ...headers, + "X-API-KEY": this.$auth.api_key, + }; + }, + _makeRequest({ + $ = this, path, headers, ...args + } = {}) { + return axios($, { + ...args, + url: this.getUrl(path), + headers: this.getHeaders(headers), + }); + }, + post(args = {}) { + return this._makeRequest({ + method: "POST", + ...args, + }); + }, + listProxyCountries(args = {}) { + return this._makeRequest({ + path: "/proxy-countries", + ...args, + }); }, }, -}; \ No newline at end of file +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7aba44d1406f2..cea36ca0edbfb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6938,7 +6938,10 @@ importers: specifiers: {} components/parsera: - specifiers: {} + specifiers: + '@pipedream/platform': 3.0.3 + dependencies: + '@pipedream/platform': 3.0.3 components/parseur: specifiers: