New Components - webscraping_ai

components/webscraping_ai/actions/ask-question/ask-question.mjs

@@ -0,0 +1,34 @@
+import webscrapingAI from "../../webscraping_ai.app.mjs";
+
+export default {
+  key: "webscraping_ai-ask-question",
+  name: "Ask Question about Webpage",
+  description: "Gets an answer to a question about a given webpage. [See the documentation](https://webscraping.ai/docs#tag/AI/operation/getQuestion)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    webscrapingAI,
+    targetUrl: {
+      propDefinition: [
+        webscrapingAI,
+        "targetUrl",
+      ],
+    },
+    question: {
+      type: "string",
+      label: "Question",
+      description: "The question to ask about the given webpage. E.g. `What is the summary of this page content?`",
+    },
+  },
+  async run({ $ }) {
+    const response = await this.webscrapingAI.getAnswerToQuestion({
+      $,
+      params: {
+        url: this.targetUrl,
+        question: this.question,
+      },
+    });
+    $.export("$summary", "Successfully retrieved answer to question");
+    return response;
+  },
+};

components/webscraping_ai/actions/scrape-website-html/scrape-website-html.mjs

@@ -0,0 +1,29 @@
+import webscrapingAI from "../../webscraping_ai.app.mjs";
+
+export default {
+  key: "webscraping_ai-scrape-website-html",
+  name: "Scrape Website HTML",
+  description: "Returns the full HTML content of a webpage specified by the URL. [See the documentation](https://webscraping.ai/docs#tag/HTML/operation/getHTML):",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    webscrapingAI,
+    targetUrl: {
+      propDefinition: [
+        webscrapingAI,
+        "targetUrl",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const response = await this.webscrapingAI.pageHtmlByUrl({
+      $,
+      params: {
+        url: this.targetUrl,
+        format: "json",
+      },
+    });
+    $.export("$summary", `Successfully scraped HTML of URL ${this.targetUrl}`);
+    return response;
+  },
+};

components/webscraping_ai/actions/scrape-website-text/scrape-website-text.mjs

@@ -0,0 +1,48 @@
+import webscrapingAI from "../../webscraping_ai.app.mjs";
+
+export default {
+  key: "webscraping_ai-scrape-website-text",
+  name: "Scrape Website Text",
+  description: "Returns the visible text content of a webpage specified by the URL. [See the documentation](https://webscraping.ai/docs#tag/Text/operation/getText).",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    webscrapingAI,
+    targetUrl: {
+      propDefinition: [
+        webscrapingAI,
+        "targetUrl",
+      ],
+    },
+    textFormat: {
+      type: "string",
+      label: "Text Format",
+      description: "The format of the returned text content. Default: `json`",
+      options: [
+        "plain",
+        "xml",
+        "json",
+      ],
+      default: "json",
+      optional: true,
+    },
+    returnLinks: {
+      type: "boolean",
+      label: "Return Links",
+      description: "Whether to include links in the returned text content. Works only when Text Format is `json`.",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const response = await this.webscrapingAI.pageTextByUrl({
+      $,
+      params: {
+        url: this.targetUrl,
+        text_format: this.textFormat,
+        return_links: this.returnLinks,
+      },
+    });
+    $.export("$summary", `Successfully scraped text from ${this.targetUrl}`);
+    return response;
+  },
+};

components/webscraping_ai/package.json

@@ -1,16 +1,18 @@
 {
   "name": "@pipedream/webscraping_ai",
-  "version": "0.0.3",
+  "version": "0.1.0",
   "description": "Pipedream WebScraping.AI Components",
-  "main": "dist/app/webscraping_ai.app.mjs",
+  "main": "webscraping_ai.app.mjs",
   "keywords": [
     "pipedream",
     "webscraping_ai"
   ],
-  "files": ["dist"],
   "homepage": "https://pipedream.com/apps/webscraping_ai",
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.0.3"
   }
 }

components/webscraping_ai/webscraping_ai.app.mjs

@@ -0,0 +1,51 @@
+import { axios } from "@pipedream/platform";
+
+export default {
+  type: "app",
+  app: "webscraping_ai",
+  propDefinitions: {
+    targetUrl: {
+      type: "string",
+      label: "Target URL",
+      description: "The URL of the webpage to scrape.",
+    },
+  },
+  methods: {
+    _baseUrl() {
+      return "https://api.webscraping.ai";
+    },
+    _makeRequest({
+      $ = this,
+      path,
+      params,
+      ...otherOpts
+    }) {
+      return axios($, {
+        url: `${this._baseUrl()}${path}`,
+        params: {
+          ...params,
+          api_key: this.$auth.api_key,
+        },
+        ...otherOpts,
+      });
+    },
+    pageHtmlByUrl(opts = {}) {
+      return this._makeRequest({
+        path: "/html",
+        ...opts,
+      });
+    },
+    pageTextByUrl(opts = {}) {
+      return this._makeRequest({
+        path: "/text",
+        ...opts,
+      });
+    },
+    getAnswerToQuestion(opts = {}) {
+      return this._makeRequest({
+        path: "/ai/question",
+        ...opts,
+      });
+    },
+  },
+};