webscraping_ai init

Author: michelle0927

components/webscraping_ai/actions/ask-question/ask-question.mjs

@@ -0,0 +1,30 @@
+import webscraping_ai from "../../webscraping_ai.app.mjs";
+import { axios } from "@pipedream/platform";
+
+export default {
+  key: "webscraping_ai-ask-question",
+  name: "Ask Question about Webpage",
+  description: "Gets an answer to a question about a given webpage. [See the documentation](https://webscraping.ai/docs)",
+  version: "0.0.{{ts}}",
+  type: "action",
+  props: {
+    webscraping_ai,
+    targetUrl: {
+      propDefinition: [
+        webscraping_ai,
+        "targetUrl",
+      ],
+    },
+    question: {
+      propDefinition: [
+        webscraping_ai,
+        "question",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const response = await this.webscraping_ai.getAnswerToQuestion();
+    $.export("$summary", `Answer: ${response.answer}`);
+    return response;
+  },
+};

components/webscraping_ai/actions/scrape-website-html/scrape-website-html.mjs

@@ -0,0 +1,45 @@
+import webscraping_ai from "../../webscraping_ai.app.mjs";
+import { axios } from "@pipedream/platform";
+
+export default {
+  key: "webscraping_ai-scrape-website-html",
+  name: "Scrape Website HTML",
+  description: "Starts a new web scraping job with specified configurations. [See the documentation]():",
+  version: "0.0.{{ts}}",
+  type: "action",
+  props: {
+    webscraping_ai,
+    targetUrl: {
+      propDefinition: [
+        "webscraping_ai",
+        "targetUrl",
+      ],
+    },
+    selectors: {
+      propDefinition: [
+        "webscraping_ai",
+        "selectors",
+      ],
+      optional: true,
+    },
+    renderingMode: {
+      propDefinition: [
+        "webscraping_ai",
+        "renderingMode",
+      ],
+      optional: true,
+    },
+    headers: {
+      propDefinition: [
+        "webscraping_ai",
+        "headers",
+      ],
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const response = await this.webscraping_ai.startScrapingJob();
+    $.export("$summary", `Started scraping job for URL ${this.targetUrl}`);
+    return response;
+  },
+};

components/webscraping_ai/actions/scrape-website-text/scrape-website-text.mjs

@@ -0,0 +1,41 @@
+import webscraping_ai from "../../webscraping_ai.app.mjs";
+import { axios } from "@pipedream/platform";
+
+export default {
+  key: "webscraping_ai-scrape-website-text",
+  name: "Scrape Website Text",
+  description: "Returns the visible text content of a webpage specified by the URL. [See the documentation]().",
+  version: "0.0.{{ts}}",
+  type: "action",
+  props: {
+    webscraping_ai: {
+      type: "app",
+      app: "webscraping_ai",
+    },
+    targetUrl: {
+      propDefinition: [
+        webscraping_ai,
+        "targetUrl",
+      ],
+    },
+    textFormat: {
+      propDefinition: [
+        webscraping_ai,
+        "textFormat",
+      ],
+      optional: true,
+    },
+    returnLinks: {
+      propDefinition: [
+        webscraping_ai,
+        "returnLinks",
+      ],
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const response = await this.webscraping_ai.getVisibleTextContent();
+    $.export("$summary", `Successfully scraped text from ${this.targetUrl}`);
+    return response;
+  },
+};

components/webscraping_ai/webscraping_ai.app.mjs

@@ -0,0 +1,117 @@
+import { axios } from "@pipedream/platform";
+
+export default {
+  type: "app",
+  app: "webscraping_ai",
+  version: "0.0.{{ts}}",
+  propDefinitions: {
+    targetUrl: {
+      type: "string",
+      label: "Target URL",
+      description: "The URL of the webpage to scrape.",
+    },
+    selectors: {
+      type: "string[]",
+      label: "Selectors",
+      description: "Optional CSS selectors to target specific elements on the page.",
+      optional: true,
+    },
+    renderingMode: {
+      type: "string",
+      label: "Rendering Mode",
+      description: "The mode to render the page (e.g., 'light', 'dark').",
+      optional: true,
+    },
+    headers: {
+      type: "string[]",
+      label: "Headers",
+      description: "Optional HTTP headers to include in the request, as JSON strings.",
+      optional: true,
+    },
+    textFormat: {
+      type: "string",
+      label: "Text Format",
+      description: "The format of the returned text content (e.g., 'plain', 'html').",
+      optional: true,
+    },
+    returnLinks: {
+      type: "boolean",
+      label: "Return Links",
+      description: "Whether to include links in the returned text content.",
+      optional: true,
+    },
+    question: {
+      type: "string",
+      label: "Question",
+      description: "The question to ask about the given webpage.",
+    },
+  },
+  methods: {
+    _baseUrl() {
+      return "https://api.webscraping.ai";
+    },
+    async _makeRequest(opts = {}) {
+      const {
+        $, method = "GET", path = "/", headers = {}, ...otherOpts
+      } = opts;
+      return axios($, {
+        method,
+        url: `${this._baseUrl()}${path}`,
+        headers: {
+          ...headers,
+          "User-Agent": "@PipedreamHQ/pipedream v0.1",
+          "Authorization": `Bearer ${this.$auth.api_key}`,
+        },
+        ...otherOpts,
+      });
+    },
+    async startScrapingJob() {
+      const data = {
+        url: this.targetUrl,
+      };
+      if (this.selectors) data.selectors = this.selectors;
+      if (this.renderingMode) data.rendering_mode = this.renderingMode;
+      if (this.headers) {
+        data.headers = this.headers.reduce((acc, headerStr) => {
+          try {
+            const header = JSON.parse(headerStr);
+            return {
+              ...acc,
+              ...header,
+            };
+          } catch (e) {
+            return acc;
+          }
+        }, {});
+      }
+      return this._makeRequest({
+        method: "POST",
+        path: "/scraping-jobs",
+        data,
+      });
+    },
+    async getVisibleTextContent() {
+      const params = {
+        url: this.targetUrl,
+      };
+      if (this.textFormat) params.text_format = this.textFormat;
+      if (this.returnLinks !== undefined) params.return_links = this.returnLinks;
+      return this._makeRequest({
+        method: "GET",
+        path: "/text-content",
+        params,
+      });
+    },
+    async getAnswerToQuestion() {
+      const data = {
+        url: this.targetUrl,
+        question: this.question,
+      };
+      return this._makeRequest({
+        method: "POST",
+        path: "/answer",
+        data,
+      });
+    },
+  },
+};