Crawlbase - new action crawl-url

components/crawlbase/actions/crawl-url/crawl-url.mjs

@@ -0,0 +1,64 @@
+import crawlbase from "../../crawlbase.app.mjs";
+import constants from "../../common/constants.mjs";
+
+export default {
+  key: "crawlbase-crawl-url",
+  name: "Crawl URL",
+  description: "Crawl a URL. [See the documentation](https://crawlbase.com/docs/crawling-api/)",
+  version: "0.0.1",
+  type: "action",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: false,
+  },
+  props: {
+    crawlbase,
+    url: {
+      type: "string",
+      label: "URL",
+      description: "The URL to crawl",
+    },
+    scraper: {
+      type: "string",
+      label: "Scraper",
+      description: "The scraper to use",
+      options: constants.SCRAPERS,
+      optional: true,
+    },
+    screenshot: {
+      type: "boolean",
+      label: "Screenshot",
+      description: "Set to `true` to take a screenshot of the page. Must use a JavaScript token for authentication.",
+      optional: true,
+    },
+    store: {
+      type: "boolean",
+      label: "Store",
+      description: "Set to `true` to store a copy of the API response in the Crawlbase Cloud Storage",
+      optional: true,
+    },
+    getHeaders: {
+      type: "boolean",
+      label: "Get Headers",
+      description: "Set to `true` to get the headers of the page",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const response = await this.crawlbase.makeRequest({
+      $,
+      params: {
+        url: this.url,
+        scraper: this.scraper,
+        screenshot: this.screenshot,
+        store: this.store,
+        get_headers: this.getHeaders,
+        format: "json",
+      },
+    });
+
+    $.export("$summary", `Successfully crawled URL: ${this.url}`);
+    return response;
+  },
+};

components/crawlbase/common/constants.mjs

@@ -0,0 +1,48 @@
+const SCRAPERS = [
+  "amazon-product-details",
+  "amazon-serp",
+  "amazon-offer-listing",
+  "amazon-best-sellers",
+  "amazon-new-releases",
+  "google-serp",
+  "google-product-offers",
+  "facebook-group",
+  "facebook-page",
+  "facebook-profile",
+  "facebook-hashtag",
+  "facebook-event",
+  "instagram-reel",
+  "instagram-post",
+  "instagram-profile",
+  "instagram-reels-audio",
+  "tiktok-product",
+  "tiktok-shop",
+  "tiktok-profile",
+  "shein-product",
+  "linkedin-profile",
+  "linkedin-company",
+  "linkedin-feed",
+  "quora-question",
+  "airbnb-serp",
+  "ebay-serp",
+  "ebay-product",
+  "ebay-seller-shop",
+  "aliexpress-product",
+  "aliexpress-serp",
+  "bing-serp",
+  "immobilienscout24-property",
+  "walmart-serp",
+  "walmart-product-details",
+  "walmart-category",
+  "bestbuy-serp",
+  "bestbuy-product-details",
+  "g2-product-reviews",
+  "eventbrite-events-list",
+  "eventbrite-event-details",
+  "generic-extractor",
+  "email-extractor",
+];
+
+export default {
+  SCRAPERS,
+};

components/crawlbase/crawlbase.app.mjs

@@ -1,11 +1,24 @@
+import { axios } from "@pipedream/platform";
+
 export default {
   type: "app",
   app: "crawlbase",
   propDefinitions: {},
   methods: {
-    // this.$auth contains connected account data
-    authKeys() {
-      console.log(Object.keys(this.$auth));
+    _baseUrl() {
+      return "https://api.crawlbase.com";
+    },
+    makeRequest({
+      $ = this, path = "", params = {}, ...opts
+    }) {
+      return axios($, {
+        url: `${this._baseUrl()}${path}`,
+        params: {
+          ...params,
+          token: `${this.$auth.api_token}`,
+        },
+        ...opts,
+      });
     },
   },
-};
+};

components/crawlbase/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/crawlbase",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Pipedream Crawlbase Components",
   "main": "crawlbase.app.mjs",
   "keywords": [
@@ -11,5 +11,8 @@
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.1.0"
   }
 }