Firecrawl - update Crawl URL to v2 (#18280)

* update crawl-url to v2

* pnpm-lock.yaml

* versions

Author: michelle0927

components/firecrawl/actions/crawl-url/crawl-url.mjs

@@ -5,7 +5,7 @@ export default {
   key: "firecrawl-crawl-url",
   name: "Crawl URL",
   description: "Crawls a given URL and returns the contents of sub-pages. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/crawl-post)",
-  version: "1.0.3",
+  version: "1.1.0",
   type: "action",
   props: {
     firecrawl,
@@ -15,6 +15,12 @@ export default {
         "url",
       ],
     },
+    prompt: {
+      type: "string",
+      label: "Prompt",
+      description: "A prompt to use to generate the crawler options (all the parameters below) from natural language. Explicitly set parameters will override the generated equivalents.",
+      optional: true,
+    },
     excludePaths: {
       type: "string[]",
       label: "Exclude Paths",
@@ -27,16 +33,20 @@ export default {
       description: "Similar to `Exclude Paths`, but if set, only the paths matching the specified patterns will be included",
       optional: true,
     },
-    maxDepth: {
+    maxDiscoveryDepth: {
       type: "integer",
-      label: "Max Depth",
-      description: "Maximum depth to crawl relative to the entered URL",
+      label: "Max Discovery Depth",
+      description: "Maximum depth to crawl based on discovery order. The root site and sitemapped pages has a discovery depth of 0. For example, if you set it to 1, and you set sitemap: 'skip', you will only crawl the entered URL and all URLs that are linked on that page.",
       optional: true,
     },
-    ignoreSitemap: {
-      type: "boolean",
-      label: "Ignore Sitemap",
-      description: "Ignore the website sitemap when crawling",
+    sitemap: {
+      type: "string",
+      label: "Sitemap",
+      description: "Sitemap mode when crawling. If you set it to 'skip', the crawler will ignore the website sitemap and only crawl the entered URL and discover pages from there onwards.",
+      options: [
+        "skip",
+        "include",
+      ],
       optional: true,
     },
     ignoreQueryParameters: {
@@ -51,10 +61,10 @@ export default {
       description: "Maximum number of pages to crawl",
       optional: true,
     },
-    allowBackwardLinks: {
+    crawlEntireDomain: {
       type: "boolean",
-      label: "Allow Backward Links",
-      description: "Enables the crawler to navigate from a specific URL to previously linked pages",
+      label: "Crawl Entire Domain",
+      description: "Allows the crawler to follow internal links to sibling or parent URLs, not just child paths.",
       optional: true,
     },
     allowExternalLinks: {

components/firecrawl/actions/extract-data/extract-data.mjs

@@ -6,7 +6,7 @@ export default {
   key: "firecrawl-extract-data",
   name: "Extract Data",
   description: "Extract structured data from one or multiple URLs. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/extract)",
-  version: "0.0.2",
+  version: "0.0.3",
   type: "action",
   props: {
     firecrawl,

components/firecrawl/actions/get-crawl-status/get-crawl-status.mjs

@@ -4,7 +4,7 @@ export default {
   key: "firecrawl-get-crawl-status",
   name: "Get Crawl Data",
   description: "Obtains the status and data from a previous crawl operation. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/crawl-get)",
-  version: "0.0.4",
+  version: "0.0.5",
   type: "action",
   props: {
     firecrawl,

components/firecrawl/actions/get-extract-status/get-extract-status.mjs

@@ -4,7 +4,7 @@ export default {
   key: "firecrawl-get-extract-status",
   name: "Get Extract Data",
   description: "Obtains the status and data from a previous extract operation. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/extract-get)",
-  version: "0.0.2",
+  version: "0.0.3",
   type: "action",
   props: {
     firecrawl,

components/firecrawl/actions/map-url/map-url.mjs

@@ -4,7 +4,7 @@ export default {
   key: "firecrawl-map-url",
   name: "Map URL",
   description: "Maps a given URL using Firecrawl's Map endpoint. Optionally, you can provide a search term to filter the mapping. [See the documentation](https://docs.firecrawl.dev/features/map)",
-  version: "0.0.1",
+  version: "0.0.2",
   type: "action",
   props: {
     firecrawl,

components/firecrawl/actions/scrape-page/scrape-page.mjs

@@ -7,7 +7,7 @@ export default {
   name: "Scrape Page",
   description:
     "Scrapes a URL and returns content from that page. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/scrape)",
-  version: "1.0.2",
+  version: "1.0.3",
   type: "action",
   props: {
     firecrawl,

components/firecrawl/actions/search/search.mjs

@@ -4,7 +4,7 @@ export default {
   key: "firecrawl-search",
   name: "Search",
   description: "Search the web and get full content from results. [See the documentation](https://docs.firecrawl.dev/features/search)",
-  version: "0.0.1",
+  version: "0.0.2",
   type: "action",
   props: {
     firecrawl,

components/firecrawl/firecrawl.app.mjs

@@ -24,18 +24,18 @@ export default {
   },
   methods: {
     _baseUrl() {
-      return "https://api.firecrawl.dev/v1";
+      return "https://api.firecrawl.dev";
     },
     _headers() {
       return {
         Authorization: `Bearer ${this.$auth.api_key}`,
       };
     },
     _makeRequest({
-      $ = this, path, ...opts
+      $ = this, path, version = "v1", ...opts
     }) {
       return axios($, {
-        url: this._baseUrl() + path,
+        url: `${this._baseUrl()}/${version}${path}`,
         headers: this._headers(),
         ...opts,
       });
@@ -44,6 +44,7 @@ export default {
       return this._makeRequest({
         method: "POST",
         path: "/crawl",
+        version: "v2",
         ...opts,
       });
     },

components/firecrawl/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/firecrawl",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "Pipedream FireCrawl Components",
   "main": "firecrawl.app.mjs",
   "keywords": [