From 4a93721f97e404a227754fbbf91b7eaa71e57e09 Mon Sep 17 00:00:00 2001 From: Michelle Bergeron Date: Thu, 5 Jun 2025 14:18:20 -0400 Subject: [PATCH 1/4] new search action --- .../firecrawl/actions/search/search.mjs | 111 ++++++++++++++++++ components/firecrawl/firecrawl.app.mjs | 7 ++ components/firecrawl/package.json | 5 +- 3 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 components/firecrawl/actions/search/search.mjs diff --git a/components/firecrawl/actions/search/search.mjs b/components/firecrawl/actions/search/search.mjs new file mode 100644 index 0000000000000..3349ce7fb8c4d --- /dev/null +++ b/components/firecrawl/actions/search/search.mjs @@ -0,0 +1,111 @@ +import firecrawl from "../../firecrawl.app.mjs"; + +export default { + key: "firecrawl-search", + name: "Search", + description: "Search the web and get full content from results. [See the documentation](https://docs.firecrawl.dev/features/search)", + version: "0.0.1", + type: "action", + props: { + firecrawl, + query: { + type: "string", + label: "Query", + description: "The query to search for", + }, + scrapeOptionFormats: { + type: "string[]", + label: "Scrape Option Formats", + description: "Search with content scraping. You can specify multiple output formats.", + options: [ + { + label: "Clean, formatted markdown content", + value: "markdown", + }, + { + label: "Processed HTML content", + value: "html", + }, + { + label: "Unmodified HTML content", + value: "rawHtml", + }, + { + label: "List of links found on the page", + value: "links", + }, + { + label: "Screenshot of the page", + value: "screenshot", + }, + { + label: "Full-page screenshot", + value: "screenshot@fullPage", + }, + { + label: "Structured data extraction", + value: "extract", + }, + ], + optional: true, + }, + timeBasedSearch: { + type: "string", + label: "Time Based Search", + description: "Filter the results by time", + options: [ + { + label: "Past hour", + value: "qdr:h", + }, + { + label: "Past 24 hours", + value: "qdr:d", + }, + { + label: "Past week", + value: "qdr:w", + }, + { + label: "Past month", + value: "qdr:m", + }, + { + label: "Past year", + value: "qdr:y", + }, + ], + optional: true, + }, + customTimeout: { + type: "integer", + label: "Custom Timeout", + description: "Set a custom timeout for search operations. E.g. `30000`", + optional: true, + }, + maxResults: { + type: "integer", + label: "Max Results", + description: "The maximum number of results to return", + default: 10, + optional: true, + }, + }, + async run({ $ }) { + const results = await this.firecrawl.search({ + data: { + query: this.query, + limit: this.maxResults, + scrapeOptions: this.scrapeOptionFormats + ? { + formats: this.scrapeOptionFormats, + } + : undefined, + tbs: this.timeBasedSearch, + timeout: this.customTimeout, + }, + }); + $.export("$summary", `Successfully searched for ${this.query}`); + return results; + }, +}; diff --git a/components/firecrawl/firecrawl.app.mjs b/components/firecrawl/firecrawl.app.mjs index 1d075c3863b80..fd6e79d1588c7 100644 --- a/components/firecrawl/firecrawl.app.mjs +++ b/components/firecrawl/firecrawl.app.mjs @@ -77,5 +77,12 @@ export default { ...opts, }); }, + search(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/search", + ...opts, + }); + }, }, }; diff --git a/components/firecrawl/package.json b/components/firecrawl/package.json index c96fe632f2267..4ec3137c71db3 100644 --- a/components/firecrawl/package.json +++ b/components/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/firecrawl", - "version": "1.1.0", + "version": "1.2.0", "description": "Pipedream FireCrawl Components", "main": "firecrawl.app.mjs", "keywords": [ @@ -13,7 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.0" + "@pipedream/platform": "^3.1.0" } } - From 4813377cfeaf646f74d24754a2789c796c6ca1a7 Mon Sep 17 00:00:00 2001 From: Michelle Bergeron Date: Thu, 5 Jun 2025 14:19:11 -0400 Subject: [PATCH 2/4] pnpm-lock.yaml --- pnpm-lock.yaml | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 81a357245fbf5..33fcc0187191c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4631,8 +4631,8 @@ importers: components/firecrawl: dependencies: '@pipedream/platform': - specifier: ^3.0.0 - version: 3.0.3 + specifier: ^3.1.0 + version: 3.1.0 components/firefish: dependencies: @@ -18733,6 +18733,9 @@ packages: '@pipedream/platform@3.0.3': resolution: {integrity: sha512-7elalas41lnT8i6EAFkqB7fT/+hkLGEQ1njS6A7CVguTrEswaIYk/seKmkfkRY7+O6qncgnXswYIKCBML9Co7w==} + '@pipedream/platform@3.1.0': + resolution: {integrity: sha512-ELmV3fpd9PIUm74/RPaqmA7hJaY4ryIn6PCNT1NP/gnqMrkRNOAr+DTV4xPaJgPiRm74WC5HvrkC8YdRveXzZA==} + '@pipedream/postgresql@2.2.3': resolution: {integrity: sha512-Co9r4UKvvimEPo9T4v+EfVn/Sqqw8+X7PtvsAF7tydq7CHpkmOT9eYAiD0Kuybg5WbuWuqDaXHHmhVtsAxgpBw==} @@ -35456,7 +35459,7 @@ snapshots: '@pipedream/linear_app@0.7.1': dependencies: '@linear/sdk': 13.0.0 - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 transitivePeerDependencies: - debug - encoding @@ -35471,7 +35474,7 @@ snapshots: '@pipedream/monday@0.7.0': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 form-data: 4.0.2 lodash.flatmap: 4.5.0 lodash.map: 4.6.0 @@ -35484,7 +35487,7 @@ snapshots: '@pipedream/notion@0.6.2': dependencies: '@notionhq/client': 2.3.0 - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 '@tryfabric/martian': 1.2.4 lodash-es: 4.17.21 notion-to-md: 3.1.8 @@ -35590,6 +35593,17 @@ snapshots: transitivePeerDependencies: - debug + '@pipedream/platform@3.1.0': + dependencies: + axios: 1.9.0 + fp-ts: 2.16.10 + io-ts: 2.2.22(fp-ts@2.16.10) + mime-types: 3.0.1 + querystring: 0.2.1 + uuid: 11.1.0 + transitivePeerDependencies: + - debug + '@pipedream/postgresql@2.2.3': dependencies: '@pipedream/platform': 2.0.0 @@ -35601,13 +35615,13 @@ snapshots: '@pipedream/quickbooks@0.5.1': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 transitivePeerDependencies: - debug '@pipedream/ramp@0.1.2': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 uuid: 10.0.0 transitivePeerDependencies: - debug @@ -35644,14 +35658,14 @@ snapshots: '@pipedream/sftp@0.4.1': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 ssh2-sftp-client: 11.0.0 transitivePeerDependencies: - debug '@pipedream/shopify@0.7.0': dependencies: - '@pipedream/platform': 3.0.3 + '@pipedream/platform': 3.1.0 async-retry: 1.3.3 bottleneck: 2.19.5 form-data: 4.0.2 From e0793e6e1ab1c63d0a6ed236a063dd4c78a7fdf0 Mon Sep 17 00:00:00 2001 From: Michelle Bergeron Date: Thu, 5 Jun 2025 14:45:42 -0400 Subject: [PATCH 3/4] versions --- components/firecrawl/actions/crawl-url/crawl-url.mjs | 2 +- components/firecrawl/actions/extract-data/extract-data.mjs | 2 +- .../firecrawl/actions/get-crawl-status/get-crawl-status.mjs | 2 +- .../firecrawl/actions/get-extract-status/get-extract-status.mjs | 2 +- components/firecrawl/actions/scrape-page/scrape-page.mjs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/firecrawl/actions/crawl-url/crawl-url.mjs b/components/firecrawl/actions/crawl-url/crawl-url.mjs index d3141cbb8f1fc..7366fe803d2c2 100644 --- a/components/firecrawl/actions/crawl-url/crawl-url.mjs +++ b/components/firecrawl/actions/crawl-url/crawl-url.mjs @@ -5,7 +5,7 @@ export default { key: "firecrawl-crawl-url", name: "Crawl URL", description: "Crawls a given URL and returns the contents of sub-pages. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/crawl-post)", - version: "1.0.2", + version: "1.0.3", type: "action", props: { firecrawl, diff --git a/components/firecrawl/actions/extract-data/extract-data.mjs b/components/firecrawl/actions/extract-data/extract-data.mjs index 123b533b77fa8..e6e143cc68c0c 100644 --- a/components/firecrawl/actions/extract-data/extract-data.mjs +++ b/components/firecrawl/actions/extract-data/extract-data.mjs @@ -6,7 +6,7 @@ export default { key: "firecrawl-extract-data", name: "Extract Data", description: "Extract structured data from one or multiple URLs. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/extract)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { firecrawl, diff --git a/components/firecrawl/actions/get-crawl-status/get-crawl-status.mjs b/components/firecrawl/actions/get-crawl-status/get-crawl-status.mjs index a7eb49c728c14..3539815b7eae7 100644 --- a/components/firecrawl/actions/get-crawl-status/get-crawl-status.mjs +++ b/components/firecrawl/actions/get-crawl-status/get-crawl-status.mjs @@ -4,7 +4,7 @@ export default { key: "firecrawl-get-crawl-status", name: "Get Crawl Data", description: "Obtains the status and data from a previous crawl operation. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/crawl-get)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { firecrawl, diff --git a/components/firecrawl/actions/get-extract-status/get-extract-status.mjs b/components/firecrawl/actions/get-extract-status/get-extract-status.mjs index 4f93d2ee7db18..78a0c704ee63c 100644 --- a/components/firecrawl/actions/get-extract-status/get-extract-status.mjs +++ b/components/firecrawl/actions/get-extract-status/get-extract-status.mjs @@ -4,7 +4,7 @@ export default { key: "firecrawl-get-extract-status", name: "Get Extract Data", description: "Obtains the status and data from a previous extract operation. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/extract-get)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { firecrawl, diff --git a/components/firecrawl/actions/scrape-page/scrape-page.mjs b/components/firecrawl/actions/scrape-page/scrape-page.mjs index 9f01378db4072..bc5049e192ebd 100644 --- a/components/firecrawl/actions/scrape-page/scrape-page.mjs +++ b/components/firecrawl/actions/scrape-page/scrape-page.mjs @@ -7,7 +7,7 @@ export default { name: "Scrape Page", description: "Scrapes a URL and returns content from that page. [See the documentation](https://docs.firecrawl.dev/api-reference/endpoint/scrape)", - version: "1.0.1", + version: "1.0.2", type: "action", props: { firecrawl, From ac7fc9b6c5df2e43d2d3ee18c2e4c57b6edda57f Mon Sep 17 00:00:00 2001 From: Michelle Bergeron Date: Thu, 5 Jun 2025 15:02:00 -0400 Subject: [PATCH 4/4] update --- components/firecrawl/actions/search/search.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/components/firecrawl/actions/search/search.mjs b/components/firecrawl/actions/search/search.mjs index 3349ce7fb8c4d..77f40f14c4147 100644 --- a/components/firecrawl/actions/search/search.mjs +++ b/components/firecrawl/actions/search/search.mjs @@ -93,6 +93,7 @@ export default { }, async run({ $ }) { const results = await this.firecrawl.search({ + $, data: { query: this.query, limit: this.maxResults,