diff --git a/components/news_api/actions/search-everything/search-everything.mjs b/components/news_api/actions/search-everything/search-everything.mjs new file mode 100644 index 0000000000000..49e77517b6383 --- /dev/null +++ b/components/news_api/actions/search-everything/search-everything.mjs @@ -0,0 +1,100 @@ +import newsapi from "../../news_api.app.mjs"; +import utils from "../../common/utils.mjs"; + +export default { + key: "news_api-search-everything", + name: "Search Everything", + description: "Search through millions of articles from over 150,000 large and small news sources and blogs. [See the documentation](https://newsapi.org/docs/endpoints/everything)", + version: "0.0.1", + type: "action", + props: { + newsapi, + q: { + propDefinition: [ + newsapi, + "q", + ], + }, + searchin: { + propDefinition: [ + newsapi, + "searchin", + ], + }, + sourceIds: { + propDefinition: [ + newsapi, + "sourceIds", + ], + }, + domains: { + type: "string[]", + label: "Domains", + description: "An array of domains to restrict the search to", + optional: true, + }, + excludeDomains: { + type: "string[]", + label: "Exclude Domains", + description: "An array of domains to remove from the results", + optional: true, + }, + from: { + type: "string", + label: "From", + description: "A date and optional time for the oldest article allowed. This should be in ISO 8601 format (e.g. `2024-11-01` or `2024-11-01T17:27:47`)", + optional: true, + }, + to: { + type: "string", + label: "To", + description: "A date and optional time for the newest article allowed. This should be in ISO 8601 format (e.g. `2024-11-01` or `2024-11-01T17:27:47`)", + optional: true, + }, + language: { + propDefinition: [ + newsapi, + "language", + ], + }, + sortBy: { + propDefinition: [ + newsapi, + "sortBy", + ], + }, + maxResults: { + propDefinition: [ + newsapi, + "maxResults", + ], + }, + }, + async run({ $ }) { + const { + status, articles, + } = await this.newsapi.searchEverything({ + $, + params: { + q: this.q, + searchin: utils.joinArray(this.searchin), + sources: utils.joinArray(this.sourceIds), + domains: utils.joinArray(this.domains), + excludeDomains: utils.joinArray(this.excludeDomains), + from: this.from, + to: this.to, + language: this.language, + sortBy: this.sortBy, + pageSize: this.maxResults, + }, + }); + + if (status === "ok") { + $.export("$summary", `Successfully retrieved ${articles.length} article${articles.length === 1 + ? "" + : "s"}`); + } + + return articles; + }, +}; diff --git a/components/news_api/actions/search-top-headlines/search-top-headlines.mjs b/components/news_api/actions/search-top-headlines/search-top-headlines.mjs new file mode 100644 index 0000000000000..e520a2bd23a2d --- /dev/null +++ b/components/news_api/actions/search-top-headlines/search-top-headlines.mjs @@ -0,0 +1,73 @@ +import newsapi from "../../news_api.app.mjs"; +import utils from "../../common/utils.mjs"; +import { ConfigurationError } from "@pipedream/platform"; + +export default { + key: "news_api-search-top-headlines", + name: "Search Top Headlines", + description: "Retrieve live top and breaking headlines for a category, single source, multiple sources, or keywords. [See the documentation](https://newsapi.org/docs/endpoints/top-headlines)", + version: "0.0.1", + type: "action", + props: { + newsapi, + q: { + propDefinition: [ + newsapi, + "q", + ], + optional: true, + }, + category: { + type: "string", + label: "Category", + description: "The category you want to get headlines for. Possible options: `business` `entertainment` `general` `health` `science` `sports` `technology`. Note: you can't mix this param with the `sources` param.", + optional: true, + }, + sourceIds: { + propDefinition: [ + newsapi, + "sourceIds", + ], + }, + maxResults: { + propDefinition: [ + newsapi, + "maxResults", + ], + }, + }, + async run({ $ }) { + if (this.category && this.sourceIds) { + throw new ConfigurationError("Please specify only one of `Category` or `SourceIds`"); + } + + const params = { + q: this.q, + category: this.category, + sources: utils.joinArray(this.sourceIds), + pageSize: this.maxResults, + }; + + // The only available country is "us", but it can't be specified along with category or sources. + // At least one of q, category, sources, or country must be entered, so adding in country if + // none of the others are specified. + if (!this.q && !this.category && !this.sourceIds) { + params.country = "us"; + } + + const { + status, articles, + } = await this.newsapi.searchTopHeadlines({ + $, + params, + }); + + if (status === "ok") { + $.export("$summary", `Successfully retrieved ${articles.length} article${articles.length === 1 + ? "" + : "s"}`); + } + + return articles; + }, +}; diff --git a/components/news_api/common/constants.mjs b/components/news_api/common/constants.mjs new file mode 100644 index 0000000000000..527f5bb44c4f4 --- /dev/null +++ b/components/news_api/common/constants.mjs @@ -0,0 +1,76 @@ +const SEARCH_IN_OPTIONS = [ + "title", + "description", + "content", +]; + +const SORT_OPTIONS = [ + "relevancy", + "popularity", + "publishedAt", +]; + +const LANGUAGES = [ + { + value: "ar", + label: "Arabic", + }, + { + value: "de", + label: "German", + }, + { + value: "en", + label: "English", + }, + { + value: "es", + label: "Spanish", + }, + { + value: "fr", + label: "French", + }, + { + value: "he", + label: "Hebrew", + }, + { + value: "it", + label: "Italian", + }, + { + value: "nl", + label: "Dutch", + }, + { + value: "no", + label: "Norwegian", + }, + { + value: "pt", + label: "Portuguese", + }, + { + value: "ru", + label: "Russian", + }, + { + value: "sv", + label: "Swedish", + }, + { + value: "ud", + label: "Urdu", + }, + { + value: "zh", + label: "Chinese", + }, +]; + +export default { + SEARCH_IN_OPTIONS, + SORT_OPTIONS, + LANGUAGES, +}; diff --git a/components/news_api/common/utils.mjs b/components/news_api/common/utils.mjs new file mode 100644 index 0000000000000..1c5c52546a575 --- /dev/null +++ b/components/news_api/common/utils.mjs @@ -0,0 +1,10 @@ +function joinArray(arr) { + if (!arr) { + return undefined; + } + return arr.join(); +} + +export default { + joinArray, +}; diff --git a/components/news_api/news_api.app.mjs b/components/news_api/news_api.app.mjs index d97a6bd8edae6..138238ac9dd18 100644 --- a/components/news_api/news_api.app.mjs +++ b/components/news_api/news_api.app.mjs @@ -1,11 +1,92 @@ +import { axios } from "@pipedream/platform"; +import constants from "./common/constants.mjs"; + export default { type: "app", app: "news_api", - propDefinitions: {}, + propDefinitions: { + sourceIds: { + type: "string[]", + label: "Source IDs", + description: "An array of source identifiers (maximum 20) for the news sources or blogs you want headlines from", + optional: true, + async options() { + const { sources } = await this.listSources(); + return sources.map(({ + id: value, name: label, + }) => ({ + value, + label, + })); + }, + }, + searchin: { + type: "string[]", + label: "Search In", + description: "The fields to restrict your q search to. Default: all fields are searched", + options: constants.SEARCH_IN_OPTIONS, + optional: true, + }, + q: { + type: "string", + label: "Query", + description: "Keywords or phrases to search for", + }, + language: { + type: "string", + label: "Language", + description: "The 2-letter ISO-639-1 code of the language you want to get headlines for", + options: constants.LANGUAGES, + optional: true, + }, + sortBy: { + type: "string", + label: "Sort By", + description: "The order to sort the articles in. Default: `publishedAt`", + options: constants.SORT_OPTIONS, + optional: true, + }, + maxResults: { + type: "integer", + label: "Max Results", + description: "The maximum number of results to return. Must be between 1 and 100.", + optional: true, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://newsapi.org/v2"; + }, + _makeRequest({ + $ = this, + path, + ...opts + }) { + return axios($, { + url: `${this._baseUrl()}${path}`, + headers: { + "X-Api-Key": `${this.$auth.api_key}`, + }, + ...opts, + }); + }, + listSources(opts = {}) { + return this._makeRequest({ + path: "/top-headlines/sources", + ...opts, + }); + }, + searchEverything(opts = {}) { + return this._makeRequest({ + path: "/everything", + ...opts, + }); + }, + searchTopHeadlines(opts = {}) { + return this._makeRequest({ + path: "/top-headlines", + ...opts, + }); }, }, -}; \ No newline at end of file +}; diff --git a/components/news_api/package.json b/components/news_api/package.json index ff89334e461f4..542a42c5868a2 100644 --- a/components/news_api/package.json +++ b/components/news_api/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/news_api", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream News API Components", "main": "news_api.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" } -} \ No newline at end of file +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1fcea59661dea..ad109d7018fe7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6563,7 +6563,10 @@ importers: moment: 2.30.1 components/news_api: - specifiers: {} + specifiers: + '@pipedream/platform': ^3.0.3 + dependencies: + '@pipedream/platform': 3.0.3 components/newscatcher: specifiers: {} @@ -13228,6 +13231,55 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sso-oidc/3.600.0_tdq3komn4zwyd65w7klbptsu34: + resolution: {integrity: sha512-7+I8RWURGfzvChyNQSyj5/tKrqRbzRl7H+BnTOf/4Vsw1nFOi5ROhlhD4X/Y0QCTacxnaoNcIrqnY7uGGvVRzw==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sts': 3.600.0 + '@aws-sdk/core': 3.598.0 + '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 + '@aws-sdk/middleware-host-header': 3.598.0 + '@aws-sdk/middleware-logger': 3.598.0 + '@aws-sdk/middleware-recursion-detection': 3.598.0 + '@aws-sdk/middleware-user-agent': 3.598.0 + '@aws-sdk/region-config-resolver': 3.598.0 + '@aws-sdk/types': 3.598.0 + '@aws-sdk/util-endpoints': 3.598.0 + '@aws-sdk/util-user-agent-browser': 3.598.0 + '@aws-sdk/util-user-agent-node': 3.598.0 + '@smithy/config-resolver': 3.0.3 + '@smithy/core': 2.2.3 + '@smithy/fetch-http-handler': 3.2.1 + '@smithy/hash-node': 3.0.2 + '@smithy/invalid-dependency': 3.0.2 + '@smithy/middleware-content-length': 3.0.2 + '@smithy/middleware-endpoint': 3.0.4 + '@smithy/middleware-retry': 3.0.6 + '@smithy/middleware-serde': 3.0.3 + '@smithy/middleware-stack': 3.0.3 + '@smithy/node-config-provider': 3.1.3 + '@smithy/node-http-handler': 3.1.2 + '@smithy/protocol-http': 4.0.3 + '@smithy/smithy-client': 3.1.6 + '@smithy/types': 3.3.0 + '@smithy/url-parser': 3.0.3 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.6 + '@smithy/util-defaults-mode-node': 3.0.6 + '@smithy/util-endpoints': 2.0.3 + '@smithy/util-middleware': 3.0.3 + '@smithy/util-retry': 3.0.2 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sts' + - aws-crt + dev: false + /@aws-sdk/client-sso/3.423.0: resolution: {integrity: sha512-znIufHkwhCIePgaYciIs3x/+BpzR57CZzbCKHR9+oOvGyufEPPpUT5bFLvbwTgfiVkTjuk6sG/ES3U5Bc+xtrA==} engines: {node: '>=14.0.0'} @@ -13463,7 +13515,7 @@ packages: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 + '@aws-sdk/client-sso-oidc': 3.600.0_tdq3komn4zwyd65w7klbptsu34 '@aws-sdk/core': 3.598.0 '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 '@aws-sdk/middleware-host-header': 3.598.0 @@ -13505,55 +13557,6 @@ packages: - aws-crt dev: false - /@aws-sdk/client-sts/3.600.0_dseaa2p5u2yk67qiepewcq3hkq: - resolution: {integrity: sha512-KQG97B7LvTtTiGmjlrG1LRAY8wUvCQzrmZVV5bjrJ/1oXAU7DITYwVbSJeX9NWg6hDuSk0VE3MFwIXS2SvfLIA==} - engines: {node: '>=16.0.0'} - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 - '@aws-sdk/core': 3.598.0 - '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 - '@aws-sdk/middleware-host-header': 3.598.0 - '@aws-sdk/middleware-logger': 3.598.0 - '@aws-sdk/middleware-recursion-detection': 3.598.0 - '@aws-sdk/middleware-user-agent': 3.598.0 - '@aws-sdk/region-config-resolver': 3.598.0 - '@aws-sdk/types': 3.598.0 - '@aws-sdk/util-endpoints': 3.598.0 - '@aws-sdk/util-user-agent-browser': 3.598.0 - '@aws-sdk/util-user-agent-node': 3.598.0 - '@smithy/config-resolver': 3.0.3 - '@smithy/core': 2.2.3 - '@smithy/fetch-http-handler': 3.2.1 - '@smithy/hash-node': 3.0.2 - '@smithy/invalid-dependency': 3.0.2 - '@smithy/middleware-content-length': 3.0.2 - '@smithy/middleware-endpoint': 3.0.4 - '@smithy/middleware-retry': 3.0.6 - '@smithy/middleware-serde': 3.0.3 - '@smithy/middleware-stack': 3.0.3 - '@smithy/node-config-provider': 3.1.3 - '@smithy/node-http-handler': 3.1.2 - '@smithy/protocol-http': 4.0.3 - '@smithy/smithy-client': 3.1.6 - '@smithy/types': 3.3.0 - '@smithy/url-parser': 3.0.3 - '@smithy/util-base64': 3.0.0 - '@smithy/util-body-length-browser': 3.0.0 - '@smithy/util-body-length-node': 3.0.0 - '@smithy/util-defaults-mode-browser': 3.0.6 - '@smithy/util-defaults-mode-node': 3.0.6 - '@smithy/util-endpoints': 2.0.3 - '@smithy/util-middleware': 3.0.3 - '@smithy/util-retry': 3.0.2 - '@smithy/util-utf8': 3.0.0 - tslib: 2.6.3 - transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - - aws-crt - dev: false - /@aws-sdk/core/3.556.0: resolution: {integrity: sha512-vJaSaHw2kPQlo11j/Rzuz0gk1tEaKdz+2ser0f0qZ5vwFlANjt08m/frU17ctnVKC1s58bxpctO/1P894fHLrA==} engines: {node: '>=14.0.0'} @@ -17846,7 +17849,7 @@ packages: '@aws-sdk/client-sns': 3.423.0 '@aws-sdk/client-sqs': 3.423.0 '@aws-sdk/client-ssm': 3.423.0 - '@aws-sdk/client-sts': 3.600.0_dseaa2p5u2yk67qiepewcq3hkq + '@aws-sdk/client-sts': 3.600.0 '@aws-sdk/s3-request-presigner': 3.609.0 '@pipedream/helper_functions': 0.3.12 '@pipedream/platform': 1.6.6