diff --git a/components/google_search_console/README.md b/components/google_search_console/README.md index cada189f1f1ec..a36ac08199d6a 100644 --- a/components/google_search_console/README.md +++ b/components/google_search_console/README.md @@ -9,3 +9,10 @@ The Google Search Console API opens a treasure trove of data and insights about - **Sync Search Results with Google Sheets**: Create a workflow that periodically pulls data from the Google Search Console API and adds it to a Google Sheet. This is useful for maintaining an evolving dataset for deeper analysis, historical reference, or sharing insights across teams without giving direct access to the Search Console. - **Automatic Sitemap Submission**: Set up a Pipedream workflow that triggers whenever a new sitemap is generated in your content management system (CMS). The workflow can then automatically submit the sitemap to Google Search Console via API, ensuring Google has the latest structure of your site for crawling and indexing. + +## Available Actions + +[Retrieve Site Performance Data](./actions/retrieve-site-performance-data/README.md) +[Submit URL for Indexing](./actions/submit-url-for-indexing/README.md) + + diff --git a/components/google_search_console/actions/retrieve-site-performance-data/README.md b/components/google_search_console/actions/retrieve-site-performance-data/README.md new file mode 100644 index 0000000000000..a6ed4ab12ceb8 --- /dev/null +++ b/components/google_search_console/actions/retrieve-site-performance-data/README.md @@ -0,0 +1,108 @@ +# Google Search Console – Site Performance (Analytics) Action + +This action queries **search performance data** for a verified website using the [Google Search Console API](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query). It allows you to extract insights like: + +- Top search queries +- Click-through rates +- Page impressions +- Device types, countries, and more + +--- + +## Use Cases + +- Automate SEO reporting +- Analyze organic search trends +- Filter and break down traffic by dimensions (e.g. query, device, country) + +--- + +## Internals + +- Supports all relevant props from the [Search Analytics Query API](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query) +- Trims and validates all input props +- Accepts optional `dimensionFilterGroups` either as an object or JSON string +- Automatically builds the POST request body using `propsMeta` metadata +- Returns the raw Search Console API response for full access to metrics +- Accumulates and displays warnings for non-blocking input issues + +--- + +## Auth + +Requires OAuth 2.0 with the following scope: `https://www.googleapis.com/auth/webmasters.readonly` + + +## Endpoint +`https://searchconsole.googleapis.com/webmasters/v3/sites/{siteUrl}/searchAnalytics/query` + + +## 📦 Example Request Payload + + + +```json +{ + + "siteUrl": "https://falc1.com/page", + + "startDate": "2025-12-22", + + "endDate": "2025-12-31", + + "dimensions": ["query", "page", "country", "device"], + + "searchType": "web", + + "rowLimit": 10, + + "startRow": 0, + + "aggregationType": "auto", + + "dataState": "final", + + "dimensionFilterGroups": [ + { + + "groupType": "and", + + "filters": [ + { + + "dimension": "query", + + "operator": "contains", + + "expression": "example" + }, + { + "dimension": "country", + + "operator": "equals", + + "expression": "USA" + } + ] + } + ] +} +``` + +### Field Descriptions + + **siteUrl**: The site you want to query data for. Must be verified in your Google Search Console account. + **startDate**: The start date of the reporting period (inclusive), in YYYY-MM-DD format. + **endDate**: The end date of the reporting period (inclusive), in YYYY-MM-DD format. + **dimensions**: The dimensions you want to break down the data by. Valid values: "query", "page", "country", "device", "searchAppearance", "date". Order matters — it affects how rows are grouped in the response. + **searchType**: The type of search data to include. Valid values: "web", "image", "video", "news", "googleNews", "discover" + **rowLimit**: Maximum number of rows to return (1–25,000) + **startRow**: Optional: Skips the first N rows — used for pagination. + **aggregationType**: Optional: How to group data. "auto" = Google's default grouping. "byPage" = Group by page (useful for getting per-page breakdowns). + **dataState**: Optional: Data freshness filter. "final" = Only finalized data (more accurate). "all" = Includes fresh but possibly incomplete data. + **dimensionFilterGroups**: Optional filter group(s) to restrict which rows are returned. Each group applies logical AND/OR across its filters. + **groupType**: Logical grouping operator for the filters inside this group. "and" = all filters must match, "or" = any filter can match + **filters**: List of individual filters to apply within the group + **dimension**: Which dimension to filter by (must match a dimension in your request) + **operator**: Filter operator — e.g., "equals", "contains", "notEquals", etc. + **expression**: Value to match against \ No newline at end of file diff --git a/components/google_search_console/actions/retrieve-site-performance-data/retrieve-site-performance-data.mjs b/components/google_search_console/actions/retrieve-site-performance-data/retrieve-site-performance-data.mjs new file mode 100644 index 0000000000000..36b3f65858c2d --- /dev/null +++ b/components/google_search_console/actions/retrieve-site-performance-data/retrieve-site-performance-data.mjs @@ -0,0 +1,200 @@ +/** + * Fetches performance data (clicks, impressions, etc.) for a verified site + * via the Google Search Console Search Analytics API. + * + * Full usage docs in README.md + */ + +import { axios } from "@pipedream/platform"; +import gsConsole from "../../google_search_console.app.mjs"; +import { + removeCustomPropFields, trimIfString, +} from "../../common/utils.mjs"; + +/* + Define prop metadata separately, including custom fields used for extended validation + and runtime behavior. + + These extended fields (like `extendedType`, `postBody`, etc.) are not part of the standard + Pipedream prop schema. + + A helper function (`removeCustomPropFields`) will later strip these non-standard fields, + returning only valid Pipedream props for use in the UI. + + Keeping the full metadata in closure allows access to helpful context (e.g. validation rules) + during runtime. +*/ +const propsMeta = { + + siteUrl: { + type: "string", + extendedType: "url", + label: "Verified Site URL", + description: "Including https:// is strongly advised", + }, + startDate: { + type: "string", + extendedType: "YYYY-MM-DD", + label: "Start Date (YYYY-MM-DD)", + postBody: true, + }, + endDate: { + type: "string", + extendedType: "YYYY-MM-DD", + label: "End Date (YYYY-MM-DD)", + postBody: true, + }, + dimensions: { + type: "string[]", + label: "Dimensions", + optional: true, + description: "e.g. ['query', 'page', 'country', 'device']", + postBody: true, + }, + searchType: { + type: "string", + label: "Search Type", + optional: true, + options: [ + "web", + "image", + "video", + "news", + "googleNews", + "discover", + ], + default: "web", + postBody: true, + }, + aggregationType: { + type: "string", + label: "Aggregation Type", + optional: true, + options: [ + "auto", + "byPage", + ], + postBody: true, + }, + rowLimit: { + type: "integer", + label: "Max rows to return", + default: 10, + postBody: true, + }, + startRow: { + type: "integer", + label: "Start row (for pagination)", + optional: true, + postBody: true, + }, + dimensionFilterGroups: { + type: "object", + label: "Dimension Filters", + optional: true, + description: "Follow Search Console API structure for filters", + postBody: true, + }, + dataState: { + type: "string", + label: "Data State", + optional: true, + options: [ + "all", + "final", + ], + default: "final", + postBody: true, + }, +}; + +export default { + name: "Retrieve Site Performance Data", + description: "Fetches search analytics from Google Search Console for a verified site.", + key: "google_search_console-retrieve-site-performance-data", + version: "0.0.1", + type: "action", + props: { + gsConsole, + // Remove non-standard fields and expose only valid props to Pipedream UI + ...removeCustomPropFields(propsMeta), + }, + + //=================== RUN ============================== + //====================================================== + + async run({ $ }) { + + /* + `dimensionFilterGroups` is expected to be an object. + If a JSON string is passed instead (e.g. from UI input), attempt to parse it. + - Returns parsed object if successful + - Returns original input if not a string + - Throws a descriptive error if JSON parsing fails + */ + this.dimensionFilterGroups = this.gsConsole.parseIfJsonString(this.dimensionFilterGroups); + + // Prepare the POST request payload + const body = {}; + + // Accumulator for non-blocking input warnings + const warnings = []; + + /* + This loop: + - Trims and validates all defined props + - Skips empty optional fields + - Accumulates non-blocking warnings + - Adds valid props to the POST request payload (`body`) if marked with `postBody: true` + */ + for (let propName in propsMeta) { + + // Just for convenience. + const meta = propsMeta[propName]; + + // Trim the value if it's a string + this[propName] = trimIfString(this[propName]); + + // Skip if the prop is optional and empty (null, undefined, or blank string) + if (meta.optional === true && ((this[propName] ?? "") === "")) continue; + + // Validate input (may throw or return warning messages) + const validationResult = this.gsConsole.validateUserInput(meta, this[propName]); + + // Push the warnings into warnings accumulator if any. + if (validationResult.warnings) warnings.push(...validationResult.warnings); + + // Include prop in the request body if marked as postBody + if (meta.postBody === true) body[propName] = this[propName]; + }; + + // Already trimmed earlier + const url = this.siteUrl; + + // Response of the POST request. + let response; + + try { + response = await axios($, { + method: "POST", + url: `https://searchconsole.googleapis.com/webmasters/v3/sites/${encodeURIComponent(url)}/searchAnalytics/query`, + headers: { + "Authorization": `Bearer ${this.gsConsole.$auth.oauth_access_token}`, + "Content-Type": "application/json", + }, + data: body, + }); + + } catch (error) { + // Identify if the error was thrown by internal validation or by the API call + const thrower = this.gsConsole.checkWhoThrewError(error); + + throw new Error(`Failed to fetch data ( ${thrower.whoThrew} error ) : ${error.message}. ` + warnings.join("\n- ")); + }; + + // Output summary and any warnings for the user + $.export("$summary", ` Fetched ${response.rows?.length || 0} rows of data. ` + warnings.join("\n- ")); + return response; + }, +}; + diff --git a/components/google_search_console/actions/submit-url-for-indexing/README.md b/components/google_search_console/actions/submit-url-for-indexing/README.md new file mode 100644 index 0000000000000..303cb9e052e69 --- /dev/null +++ b/components/google_search_console/actions/submit-url-for-indexing/README.md @@ -0,0 +1,31 @@ +# 🚀 Google Indexing API – URL Submission Action + +This action submits a specific URL to the **Google Indexing API** to notify Google that the content has been **updated** and should be re-crawled. + +--- + +## ✅ Use Case + +Use this action when: +- You’ve updated content on a page +- You want to request Google to reindex that URL as soon as possible + +--- + +## 🧠 Internals + +- Validates and trims the input URL +- Uses the `URL_UPDATED` type to inform Google of a content change +- Returns the raw API response +- Displays a user-friendly summary +- Accumulates and returns non-blocking warnings (e.g. unusual characters in the URL) + +--- + +## 🔐 Auth + ++Requires OAuth 2.0 with [indexing](https://www.googleapis.com/auth/indexing) scope: + +## 🔗 Endpoint + +[https://indexing.googleapis.com/v3/urlNotifications:publish](https://www.googleapis.com/auth/indexing) \ No newline at end of file diff --git a/components/google_search_console/actions/submit-url-for-indexing/submit-url-for-indexing.mjs b/components/google_search_console/actions/submit-url-for-indexing/submit-url-for-indexing.mjs new file mode 100644 index 0000000000000..52722f9534e37 --- /dev/null +++ b/components/google_search_console/actions/submit-url-for-indexing/submit-url-for-indexing.mjs @@ -0,0 +1,80 @@ +/** + * Submits a URL to the Google Indexing API to notify that content was updated. + * Uses `URL_UPDATED` type. Full docs in README.md + */ + +import { axios } from "@pipedream/platform"; +import gsConsole from "../../google_search_console.app.mjs"; +import { trimIfString } from "../../common/utils.mjs"; + +export default { + + name: "Submit URL for Indexing", + description: "Sends a URL update notification to the Google Indexing API", + key: "google_search_console-submit-url-for-indexing", + version: "0.0.1", + type: "action", + props: { + gsConsole, + siteUrl: { + type: "string", + label: "URL for indexing", + description: "URL to be submitted for indexing", + }, + }, + + //=================== RUN ============================== + //====================================================== + + async run({ $ }) { + + this.siteUrl = trimIfString(this.siteUrl); + + // Accumulator for non-blocking warnings + let warnings = []; + + /* + Validate the submitted site URL. + - May throw if invalid + - May return warnings for issues like suspicious characters + */ + const urlCheck = this.gsConsole.checkIfUrlValid(this.siteUrl); + + if (urlCheck.warnings) warnings.push(...urlCheck.warnings); + + // Prepare the API response object + let response; + + try { + // Submit the URL to the Google Indexing API + response = await axios($, { + method: "POST", + url: "https://indexing.googleapis.com/v3/urlNotifications:publish", + headers: { + "Authorization": `Bearer ${this.gsConsole.$auth.oauth_access_token}`, + "Content-Type": "application/json", + }, + data: { + url: this.siteUrl, + type: "URL_UPDATED", // Notifies Google that the content at this URL has been updated + }, + }); + } catch (error) { + /* + Determine whether the error originated from: + - Internal validation logic + - The external API call + + Helps distinguish between coding issues vs API issues. + */ + const thrower = gsConsole.methods.checkWhoThrewError(error); + + throw new Error(`Failed to fetch data ( ${thrower.whoThrew} error ) : ${error.message}. ` + warnings.join("\n- ")); + + }; + // Output a summary message and any accumulated warnings + $.export("$summary", ` URL submitted to Google: ${this.siteUrl}` + warnings.join("\n- ")); + // Return the raw API response + return response; + }, +}; diff --git a/components/google_search_console/common/methods.mjs b/components/google_search_console/common/methods.mjs new file mode 100644 index 0000000000000..5952273c822f7 --- /dev/null +++ b/components/google_search_console/common/methods.mjs @@ -0,0 +1,312 @@ +export default { + + /* ============================================================================================ + Throws a custom error and marks it with a flag (`isCustom`) for easier debugging and testing. +================================================================================================ */ + + throwCustomError(msg) { + const err = new Error(msg); + err.isCustom = true; + throw err; + }, + + /* ============================================================================================ + Determines whether an error originated from your own validation code or from the API request. + Useful for debugging and crafting more helpful error messages. +=============================================================================================== */ + + // ===================================================================== + checkWhoThrewError(error) { + return { + whoThrew: error?.response?.status + ? "API response" + : "Internal Code", + error, + }; + }, + + /* ========================================================================================== + Throws if the input is not a string or is a blank string (only whitespace, tabs, + newlines, etc.). +============================================================================================= */ + + throwIfBlankOrNotString(input, reason) { + + if (typeof input !== "string") { + this.throwCustomError( + `Expected a string, but got ${typeof input}` + this._reasonMsg(reason), + ); + } + + if (input.trim() === "") { + this.throwCustomError( + "Expected a non-empty string with visible characters." + this._reasonMsg(reason), + ); + } + + return input; + }, + + /* ============================================================================================== + Throws if the input is not a whole number (non-negative integer). +================================================================================================= */ + + throwIfNotWholeNumber(input, reason) { + if (!Number.isInteger(input) || input < 0) { + this.throwCustomError( `Expected a whole number (0 or positive integer), but got ${typeof input}` + this._reasonMsg(reason)); + + } + + return input; + }, + + /* ============================================================================================= + Throws if the input is not an array. +================================================================================================= */ + + throwIfNotArray(input, reason) { + + if (!Array.isArray(input)) { + this.throwCustomError("Invalid argument type. Expected an array " + this._reasonMsg(reason)); + }; + + return input; + }, + + /* ============================================================================================= + Throws if the input is not an array of strings. +================================================================================================ */ + + throwIfNotArrayOfStrings(input, reason) { + + this.throwIfNotArray(input, reason); + + for (let i = 0; i < input.length; i++) { + if (typeof input[i] !== "string") { + this.throwCustomError("Expected an array of strings " + this._reasonMsg(reason)); + } + }; + + return input; + }, + + /* ============================================================================================== + Throws if the input is not a plain object (arrays are excluded). +================================================================================================= */ + + throwIfNotObject(input, reason) { + + if ( typeof input === "object" && input !== null && !Array.isArray(input)) { + return input; + }; + + this.throwCustomError("Invalid argument type. Expected an object " + this._reasonMsg(reason)); + + }, + + /* ============================================================================================== + Throws if the input is neither an object nor an array. +================================================================================================= */ + + throwIfNotObjectOrArray(input, reason) { + + if ( typeof input === "object" && input !== null) { + return input; + }; + + this.throwCustomError("Invalid argument type. Expected an object or array" + this._reasonMsg(reason)); + + }, + + /* ============================================================================================== + Validates that a string follows the YYYY-MM-DD date format. + Throws if the format or any individual part (year, month, day) is invalid. +================================================================================================= */ + + throwIfNotYMDDashDate(input, reason) { + + this.throwIfBlankOrNotString(input, reason); + + const parts = input.trim().split("-"); + + if (parts.length !== 3) { + this.throwCustomError("Date must be in format YYYY-MM-DD" + this._reasonMsg(reason)); + }; + + let [ + year, + month, + day, + ] = parts; + + // --- YEAR --- + year = +year; + if (!Number.isInteger(year) || year < 1990 || year > 2100) { + this.throwCustomError("Year must be between 1990 and 2100" + this._reasonMsg(reason)); + }; + + const monthNum = +month; + if (month.length !== 2 || Number.isNaN(monthNum) || monthNum < 1 || monthNum > 12 ) { + this.throwCustomError(`Month must be between 01 and 12. Got: '${month}'` + this._reasonMsg(reason)); + }; + + const dayNum = +day; + if (day.length !== 2 || Number.isNaN(dayNum) || dayNum < 1 || dayNum > 31 ) { + this.throwCustomError(`Day must be between 01 and 31. Got: '${day}'` + this._reasonMsg(reason)); + }; + + return input; + }, + + /* ============================================================================================== + Validates a URL string: + - Rejects blank strings, spaces, tabs, and newlines + - Warns about suspicious or unusual characters + - Adds a warning if protocol is missing or malformed + Returns: + { + warnings: string[], + url: string (trimmed original input) + } +================================================================================================= */ + + // ===================================================================== + checkIfUrlValid(input, reason) { + + // Throws an error if the input is not a string or if its a blank string; + this.throwIfBlankOrNotString(input); + + // Warnin accumulator + let warnings = []; + ; + // Trim the input (already checked for string); + const trimmedInput = input.trim(); + + // Reject if spaces, tabs, or newlines are present + if ((/[ \t\n]/.test(trimmedInput))) { + this.throwCustomError( "Url contains invalid characters like space, backslash etc., please check." + this._reasonMsg(reason)); + }; + + // Warn about suspicious characters + const dubiousCharRegex = /[\\<>{}|"`^]|\u200B|\u200C|\u200D|\u2060|\uFEFF|\u00A0/g; + + const dubiousMatches = trimmedInput.match(dubiousCharRegex); + + if (dubiousMatches) { + const uniqueChars = [ + ...new Set(dubiousMatches), + ].join(" "); + + warnings.push(` URL contains dubious or non-standard characters " ${uniqueChars} " ` + + `that may be rejected by Google. Proceed only if you know what you are doing. ${this._reasonMsg(reason)}`) ; + }; + + // urlObject for further use if the next check passes. + let urlObject; + // Tries to create a new URL object with the input string; + try { + urlObject = new URL(trimmedInput); // throws if invalid or has no protocol + + // Warn if user typed only one slash (e.g., https:/) + if (/^(https?):\/(?!\/)/.test(input)) { + + warnings.push(` It looks like you're missing one slash after "${urlObject.protocol}".` + + `Did you mean "${urlObject.protocol}//..."? ${this._reasonMsg(reason)} `); + + }; + + } catch (err) { + // If the URL is invalid, try to create a new URL object with "http://" in case user forgot to add it; + try { + // If it works then there was no protocol in the input string; + urlObject = new URL("http://" + trimmedInput); + + warnings.push(" URL does not have http or https protocol \""); + + } catch (err) { + // If after all checks we are here that means that the url + // contain potentially unacceptable characters. + warnings.push(` URL contains potentionally unacceptable characters" ${this._reasonMsg(reason)}`); + + }; + + }; + + // Here I wanted to check for "ftp" and other protocols but if user + // uses them they know what they are doing. + + return { + warnings: warnings, + url: input, + }; + + }, + + /* ============================================================================================== + Appends a reason string to error messages for additional context. +================================================================================================= */ + + _reasonMsg(reason) { + + return (reason && typeof reason === "string") + ? ` Reason: ${reason} ` + : ""; + }, + + /* ============================================================================================== + Calls the appropriate validation method based on the input's type. + Uses either the standard `type` or an extended override via `extendedType`. + Throws if no validator is found or if the value fails validation. +================================================================================================= */ + + validateUserInput(propMeta, value) { + + this.throwIfNotObject(propMeta, propMeta.label); + + // Check if there is extra type in prop meta. Like string can be also date type + const type = ("extendedType" in propMeta) + ? propMeta.extendedType + : propMeta.type; + + // Decide which method to call based on the input we want to check. + + const validators = { + "string": this.throwIfBlankOrNotString, + "integer": this.throwIfNotWholeNumber, + "string[]": this.throwIfNotArrayOfStrings, + "object": this.throwIfNotObjectOrArray, + "array": this.throwIfNotArray, + "YYYY-MM-DD": this.throwIfNotYMDDashDate, + "url": this.checkIfUrlValid, + }; + + const validator = validators[type]; + + if (!validator) { + this.throwCustomError(`No validator found for type "${type}"`, propMeta.label); + }; + + // Calls the correct function and passes label + // to show as a reason if it throws an error or issues a warning. + return validator.call(this, value, propMeta.label); + + }, + + /* ============================================================================================== + Attempts to parse a string as JSON. + - Returns the parsed object if valid + - Returns the original value if not a string + - Throws a custom error if parsing fails +================================================================================================= */ + + parseIfJsonString(input, reason) { + if (typeof input !== "string") return input; + + try { + return JSON.parse(input); + } catch (err) { + this.throwCustomError( `Can not parse JSON. Error : ${err}` + this._reasonMsg(reason)); + }; + }, + +}; diff --git a/components/google_search_console/common/tests/README.md b/components/google_search_console/common/tests/README.md new file mode 100644 index 0000000000000..d234532b520e0 --- /dev/null +++ b/components/google_search_console/common/tests/README.md @@ -0,0 +1,44 @@ +# Local Tests Directory + +This folder contains **non-production test scripts** used for developing and validating Pipedream components locally. +**You can safely delete the entire folder** — it does not affect runtime or deployment. + +--- + +## Folder Structure + +### `action-tests/` +Component-level test runners that simulate a real Pipedream `$` context using `mockery-dollar.mjs`. + +- `test-retrieve-site-performance-data.mjs` – Tests the Search Analytics action +- `test-submit-url-for-indexing.mjs` – Tests the Indexing API action + +--- + +### `methods/` +Unit tests for reusable validation and utility methods found in `.app.mjs`. + +- `test-checkIfUrlValid.mjs` – Tests URL validation helper +- `test-throwIfNotYMDDashDate.mjs` – Tests strict date validation + +#### `bogus-data/` +Mocked data used to simulate edge-case user inputs and trigger validations: +- `bogus-data-url.mjs` – Invalid or suspicious URLs +- `bogus-data-google-date.mjs` – Date values to test against expected format + +--- + +### Root-level Utilities + +- `mockery-dollar.mjs` – Mocks the `$` object Pipedream injects into actions +- `get-token.mjs` – Script for manually supplying a Google OAuth token during local testing + +--- + +## ⚠️ Notes + +- Some files may contain **hardcoded tokens** — be sure to exclude them from commits. +- All files here are meant for **local testing only**. +- Delete this folder any time before publishing — it's safe and has no impact on your app. + +--- \ No newline at end of file diff --git a/components/google_search_console/common/tests/action-tests/test-retrieve-site-performance-data.mjs b/components/google_search_console/common/tests/action-tests/test-retrieve-site-performance-data.mjs new file mode 100644 index 0000000000000..aee46bc08b14b --- /dev/null +++ b/components/google_search_console/common/tests/action-tests/test-retrieve-site-performance-data.mjs @@ -0,0 +1,251 @@ +/** + * ───────────────────────────────────────────────────────────────────────────── + * LOCAL TEST RUNNER – DO NOT DEPLOY + * + * This file is used to run and debug a Pipedream action **locally** outside + * of the Pipedream platform. Safe to delete. It does **not** affect production. + * + * It: + * - Injects mocked `$` object (logs, summary) + * - Bypasses OAuth by using hardcoded access token (get-token.mjs) + * - Validates, builds, and sends the Search Console request + * + * You MUST: + * - Replace `Authorization` token with a valid one manually + * - Ensure `siteUrl` is verified in your Search Console + * + * ───────────────────────────────────────────────────────────────────────────── + */ + +import mockery$ from "../mockery-dollar.mjs"; +import { axios } from "@pipedream/platform"; +import gsConsole from "../../../google_search_console.app.mjs"; +import { + removeCustomPropFields, trimIfString, +} from "../../../common/utils.mjs"; + +// Define prop metadata separately and spread it into the props object. +// Useful for accessing extended metadata during runtime — available because it stays in closure. + +// ====================== TEST PREPARATION ===================================================== + +// TEST ONLY +const mockeryData = { + siteUrl: "https://falc1.com/", // Must be verified in your GSC account + + // Required + startDate: "2025-12-22", + endDate: "2025-12-31", + + // Recommended + dimensions: [ + "query", + "page", + "country", + "device", + ], // valid values only + + searchType: "web", // one of: web, image, video, news, googleNews, discover + + rowLimit: 10, + + startRow: 0, // Optional pagination + + // Optional but valid + aggregationType: "auto", // or "byPage" + dataState: "final", // or "all" + + // Optional filter (valid structure) + dimensionFilterGroups: [ + { + groupType: "and", // "and" or "or" + filters: [ + { + dimension: "query", + operator: "contains", // "equals", "contains", "notEquals", etc. + expression: "example", + }, + { + dimension: "country", + operator: "equals", + expression: "USA", + }, + ], + }, + ], +}; + +const propsMeta = { + + siteUrl: { + type: "string", + extendedType: "url", + label: "Verified Site URL", + description: "Including https:// is strongly advised", + }, + startDate: { + type: "string", + extendedType: "YYYY-MM-DD", + label: "Start Date (YYYY-MM-DD)", + postBody: true, + }, + endDate: { + type: "string", + extendedType: "YYYY-MM-DD", + label: "End Date (YYYY-MM-DD)", + postBody: true, + }, + dimensions: { + type: "string[]", + label: "Dimensions", + optional: true, + description: "e.g. ['query', 'page', 'country', 'device']", + postBody: true, + }, + searchType: { + type: "string", + label: "Search Type", + optional: true, + options: [ + "web", + "image", + "video", + "news", + "googleNews", + "discover", + ], + default: "web", + postBody: true, + }, + aggregationType: { + type: "string", + label: "Aggregation Type", + optional: true, + options: [ + "auto", + "byPage", + ], + postBody: true, + }, + rowLimit: { + type: "integer", + label: "Max rows to return", + default: 10, + postBody: true, + }, + startRow: { + type: "integer", + label: "Start row (for pagination)", + optional: true, + postBody: true, + }, + dimensionFilterGroups: { + type: "object", + label: "Dimension Filters", + optional: true, + description: "Follow Search Console API structure for filters", + postBody: true, + }, + dataState: { + type: "string", + label: "Data State", + optional: true, + options: [ + "all", + "final", + ], + default: "final", + postBody: true, + }, + +}; + +const testAction = { + + ...mockeryData, + + name: "Get Site Performance Data", + description: "Fetches search analytics data for a verified site.", + key: "get_search_console_analytics", + version: "0.0.47", + type: "action", + props: { + gsConsole, + // Remove custom prop metadata and spread only valid prop fields + ...removeCustomPropFields(propsMeta), + }, + + async run({ $ }) { + + // body for POST request. Will be filled in the following loop. + const body = {}; + + // warnings accumulator + let warnings = []; + + /* This loop performs the following tasks: + - Validates user input + - Populates the `body` object for the upcoming POST request + - Accumulates messages to display to the user at the end of the action +*/ + for (let propName in propsMeta) { + + console.log("===VALUE", this[propName]); + const meta = propsMeta[propName]; + + // Trim the input if it's a string + this[propName] = trimIfString(this[propName]); + + // If the optional prop is undefined, null, or a blank string — skip it + if (meta.optional === true && ((this[propName] ?? "") === "")) continue; + + // Validate the input and throw an error if it's invalid. + // Also return an empty string or a warning message, if applicable + const validationResult = gsConsole.methods.validateUserInput(meta, this[propName]); + + // Push the warnings into warnings accumulator if any. + if (validationResult.warnings) warnings.push(...validationResult.warnings); + + // If the prop should be included in the POST request, add it to the body + if (meta.postBody === true) body[propName] = this[propName]; + + console.log(" SUCCESS"); + }; + + // Trimmed in loop above + const url = this.siteUrl; + + // Response of the POST request. + let response; + + try { + response = await axios($, { + method: "POST", + url: `https://searchconsole.googleapis.com/webmasters/v3/sites/${encodeURIComponent(url)}/searchAnalytics/query`, + headers: { + // TEMP: Local-only token for manual testing. Do not commit this to source control. + "Authorization": "Bearer *HARDCODED TOKEN HERE*", + "Content-Type": "application/json", + }, + data: body, + }); + + } catch (error) { + // Check who threw the error. Internal code or the request. To ease debugging. + const thrower = gsConsole.methods.checkWhoThrewError(error); + + throw new Error(`Failed to fetch data ( ${thrower.whoThrew} error ) : ${error.message}. ` + warnings.join("\n- ")); + + }; + + $.export("$summary", ` Fetched ${response.rows?.length || 0} rows of data. ` + warnings.join("\n- ")); + return response; + }, +}; + +// await is just in case if node wants to finish its job before time =) +async function runTest() { + await testAction.run(mockery$); +} + +runTest(); diff --git a/components/google_search_console/common/tests/action-tests/test-submit-url-for-indexing.mjs b/components/google_search_console/common/tests/action-tests/test-submit-url-for-indexing.mjs new file mode 100644 index 0000000000000..3c202f520ddc6 --- /dev/null +++ b/components/google_search_console/common/tests/action-tests/test-submit-url-for-indexing.mjs @@ -0,0 +1,92 @@ +/** + * ───────────────────────────────────────────────────────────────────────────── + * LOCAL TEST RUNNER – DO NOT DEPLOY + * + * This file is used to run and debug a Pipedream action **locally** outside + * of the Pipedream platform. Safe to delete. It does **not** affect production. + * + * It: + * - Injects a mocked `$` object (`mockery$`) + * - Bypasses OAuth using a hardcoded access token (get-token.mjs) + * - Sends a URL submission request to the Google Indexing API + * + * You MUST: + * - Replace the `Authorization` token with a valid one manually + * - Ensure the `siteUrl` is verified in your Search Console account + * + * ───────────────────────────────────────────────────────────────────────────── + */ + +import mockery$ from "../mockery-dollar.mjs"; // TEST ONLY. Delete in production code +import { axios } from "@pipedream/platform"; +import gsConsole from "../../../google_search_console.app.mjs"; +import { trimIfString } from "../../../common/utils.mjs"; + +const mockeryData = { + siteUrl: "https://falc1.com/", // Must be verified in your GSC account +}; + +const testAction = { // TEST ONLY. Replace to export in real code + + ...mockeryData, // TEST ONLY. Delete in production code + + name: "Google test", + description: "This is a demo action", + key: "google_test", + version: "0.0.23", + type: "action", + props: { + gsConsole, + url: { + type: "string", + label: "URL for indexing", + description: "URL to be submitted for indexing", + }, + }, + async run({ $ }) { + + this.url = trimIfString(this.url); + + // warnings accumulator + let warnings = []; + + const urlCheck = gsConsole.methods.checkIfUrlValid(this.url); // TEST ONLY. Replace to "this" + + if (urlCheck.warnings) warnings.push(...urlCheck.warnings); + + // Response of the POST request. + let response; + + try { + response = await axios($, { + method: "POST", + url: "https://indexing.googleapis.com/v3/urlNotifications:publish", + headers: { + // Tested with real hardcoded token that had required scopes. + "Authorization": "Bearer *HARDCODED TOKEN HERE*", // Replace with a token from get-token.mjs with scope: "https://www.googleapis.com/auth/indexing". + "Content-Type": "application/json", + }, + data: { + url: this.url, + type: "URL_UPDATED", + }, + }); + } catch (error) { + // Check who threw the error. Internal code or the request. To ease debugging. + const thrower = gsConsole.methods.checkWhoThrewError(error); + + throw new Error(`Failed to fetch data ( ${thrower.whoThrew} error ) : ${error.message}. ` + warnings.join("\n- ")); + + }; + + $.export("$summary", ` URL submitted to Google: ${this.siteUrl}` + warnings.join("\n- ")); + return response; + }, +}; + +// await is just in case if node wants to finish its job before time =) +async function runTest() { + await testAction.run(mockery$); +} + +runTest(); diff --git a/components/google_search_console/common/tests/get-token.mjs b/components/google_search_console/common/tests/get-token.mjs new file mode 100644 index 0000000000000..c2fe5dabf56cb --- /dev/null +++ b/components/google_search_console/common/tests/get-token.mjs @@ -0,0 +1,81 @@ +// Bare-bones OAuth script to fetch a token. +// For testing purposes only — no timeouts, no browser response. +// Manually terminate the script from the console when finished. + +import open from "open"; +import http from "http"; + +// --- Google OAuth Config --- +const clientId = "*CLIEN ID HERE*"; +const redirectUri = "http://localhost:3000"; +const scopes = [ + "https://www.googleapis.com/auth/webmasters.readonly", + "https://www.googleapis.com/auth/indexing", +].join(" "); + +const authUrl = new URL("https://accounts.google.com/o/oauth2/v2/auth"); +authUrl.searchParams.set("client_id", clientId); +authUrl.searchParams.set("redirect_uri", redirectUri); +authUrl.searchParams.set("response_type", "code"); +authUrl.searchParams.set("scope", scopes); +authUrl.searchParams.set("access_type", "offline"); +authUrl.searchParams.set("prompt", "consent"); + +new Promise((resolve, reject) => { + // Start a local HTTP server to catch Google's OAuth redirect + http.createServer((incomingRequest) => { + const fullRequestUrl = new URL(incomingRequest.url, `http://${incomingRequest.headers.host}`); + const authorizationCode = fullRequestUrl.searchParams.get("code"); + + if (authorizationCode) { + console.log("AUTH CODE RECEIVED:", authorizationCode); + resolve(authorizationCode); + } else { + console.log("Unexpected request received — no code param found."); + reject("Unexpected request received — no code param."); + } + }).listen(3000, () => { + console.log("🌐 Listening on http://localhost:3000"); + console.log("🔗 Opening browser for Google OAuth..."); + open(authUrl.toString()); + }); +}) + .then((authorizationCode) => { + console.log("✅ Got the code! Exchanging for tokens..."); + + fetch("https://oauth2.googleapis.com/token", { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + code: authorizationCode, + client_id: clientId, + client_secret: "*CLIENT SECRET HERE", + redirect_uri: redirectUri, + grant_type: "authorization_code", + }), + }) + .then((res) => res.json()) + .then((data) => { + if (data.access_token) { + console.log("🎉 Tokens received!"); + console.log("Access Token:", data.access_token); + console.log("Refresh Token:", data.refresh_token); + console.log("Expires In:", data.expires_in); + console.log("Token retrieval complete. Shutting down server..."); + process.exit(0); + } else { + console.error(" Failed to get tokens:", data); + process.exit(1); + } + }) + .catch((err) => { + console.error("Error while exchanging code:", err); + process.exit(1); + }); + }) + .catch((error) => { + console.error("Error during authorization:", error); + process.exit(1); + }); diff --git a/components/google_search_console/common/tests/methods/bogus-data/bogus-data-google-date.mjs b/components/google_search_console/common/tests/methods/bogus-data/bogus-data-google-date.mjs new file mode 100644 index 0000000000000..e9ac872f1dbfe --- /dev/null +++ b/components/google_search_console/common/tests/methods/bogus-data/bogus-data-google-date.mjs @@ -0,0 +1,128 @@ +export default { + aSingleWordString: { + value: "wrgwgwepkgprgprpwnwwrehwrh", + jsType: "string", + extendedType: "singleWordString", + }, + aMultiWordString: { + value: "etheth etphmpethm ethpethm", + jsType: "string", + extendedType: "aMultiWordString", + }, + aPositiveInteger: { + value: 15115, + jsType: "number", + extendedType: "positiveInteger", + }, + aNegativeInteger: { + value: -15115, + jsType: "number", + extendedType: "negativeInteger", + }, + aPositiveFloat: { + value: 12.155, + jsType: "number", + extendedType: "positiveFloat", + }, + aNegativeFloat: { + value: -12.155, + jsType: "number", + extendedType: "negativeFloat", + }, + aZero: { + value: 0, + jsType: "number", + extendedType: "zero", + }, + anArray: { + value: [ + 13135, + 35.15, + "3feqe", + [ + 1, + 2, + 3, + ], + { + some: "val", + }, + 0, + ], + jsType: "object", + extendedType: "trueArray", + }, + + anArrayOfStrings: { + value: [ + "string1", + "string2", + "string3", + ], + jsType: "object", + extendedType: "arrayOfStrings", + }, + anObject: { + value: { + some: "val", + some2: 23, + some3: [ + 1, + 2, + 3, + ], + }, + jsType: "object", + extendedType: "trueObject", + }, + + anEmptyString: { + value: "", + jsType: "string", + extendedType: "emptyString", + }, + + aSpace: { + value: " ", + jsType: "string", + extendedType: "space", + }, + + aValidYMDDashDate: { + value: "2025-05-28", + jsType: "string", + extendedType: "ymdDashDate", + }, + + aCharinYMDDashDate: { + value: "20G5-h5-28", + jsType: "string", + extendedType: "ymdDashDate", + }, + + aShortYearYMDDashDate: { + value: "20-23-28", + jsType: "string", + extendedType: "ymdDashDate", + }, + + aMonthOusideValYMDDashDate: { + value: "2025-23-28", + jsType: "string", + extendedType: "ymdDashDate", + }, + + aDayOusideValYMDDashDate: { + value: "2025-12-56", + jsType: "string", + extendedType: "ymdDashDate", + }, + + anInvalidSeparatorYMDDashDate: { + value: "2025/12.56", + jsType: "string", + extendedType: "ymdDashDate", + }, + +}; + diff --git a/components/google_search_console/common/tests/methods/bogus-data/bogus-data-url.mjs b/components/google_search_console/common/tests/methods/bogus-data/bogus-data-url.mjs new file mode 100644 index 0000000000000..b8933a29ffd31 --- /dev/null +++ b/components/google_search_console/common/tests/methods/bogus-data/bogus-data-url.mjs @@ -0,0 +1,133 @@ +export default { + aSingleWordString: { + value: "wrgwgwepkgprgprpwnwwrehwrh", + jsType: "string", + extendedType: "singleWordString", + }, + aMultiWordString: { + value: "etheth etphmpethm ethpethm", + jsType: "string", + extendedType: "aMultiWordString", + }, + aPositiveInteger: { + value: 15115, + jsType: "number", + extendedType: "positiveInteger", + }, + aNegativeInteger: { + value: -15115, + jsType: "number", + extendedType: "negativeInteger", + }, + aPositiveFloat: { + value: 12.155, + jsType: "number", + extendedType: "positiveFloat", + }, + aNegativeFloat: { + value: -12.155, + jsType: "number", + extendedType: "negativeFloat", + }, + aZero: { + value: 0, + jsType: "number", + extendedType: "zero", + }, + anArray: { + value: [ + 13135, + 35.15, + "3feqe", + [ + 1, + 2, + 3, + ], + { + some: "val", + }, + 0, + ], + jsType: "object", + extendedType: "trueArray", + }, + + anArrayOfStrings: { + value: [ + "string1", + "string2", + "string3", + ], + jsType: "object", + extendedType: "arrayOfStrings", + }, + anObject: { + value: { + some: "val", + some2: 23, + some3: [ + 1, + 2, + 3, + ], + }, + jsType: "object", + extendedType: "trueObject", + }, + + anEmptyString: { + value: "", + jsType: "string", + extendedType: "emptyString", + }, + + aSpace: { + value: " ", + jsType: "string", + extendedType: "space", + }, + + aValidHttpUrl: { + value: "http://example.com", + jsType: "string", + extendedType: "url", + }, + + aValidHttpsUrl: { + value: "https://example.com", + jsType: "string", + extendedType: "url", + }, + + aValidUrlWithoutProtocol: { + value: "example.com", + jsType: "string", + extendedType: "url", + }, + anUrlWithSpace: { + value: "https://exa mple.com", + jsType: "string", + extendedType: "url", + }, + + aReverseSlashUrl: { + value: "http://example.com\\path", + jsType: "string", + extendedType: "url", + }, + + aOneSlashUrl: { + value: "http:/example.com", + jsType: "string", + extendedType: "url", + }, + + aValidUrlWithDubiousCharacters: { + value: "http://example.com\\