Fix filter handling for Google Search Console action (#16467)

* Fixing retrieve-site-perf-data action

* Update pnpm-lock.yaml

Author: dannyroosevelt

components/google_search_console/README.md

@@ -1,18 +1,39 @@
 # Overview
 
-The Google Search Console API opens a treasure trove of data and insights about your website’s presence in Google Search results. You can get detailed reports on your site's search traffic, manage and test your site's sitemaps and robots.txt files, and see which queries bring users to your site. On Pipedream, utilize this API to automate checks on site performance, integrate with other tools for deeper analysis, or keep tabs on your SEO strategy's effectiveness.
+The Google Search Console API opens a treasure trove of data and insights about your website's presence in Google Search results. You can get detailed reports on your site's search traffic, manage and test your site's sitemaps and robots.txt files, and see which queries bring users to your site. On Pipedream, utilize this API to automate checks on site performance, integrate with other tools for deeper analysis, or keep tabs on your SEO strategy's effectiveness.
 
-# Example Use Cases
+## Working with Domain Properties and Subdomains
 
-- **SEO Performance Report to Slack**: Automate daily or weekly SEO performance reports. Use the Google Search Console API to fetch search analytics data, then send a summary report to a Slack channel, keeping the team informed about trends, keyword rankings, and click-through rates.
+Google Search Console distinguishes between URL properties and Domain properties:
 
-- **Sync Search Results with Google Sheets**: Create a workflow that periodically pulls data from the Google Search Console API and adds it to a Google Sheet. This is useful for maintaining an evolving dataset for deeper analysis, historical reference, or sharing insights across teams without giving direct access to the Search Console.
+- **URL properties** are specific site URLs (e.g., `https://example.com` or `https://www.example.com`)
+- **Domain properties** include all subdomains and protocols (e.g., `sc-domain:example.com`)
 
-- **Automatic Sitemap Submission**: Set up a Pipedream workflow that triggers whenever a new sitemap is generated in your content management system (CMS). The workflow can then automatically submit the sitemap to Google Search Console via API, ensuring Google has the latest structure of your site for crawling and indexing.
+When working with subdomains:
+
+1. Select the domain property from the dropdown (e.g., `sc-domain:example.com`)
+2. Enter the subdomain URL in the "Subdomain Filter" field (e.g., `https://mcp.example.com`)
+3. By default, this will filter for pages containing that subdomain URL, including all subpages like `https://mcp.example.com/app/slack`
+
+This approach ensures you can access subdomain data even if the subdomain isn't individually verified in Search Console.
+
+### Important: Getting Data for Individual Pages
+
+To see data broken down by individual pages (rather than just aggregate data):
 
-## Available Actions
+- Add "page" to your dimensions list
+- This will return separate rows for each page, rather than a single aggregated row
 
-[Retrieve Site Performance Data](./actions/retrieve-site-performance-data/README.md)
-[Submit URL for Indexing](./actions/submit-url-for-indexing/README.md)
+For advanced filtering needs, you can also:
 
+- Change the filter dimension (page, query, country, etc.)
+- Change the filter operator (contains, equals, etc.)
+- Or use the advanced filters for complete customization
 
+## Example Use Cases
+
+- **SEO Performance Report to Slack**: Automate daily or weekly SEO performance reports. Use the Google Search Console API to fetch search analytics data, then send a summary report to a Slack channel, keeping the team informed about trends, keyword rankings, and click-through rates.
+
+- **Sync Search Results with Google Sheets**: Create a workflow that periodically pulls data from the Google Search Console API and adds it to a Google Sheet. This is useful for maintaining an evolving dataset for deeper analysis, historical reference, or sharing insights across teams without giving direct access to the Search Console.
+
+- **Automatic Sitemap Submission**: Set up a Pipedream workflow that triggers whenever a new sitemap is generated in your content management system (CMS). The workflow can then automatically submit the sitemap to Google Search Console via API, ensuring Google has the latest structure of your site for crawling and indexing.

components/google_search_console/actions/retrieve-site-performance-data/retrieve-site-performance-data.mjs

@@ -5,14 +5,16 @@ export default {
   name: "Retrieve Site Performance Data",
   description: "Fetches search analytics from Google Search Console for a verified site.",
   key: "google_search_console-retrieve-site-performance-data",
-  version: "0.0.2",
+  version: "0.0.3",
   type: "action",
   props: {
     googleSearchConsole,
     siteUrl: {
-      type: "string",
-      label: "Verified Site URL",
-      description: "Including https:// is strongly advised",
+      propDefinition: [
+        googleSearchConsole,
+        "siteUrl",
+      ],
+      description: "Select a verified site from your Google Search Console. For subdomains, select the domain property and use dimension filters.",
     },
     startDate: {
       type: "string",
@@ -22,13 +24,21 @@ export default {
     endDate: {
       type: "string",
       label: "End Date (YYYY-MM-DD)",
-      description: "Enddate of the range for which to retrieve site performance data",
+      description: "End date of the range for which to retrieve site performance data",
     },
     dimensions: {
       type: "string[]",
       label: "Dimensions",
       optional: true,
       description: "e.g. ['query', 'page', 'country', 'device']",
+      options: [
+        "country",
+        "device",
+        "page",
+        "query",
+        "searchAppearance",
+        "date",
+      ],
     },
     searchType: {
       type: "string",
@@ -68,11 +78,45 @@ export default {
       description: "Start row (for pagination)",
       optional: true,
     },
-    dimensionFilterGroups: {
+    subdomainFilter: {
+      type: "string",
+      label: "Subdomain Filter",
+      optional: true,
+      description: "Filter results to a specific subdomain when using a domain property (e.g., `https://subdomain.example.com`). This will include all subpages of the subdomain.",
+    },
+    filterDimension: {
+      type: "string",
+      label: "Filter Dimension",
+      optional: true,
+      description: "Dimension to filter by (defaults to page when subdomain filter is used). Using 'page' will match the subdomain and all its subpages.",
+      options: [
+        "country",
+        "device",
+        "page",
+        "query",
+      ],
+      default: "page",
+    },
+    filterOperator: {
+      type: "string",
+      label: "Filter Operator",
+      optional: true,
+      description: "Operator to use for filtering (defaults to contains when subdomain filter is used)",
+      options: [
+        "contains",
+        "equals",
+        "notContains",
+        "notEquals",
+        "includingRegex",
+        "excludingRegex",
+      ],
+      default: "contains",
+    },
+    advancedDimensionFilters: {
       type: "object",
-      label: "Dimension Filters",
+      label: "Advanced Dimension Filters",
       optional: true,
-      description: "Follow Search Console API structure for filters",
+      description: "For advanced use cases: custom dimension filter groups following Search Console API structure.",
     },
     dataState: {
       type: "string",
@@ -90,7 +134,10 @@ export default {
     const {
       googleSearchConsole,
       siteUrl,
-      dimensionFilterGroups,
+      subdomainFilter,
+      filterDimension,
+      filterOperator,
+      advancedDimensionFilters,
       ...fields
     } = this;
 
@@ -102,23 +149,56 @@ export default {
       return acc;
     }, {});
 
+    // Build dimension filters based on user input
+    let dimensionFilterGroups;
+
+    if (subdomainFilter) {
+      // If user provided a subdomain filter, create the filter structure
+      dimensionFilterGroups = {
+        filterGroups: [
+          {
+            filters: [
+              {
+                dimension: filterDimension || "page",
+                operator: filterOperator || "contains",
+                expression: subdomainFilter,
+              },
+            ],
+          },
+        ],
+      };
+    } else if (advancedDimensionFilters) {
+      // If user provided advanced filters, use those
+      dimensionFilterGroups = googleSearchConsole.parseIfJsonString(advancedDimensionFilters);
+    }
+
     let response;
     try {
       response = await googleSearchConsole.getSitePerformanceData({
         $,
         url: siteUrl,
         data: {
           ...body,
-          dimensionFilterGroups: googleSearchConsole.parseIfJsonString(dimensionFilterGroups),
+          dimensionFilterGroups,
         },
       });
     } catch (error) {
       // Identify if the error was thrown by internal validation or by the API call
       const thrower = googleSearchConsole.checkWhoThrewError(error);
-      throw new Error(`Failed to fetch data ( ${thrower.whoThrew} error ) : ${error.message}. `);
-    };
 
-    $.export("$summary", ` Fetched ${response.rows?.length || 0} rows of data. `);
+      // Add more helpful error messages for common 403 errors
+      if (error.response?.status === 403) {
+        const message = "Access denied. If you're trying to access a subdomain, select the domain property (sc-domain:example.com) and use the subdomain filter to filter for your subdomain.";
+        throw new Error(`Failed to fetch data: ${message}`);
+      }
+
+      throw new Error(`Failed to fetch data (${thrower.whoThrew} error): ${error.message}`);
+    }
+
+    const rowCount = response.rows?.length || 0;
+    $.export("$summary", `Fetched ${rowCount} ${rowCount === 1
+      ? "row"
+      : "rows"} of data.`);
     return response;
   },
 };

components/google_search_console/actions/submit-url-for-indexing/submit-url-for-indexing.mjs

@@ -5,22 +5,41 @@ export default {
   name: "Submit URL for Indexing",
   description: "Sends a URL update notification to the Google Indexing API",
   key: "google_search_console-submit-url-for-indexing",
-  version: "0.0.2",
+  version: "0.0.3",
   type: "action",
   props: {
     googleSearchConsole,
     siteUrl: {
       type: "string",
       label: "URL for indexing",
-      description: "URL to be submitted for indexing",
+      description: "URL to be submitted for indexing (must be a canonical URL that's verified in Google Search Console)",
+    },
+    notificationType: {
+      type: "string",
+      label: "Notification Type",
+      description: "Type of notification to send to Google",
+      options: [
+        {
+          label: "URL Updated (content has been updated)",
+          value: "URL_UPDATED",
+        },
+        {
+          label: "URL Deleted (page no longer exists)",
+          value: "URL_DELETED",
+        },
+      ],
+      default: "URL_UPDATED",
     },
   },
   async run({ $ }) {
-    const siteUrl = trimIfString(this.siteUrl);
+    const {
+      siteUrl, notificationType,
+    } = this;
+    const trimmedUrl = trimIfString(siteUrl);
 
     const warnings = [];
 
-    const urlCheck = this.googleSearchConsole.checkIfUrlValid(siteUrl);
+    const urlCheck = this.googleSearchConsole.checkIfUrlValid(trimmedUrl);
     if (urlCheck.warnings) {
       warnings.push(...urlCheck.warnings);
     }
@@ -30,18 +49,32 @@ export default {
       response = await this.googleSearchConsole.submitUrlForIndexing({
         $,
         data: {
-          url: siteUrl,
-          type: "URL_UPDATED", // Notifies Google that the content at this URL has been updated
+          url: trimmedUrl,
+          type: notificationType,
         },
       });
     } catch (error) {
       const thrower = this.googleSearchConsole.checkWhoThrewError(error);
-      throw new Error(`Failed to fetch data ( ${thrower.whoThrew} error ) : ${error.message}. ` + warnings.join("\n- "));
 
-    };
+      // Add more helpful error messages for common errors
+      if (error.response?.status === 403) {
+        throw new Error("Access denied. Make sure the URL belongs to a property you have access to in Google Search Console.");
+      }
+
+      if (error.response?.status === 400) {
+        throw new Error("Invalid request. Ensure the URL is canonical and belongs to a verified property.");
+      }
+
+      throw new Error(`Failed to submit URL (${thrower.whoThrew} error): ${error.message}`);
+    }
+
+    // Format warnings string if any warnings exist
+    const warningsString = warnings.length > 0
+      ? `\n- ${warnings.join("\n- ")}`
+      : "";
 
     // Output a summary message and any accumulated warnings
-    $.export("$summary", ` URL submitted to Google: ${this.siteUrl}`  + warnings.join("\n- "));
+    $.export("$summary", `URL submitted to Google: ${trimmedUrl}${warningsString}`);
 
     return response;
   },

components/google_search_console/google_search_console.app.mjs

@@ -4,7 +4,17 @@ import methods from "./common/methods.mjs";
 export default {
   type: "app",
   app: "google_search_console",
-  propDefinitions: {},
+  propDefinitions: {
+    siteUrl: {
+      type: "string",
+      label: "Site",
+      description: "Select a verified site from your Search Console",
+      async options({ prevContext }) {
+        const { nextPageToken } = prevContext || {};
+        return this.listSiteOptions(nextPageToken);
+      },
+    },
+  },
   methods: {
     ...methods,
     _makeRequest({
@@ -21,6 +31,35 @@ export default {
         ...opts,
       });
     },
+    async getSites(params = {}) {
+      return this._makeRequest({
+        method: "GET",
+        url: "https://searchconsole.googleapis.com/webmasters/v3/sites",
+        ...params,
+      });
+    },
+    async listSiteOptions(pageToken) {
+      const params = {};
+      if (pageToken) {
+        params.pageToken = pageToken;
+      }
+
+      const {
+        siteEntry = [], nextPageToken,
+      } = await this.getSites({
+        params,
+      });
+
+      return {
+        options: siteEntry.map((site) => ({
+          label: site.siteUrl,
+          value: site.siteUrl,
+        })),
+        context: {
+          nextPageToken,
+        },
+      };
+    },
     getSitePerformanceData({
       url, ...opts
     }) {

components/google_search_console/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/google_search_console",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Pipedream google_search_console Components",
   "main": "google_search_console.app.mjs",
   "keywords": [