add custom urls checker

Author: antoinekm

src/index.ts

@@ -7,6 +7,7 @@ import {
   getPageIndexingStatus,
   convertToFilePath,
   checkSiteUrl,
+  checkCustomUrls,
 } from "./shared/gsc";
 import { getSitemapPages } from "./shared/sitemap";
 import { Status } from "./shared/types";
@@ -56,6 +57,7 @@ export const index = async (input: string = process.argv[2], options: IndexOptio
   }
   if (!options.urls) {
     options.urls = args["urls"] ? args["urls"].split(",") : undefined;
+  }
   if (!options.quota) {
     options.quota = {
       rpmRetry: args["rpm-retry"] === "true" || process.env.GIS_QUOTA_RPM_RETRY === "true",
@@ -86,9 +88,12 @@ export const index = async (input: string = process.argv[2], options: IndexOptio
       process.exit(1);
     }
 
-    console.log(`👉 Found ${pages.length} URLs in ${sitemaps.length} sitemap`);
-
     pages = pagesFromSitemaps;
+
+    console.log(`👉 Found ${pages.length} URLs in ${sitemaps.length} sitemap`);
+  } else {
+    pages = checkCustomUrls(siteUrl, pages);
+    console.log(`👉 Found ${pages.length} URLs in the provided list`);
   }
 
   const statusPerUrl: Record<string, { status: Status; lastCheckedAt: string }> = existsSync(cachePath)

src/shared/gsc.ts

@@ -121,6 +121,35 @@ export async function checkSiteUrl(accessToken: string, siteUrl: string) {
   process.exit(1);
 }
 
+/**
+ * Checks if the given URLs are valid.
+ * @param siteUrl - The URL of the site.
+ * @param urls - The URLs to check.
+ * @returns An array containing the corrected URLs if found, otherwise the original URLs
+ */
+export function checkCustomUrls(siteUrl: string, urls: string[]) {
+  const protocol = siteUrl.startsWith("http://") ? "http://" : "https://";
+  const domain = siteUrl.replace("https://", "").replace("http://", "").replace("sc-domain:", "");
+  const formattedUrls: string[] = urls.map((url) =>{
+    url = url.trim();
+    if (url.startsWith("/")) {
+      // the url is a relative path (e.g. /about)
+      return `${protocol}${domain}${url}`;
+    } else if (url.startsWith("http://") || url.startsWith("https://")) {
+      // the url is already a full url (e.g. https://domain.com/about)
+      return url;
+    } else if (url.startsWith(domain)) {
+      // the url is a full url without the protocol (e.g. domain.com/about)
+      return `${protocol}${url}`;
+    } else {
+      // the url is a relative path without the leading slash (e.g. about)
+      return `${protocol}${domain}/${url}`
+    }
+  });
+
+  return formattedUrls;
+}
+
 /**
  * Retrieves the indexing status of a page.
  * @param accessToken - The access token for authentication.