Merge pull request #65 from ArkeeAgency/antoinekm/force-reindex

add custom urls option

Author: goenning

.changeset/new-rivers-tell.md

@@ -0,0 +1,5 @@
+---
+"google-indexing-script": patch
+---
+
+Fix siteUrls convertions

.changeset/pretty-masks-raise.md

@@ -0,0 +1,5 @@
+---
+"google-indexing-script": minor
+---
+
+Add custom URLs option

README.md

@@ -131,7 +131,7 @@ index("seogets.com", {
   .catch(console.error);
 ```
 
-Read the [API documentation](https://paka.dev/npm/google-indexing-script) for more details.
+Read the [API documentation](https://jsdocs.io/package/google-indexing-script) for more details.
 
 </details>
 

src/index.ts

@@ -7,6 +7,7 @@ import {
   getPageIndexingStatus,
   convertToFilePath,
   checkSiteUrl,
+  checkCustomUrls,
 } from "./shared/gsc";
 import { getSitemapPages } from "./shared/sitemap";
 import { Status } from "./shared/types";
@@ -26,6 +27,7 @@ export type IndexOptions = {
   client_email?: string;
   private_key?: string;
   path?: string;
+  urls?: string[];
   quota?: {
     rpmRetry?: boolean; // read requests per minute: retry after waiting time
   };
@@ -53,6 +55,9 @@ export const index = async (input: string = process.argv[2], options: IndexOptio
   if (!options.path) {
     options.path = args["path"] || process.env.GIS_PATH;
   }
+  if (!options.urls) {
+    options.urls = args["urls"] ? args["urls"].split(",") : undefined;
+  }
   if (!options.quota) {
     options.quota = {
       rpmRetry: args["rpm-retry"] === "true" || process.env.GIS_QUOTA_RPM_RETRY === "true",
@@ -72,15 +77,24 @@ export const index = async (input: string = process.argv[2], options: IndexOptio
 
   siteUrl = await checkSiteUrl(accessToken, siteUrl);
 
-  const [sitemaps, pages] = await getSitemapPages(accessToken, siteUrl);
+  let pages = options.urls || [];
+  if (pages.length === 0) {
+    console.log(`🔎 Fetching sitemaps and pages...`);
+    const [sitemaps, pagesFromSitemaps] = await getSitemapPages(accessToken, siteUrl);
 
-  if (sitemaps.length === 0) {
-    console.error("❌ No sitemaps found, add them to Google Search Console and try again.");
-    console.error("");
-    process.exit(1);
-  }
+    if (sitemaps.length === 0) {
+      console.error("❌ No sitemaps found, add them to Google Search Console and try again.");
+      console.error("");
+      process.exit(1);
+    }
+
+    pages = pagesFromSitemaps;
 
-  console.log(`👉 Found ${pages.length} URLs in ${sitemaps.length} sitemap`);
+    console.log(`👉 Found ${pages.length} URLs in ${sitemaps.length} sitemap`);
+  } else {
+    pages = checkCustomUrls(siteUrl, pages);
+    console.log(`👉 Found ${pages.length} URLs in the provided list`);
+  }
 
   const statusPerUrl: Record<string, { status: Status; lastCheckedAt: string }> = existsSync(cachePath)
     ? JSON.parse(readFileSync(cachePath, "utf8"))
@@ -109,7 +123,7 @@ export const index = async (input: string = process.argv[2], options: IndexOptio
   const shouldRecheck = (status: Status, lastCheckedAt: string) => {
     const shouldIndexIt = indexableStatuses.includes(status);
     const isOld = new Date(lastCheckedAt) < new Date(Date.now() - CACHE_TIMEOUT);
-    return shouldIndexIt && isOld;;
+    return shouldIndexIt && isOld;
   };
 
   await batch(

src/shared/gsc.ts

@@ -92,11 +92,11 @@ export async function checkSiteUrl(accessToken: string, siteUrl: string) {
   // Convert the site URL into all possible formats
   if (siteUrl.startsWith("https://")) {
     formattedUrls.push(siteUrl);
-    formattedUrls.push(convertToHTTP(siteUrl));
+    formattedUrls.push(convertToHTTP(siteUrl.replace("https://", "")));
     formattedUrls.push(convertToSCDomain(siteUrl));
   } else if (siteUrl.startsWith("http://")) {
     formattedUrls.push(siteUrl);
-    formattedUrls.push(convertToHTTPS(siteUrl));
+    formattedUrls.push(convertToHTTPS(siteUrl.replace("http://", "")));
     formattedUrls.push(convertToSCDomain(siteUrl));
   } else if (siteUrl.startsWith("sc-domain:")) {
     formattedUrls.push(siteUrl);
@@ -121,6 +121,35 @@ export async function checkSiteUrl(accessToken: string, siteUrl: string) {
   process.exit(1);
 }
 
+/**
+ * Checks if the given URLs are valid.
+ * @param siteUrl - The URL of the site.
+ * @param urls - The URLs to check.
+ * @returns An array containing the corrected URLs if found, otherwise the original URLs
+ */
+export function checkCustomUrls(siteUrl: string, urls: string[]) {
+  const protocol = siteUrl.startsWith("http://") ? "http://" : "https://";
+  const domain = siteUrl.replace("https://", "").replace("http://", "").replace("sc-domain:", "");
+  const formattedUrls: string[] = urls.map((url) => {
+    url = url.trim();
+    if (url.startsWith("/")) {
+      // the url is a relative path (e.g. /about)
+      return `${protocol}${domain}${url}`;
+    } else if (url.startsWith("http://") || url.startsWith("https://")) {
+      // the url is already a full url (e.g. https://domain.com/about)
+      return url;
+    } else if (url.startsWith(domain)) {
+      // the url is a full url without the protocol (e.g. domain.com/about)
+      return `${protocol}${url}`;
+    } else {
+      // the url is a relative path without the leading slash (e.g. about)
+      return `${protocol}${domain}/${url}`;
+    }
+  });
+
+  return formattedUrls;
+}
+
 /**
  * Retrieves the indexing status of a page.
  * @param accessToken - The access token for authentication.

tsconfig.json

@@ -7,7 +7,7 @@
     "sourceMap": true,
     "outDir": "./dist",
     "moduleResolution": "node",
-    "removeComments": true,
+    "removeComments": false,
     "noImplicitAny": false,
     "strictNullChecks": true,
     "strictFunctionTypes": true,