Merge remote-tracking branch 'upstream/main' into antoinekm/force-reindex

Author: antoinekm

.changeset/nervous-carrots-complain.md

@@ -1,5 +1,29 @@
 # google-indexing-script
 
+## 0.2.0
+
+### Minor Changes
+
+- [#62](https://github.com/goenning/google-indexing-script/pull/62) [`93dd956`](https://github.com/goenning/google-indexing-script/commit/93dd956dca4065b97d6076db772560fba57aec50) Thanks [@hasanafzal8485](https://github.com/hasanafzal8485)! - Don't want the same URL use my API limit again until his previous cache limit is completed
+
+## 0.1.0
+
+### Minor Changes
+
+- [#55](https://github.com/goenning/google-indexing-script/pull/55) [`908938a`](https://github.com/goenning/google-indexing-script/commit/908938a701d964b75331e322fbea8d77e6db976e) Thanks [@AntoineKM](https://github.com/AntoineKM)! - feat(get-publish-metadata): optional retries if rate limited
+
+## 0.0.5
+
+### Patch Changes
+
+- [#44](https://github.com/goenning/google-indexing-script/pull/44) [`77b94ed`](https://github.com/goenning/google-indexing-script/commit/77b94edeef863721c07bd3e12d6d38052723f422) Thanks [@AntoineKM](https://github.com/AntoineKM)! - Add site url checker
+
+## 0.0.4
+
+### Patch Changes
+
+- [#40](https://github.com/goenning/google-indexing-script/pull/40) [`074f2c7`](https://github.com/goenning/google-indexing-script/commit/074f2c7ebbafff3a03ebf07baf7b21922a98698d) Thanks [@AntoineKM](https://github.com/AntoineKM)! - Add documentation comments
+
 ## 0.0.3
 
 ### Patch Changes

.changeset/orange-badgers-accept.md

@@ -34,4 +34,10 @@ After making your changes, you can build the project with the following command:
 
 ```bash
 npm run build
-```
+```
+
+# Pull Request
+
+1. Make sure your code is formatted with `prettier`
+2. Make sure your code passes the tests
+3. Make sure you added the changes with `npm run changeset`

CHANGELOG.md

@@ -4,7 +4,8 @@ Use this script to get your entire site indexed on Google in less than 48 hours.
 
 You can read more about the motivation behind it and how it works in this blog post https://seogets.com/blog/google-indexing-script
 
-> [!IMPORTANT]  
+> [!IMPORTANT]
+>
 > 1. Indexing != Ranking. This will not help your page rank on Google, it'll just let Google know about the existence of your pages.
 > 2. This script uses [Google Indexing API](https://developers.google.com/search/apis/indexing-api/v3/quickstart). We do not recommend using this script on spam/low-quality content.
 
@@ -70,8 +71,6 @@ gis <domain or url>
 gis seogets.com
 ```
 
-When in doubt try both 😀
-
 Here are some other ways to run the script:
 
 ```bash
@@ -82,6 +81,7 @@ google-indexing-script seogets.com
 # cloned repository
 npm run index seogets.com
 ```
+
 </details>
 
 <details>
@@ -94,6 +94,7 @@ Run the script with the domain or url you want to index.
 ```bash
 GIS_CLIENT_EMAIL=your-client-email GIS_PRIVATE_KEY=your-private-key gis seogets.com
 ```
+
 </details>
 
 <details>
@@ -106,6 +107,7 @@ Once you have the values, run the script with the domain or url you want to inde
 ```bash
 gis seogets.com --client-email your-client-email --private-key your-private-key
 ```
+
 </details>
 
 <details>
@@ -118,29 +120,68 @@ npm i google-indexing-script
 ```
 
 ```javascript
-import { index } from 'google-indexing-script'
-import serviceAccount from './service_account.json'
+import { index } from "google-indexing-script";
+import serviceAccount from "./service_account.json";
 
-index('seogets.com', {
+index("seogets.com", {
   client_email: serviceAccount.client_email,
-  private_key: serviceAccount.private_key
+  private_key: serviceAccount.private_key,
 })
   .then(console.log)
-  .catch(console.error)
+  .catch(console.error);
 ```
 
 Read the [API documentation](https://paka.dev/npm/google-indexing-script) for more details.
+
 </details>
 
 Here's an example of what you should expect:
 
 ![](./output.png)
 
 > [!IMPORTANT]
+>
 > - Your site must have 1 or more sitemaps submitted to Google Search Console. Otherwise, the script will not be able to find the pages to index.
 > - You can run the script as many times as you want. It will only index the pages that are not already indexed.
 > - Sites with a large number of pages might take a while to index, be patient.
 
+## Quota
+
+Depending on your account several quotas are configured for the API (see [docs](https://developers.google.com/search/apis/indexing-api/v3/quota-pricing#quota)). By default the script exits as soon as the rate limit is exceeded. You can configure a retry mechanism for the read requests that apply on a per minute time frame.
+
+<details>
+<summary>With environment variables</summary>
+
+```bash
+export GIS_QUOTA_RPM_RETRY=true
+```
+
+</details>
+
+<details>
+<summary>As a npm module</summary>
+
+```javascript
+import { index } from 'google-indexing-script'
+import serviceAccount from './service_account.json'
+
+index('seogets.com', {
+  client_email: serviceAccount.client_email,
+  private_key: serviceAccount.private_key
+  quota: {
+    rpmRetry: true
+  }
+})
+  .then(console.log)
+  .catch(console.error)
+```
+
+</details>
+
+## 🔀 Alternative
+
+If you prefer a hands-free, and less technical solution, you can use a SaaS platform like [TagParrot](https://tagparrot.com/?via=goenning).
+
 ## 📄 License
 
 MIT License

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 {
   "name": "google-indexing-script",
-  "version": "0.0.3",
+  "version": "0.2.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "bin": {

README.md

@@ -15,23 +15,29 @@ import { readFileSync, existsSync, mkdirSync, writeFileSync } from "fs";
 import path from "path";
 
 const CACHE_TIMEOUT = 1000 * 60 * 60 * 24 * 14; // 14 days
+export const QUOTA = {
+  rpm: {
+    retries: 3,
+    waitingTime: 60000, // 1 minute
+  },
+};
 
 export type IndexOptions = {
   client_email?: string;
   private_key?: string;
   path?: string;
   urls?: string[];
+  quota?: {
+    rpmRetry?: boolean; // read requests per minute: retry after waiting time
+  };
 };
 
 /**
  * Indexes the specified domain or site URL.
  * @param input - The domain or site URL to index.
  * @param options - (Optional) Additional options for indexing.
  */
-export const index = async (
-  input: string = process.argv[2],
-  options: IndexOptions = {},
-) => {
+export const index = async (input: string = process.argv[2], options: IndexOptions = {}) => {
   if (!input) {
     console.error("❌ Please provide a domain or site URL as the first argument.");
     console.error("");
@@ -50,6 +56,10 @@ export const index = async (
   }
   if (!options.urls) {
     options.urls = args["urls"] ? args["urls"].split(",") : undefined;
+  if (!options.quota) {
+    options.quota = {
+      rpmRetry: args["rpm-retry"] === "true" || process.env.GIS_QUOTA_RPM_RETRY === "true",
+    };
   }
 
   const accessToken = await getAccessToken(options.client_email, options.private_key, options.path);
@@ -108,7 +118,7 @@ export const index = async (
   const shouldRecheck = (status: Status, lastCheckedAt: string) => {
     const shouldIndexIt = indexableStatuses.includes(status);
     const isOld = new Date(lastCheckedAt) < new Date(Date.now() - CACHE_TIMEOUT);
-    return shouldIndexIt || isOld;
+    return shouldIndexIt && isOld;;
   };
 
   await batch(
@@ -155,7 +165,9 @@ export const index = async (
 
   for (const url of indexablePages) {
     console.log(`📄 Processing url: ${url}`);
-    const status = await getPublishMetadata(accessToken, url);
+    const status = await getPublishMetadata(accessToken, url, {
+      retriesOnRateLimit: options.quota.rpmRetry ? QUOTA.rpm.retries : 0,
+    });
     if (status === 404) {
       await requestIndexing(accessToken, url);
       console.log("🚀 Indexing requested successfully. It may take a few days for Google to process it.");

package.json

@@ -1,4 +1,5 @@
 import { webmasters_v3 } from "googleapis";
+import { QUOTA } from "..";
 import { Status } from "./types";
 import { fetchRetry } from "./utils";
 
@@ -20,7 +21,7 @@ export function convertToSiteUrl(input: string) {
  * @returns The converted file path
  */
 export function convertToFilePath(path: string) {
-  return path.replace("http://", "http_").replace("https://", "https_").replace("/", "_");
+  return path.replace("http://", "http_").replace("https://", "https_").replaceAll("/", "_");
 }
 
 /**
@@ -202,9 +203,10 @@ export function getEmojiForStatus(status: Status) {
  * Retrieves metadata for publishing from the given URL.
  * @param accessToken - The access token for authentication.
  * @param url - The URL for which to retrieve metadata.
+ * @param options - The options for the request.
  * @returns The status of the request.
  */
-export async function getPublishMetadata(accessToken: string, url: string) {
+export async function getPublishMetadata(accessToken: string, url: string, options?: { retriesOnRateLimit: number }) {
   const response = await fetchRetry(
     `https://indexing.googleapis.com/v3/urlNotifications/metadata?url=${encodeURIComponent(url)}`,
     {
@@ -223,12 +225,23 @@ export async function getPublishMetadata(accessToken: string, url: string) {
   }
 
   if (response.status === 429) {
-    console.error("🚦 Rate limit exceeded, try again later.");
-    console.error("");
-    console.error("   Quota: https://developers.google.com/search/apis/indexing-api/v3/quota-pricing#quota");
-    console.error("   Usage: https://console.cloud.google.com/apis/enabled");
-    console.error("");
-    process.exit(1);
+    if (options?.retriesOnRateLimit && options?.retriesOnRateLimit > 0) {
+      const RPM_WATING_TIME = (QUOTA.rpm.retries - options.retriesOnRateLimit + 1) * QUOTA.rpm.waitingTime; // increase waiting time for each retry
+      console.log(
+        `🚦 Rate limit exceeded for read requests. Retries left: ${options.retriesOnRateLimit}. Waiting for ${
+          RPM_WATING_TIME / 1000
+        }sec.`
+      );
+      await new Promise((resolve) => setTimeout(resolve, RPM_WATING_TIME));
+      await getPublishMetadata(accessToken, url, { retriesOnRateLimit: options.retriesOnRateLimit - 1 });
+    } else {
+      console.error("🚦 Rate limit exceeded, try again later.");
+      console.error("");
+      console.error("   Quota: https://developers.google.com/search/apis/indexing-api/v3/quota-pricing#quota");
+      console.error("   Usage: https://console.cloud.google.com/apis/enabled");
+      console.error("");
+      process.exit(1);
+    }
   }
 
   if (response.status >= 500) {

src/index.ts

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "target": "esnext",
     "module": "commonjs",
-    "lib": ["dom", "es6", "es2017", "esnext.asynciterable"],
+    "lib": ["dom", "es6", "es2021", "esnext.asynciterable"],
     "skipLibCheck": true,
     "sourceMap": true,
     "outDir": "./dist",