feat(confluence): API retries with exponential backoff (#262)

This PR introduces retries with exponential backoff to the Confluence API.

Author: benjidotsh

workspaces/confluence/.changeset/plenty-radios-act.md

@@ -0,0 +1,5 @@
+---
+'@philips-software/backstage-plugin-search-confluence-backend': minor
+---
+
+Added API retries with exponential backoff

workspaces/confluence/plugins/search-confluence-backend/README.md

@@ -29,12 +29,17 @@ confluence:
   auth:
     token: ${Your PAT Token}
   category:
-# provide the list of categories you want to use to find spaces that will be indexed
-# example
-# - space1
-# - space2
+    # provide the list of categories you want to use to find spaces that will be indexed
+    # example
+    # - space1
+    # - space2
+  retries:
+    attempts: 3 # optional, defaults to 3
+    delay: 5000 # optional, defaults to 5000
 ```
 
+> Note: if the Confluence API rate limit is exceeded, the plugin will automatically retry with increasing wait times for a specified number of retry attempts. This can increase the duration of the task. Take this into account when setting the `schedule.frequency` and `schedule.timeout` values.
+
 ## Backend Configuration (Follows new backend)
 
 Add the collator to your backend instance, along with the search plugin itself

workspaces/confluence/plugins/search-confluence-backend/config.d.ts

@@ -46,5 +46,24 @@ export interface Config {
      * @visibility backend
      */
     category: string[];
+
+    /**
+     * @visibility backend
+     */
+    retries?: {
+      /**
+       * Number of attempts to retry.
+       * Defaults to 3.
+       * @visibility backend
+       */
+      attempts?: string;
+
+      /**
+       * Minimum delay between retries in milliseconds.
+       * Defaults to 5000.
+       * @visibility backend
+       */
+      delay?: string;
+    };
   };
 }

workspaces/confluence/plugins/search-confluence-backend/package.json

@@ -36,9 +36,11 @@
     "@backstage/backend-plugin-api": "backstage:^",
     "@backstage/backend-tasks": "^0.6.1",
     "@backstage/config": "backstage:^",
+    "@backstage/errors": "backstage:^",
     "@backstage/plugin-search-backend-node": "backstage:^",
     "@backstage/plugin-search-common": "backstage:^",
     "p-limit": "^3.1.0",
+    "retry": "^0.13.1",
     "uuid": "^9.0.1",
     "winston": "^3.2.1"
   },
@@ -47,6 +49,7 @@
     "@backstage/plugin-auth-backend": "backstage:^",
     "@backstage/plugin-auth-backend-module-guest-provider": "backstage:^",
     "@backstage/types": "backstage:^",
+    "@types/retry": "^0.12.5",
     "@types/supertest": "^6.0.2",
     "@types/uuid": "^9.0.8",
     "jest-fetch-mock": "^3.0.3",

…Factory/ConflunceCollaterFactory.test.ts …actory/ConfluenceCollaterFactory.test.tsworkspaces/confluence/plugins/search-confluence-backend/src/search/ConfluenceCollatorFactory/ConflunceCollaterFactory.test.ts renamed to workspaces/confluence/plugins/search-confluence-backend/src/search/ConfluenceCollatorFactory/ConfluenceCollaterFactory.test.ts workspaces/confluence/plugins/search-confluence-backend/src/search/ConfluenceCollatorFactory/ConflunceCollaterFactory.test.ts renamed to workspaces/confluence/plugins/search-confluence-backend/src/search/ConfluenceCollatorFactory/ConfluenceCollaterFactory.test.ts

@@ -2,6 +2,7 @@ import { ConfluenceCollatorFactory } from './ConfluenceCollatorFactory';
 import { ConfigReader } from '@backstage/config';
 import { getVoidLogger } from '@backstage/backend-common';
 import fetch from 'jest-fetch-mock';
+import retry from 'retry';
 
 const createConfig = new ConfigReader({
   confluence: {
@@ -274,6 +275,28 @@ describe('Testing ConfluenceCollatorFactory', () => {
     ).rejects.toThrow('Not Found');
   });
 
+  it('should retry for a rate limit error response', async () => {
+    const retryMock = jest.fn();
+    jest.spyOn(retry, 'operation').mockReturnValue({
+      ...retry.operation(),
+      attempt: fn => fn(1),
+      retry: retryMock,
+    });
+
+    const confluenceCollatorFactory = ConfluenceCollatorFactory.fromConfig(
+      createConfig,
+      { logger },
+    );
+
+    fetch.mockResponseOnce('Too Many Requests', { status: 429 });
+
+    await expect(
+      confluenceCollatorFactory.get('https://example.com'),
+    ).rejects.toThrow('Too Many Requests');
+
+    expect(retryMock).toHaveBeenCalled();
+  });
+
   it('should return Bearer token if token is present', () => {
     const confluenceCollatorFactory = ConfluenceCollatorFactory.fromConfig(
       createTokenConfig,

workspaces/confluence/plugins/search-confluence-backend/src/search/ConfluenceCollatorFactory/ConfluenceCollatorFactory.ts

@@ -12,8 +12,11 @@ import {
   IndexableAncestorRef,
   IndexableConfluenceDocument,
   ConfluenceInstanceConfig,
+  ConfluenceRetryConfig,
 } from './types';
 import { v4 as uuidv4 } from 'uuid';
+import retry from 'retry';
+import { ResponseError } from '@backstage/errors';
 
 type ConfluenceCollatorOptions = {
   logger: Logger;
@@ -27,6 +30,7 @@ type ConfluenceCollatorOptions = {
     token?: string;
   };
   category: string[];
+  retries?: ConfluenceRetryConfig;
 };
 
 export const confluenceDefaultSchedule = {
@@ -49,6 +53,7 @@ export class ConfluenceCollatorFactory implements DocumentCollatorFactory {
   private parallelismLimit: number;
   private wikiUrl: string;
   private auth: { username?: string; password?: string; token?: string };
+  private retries?: ConfluenceRetryConfig;
   public category: string[];
   static fromConfig(
     config: Config,
@@ -73,6 +78,7 @@ export class ConfluenceCollatorFactory implements DocumentCollatorFactory {
       wikiUrl: confluenceOptions.wikiUrl,
       auth: auth,
       category: confluenceOptions.category,
+      retries: confluenceOptions.retries,
     });
   }
 
@@ -83,6 +89,7 @@ export class ConfluenceCollatorFactory implements DocumentCollatorFactory {
     this.wikiUrl = options.wikiUrl;
     this.auth = options.auth;
     this.category = options.category;
+    this.retries = options.retries;
   }
 
   async getCollator() {
@@ -282,24 +289,45 @@ export class ConfluenceCollatorFactory implements DocumentCollatorFactory {
 
   async get<T = any>(requestUrl: string): Promise<T> {
     const Authorization = this.getAuthorizationHeader();
-    const res = await fetch(requestUrl, {
-      method: 'get',
-      headers: {
-        Authorization,
-      },
-    });
 
-    if (!res.ok) {
-      this.logger.warn(
-        'non-ok response from confluence',
-        requestUrl,
-        res.status,
-        await res.text(),
-      );
+    return new Promise<T>((resolve, reject) => {
+      const operationRetry = retry.operation({
+        retries: this.retries?.attempts ?? 3,
+        minTimeout: this.retries?.delay ?? 5000,
+      });
 
-      throw new Error(`Request failed with ${res.status} ${res.statusText}`);
-    }
+      operationRetry.attempt(async attempt => {
+        try {
+          const res = await fetch(requestUrl, {
+            method: 'get',
+            headers: {
+              Authorization,
+            },
+          });
+
+          if (!res.ok) {
+            this.logger.warn(
+              `non-ok response from confluence (attempt ${attempt})`,
+              requestUrl,
+              res.status,
+              await res.text(),
+            );
+
+            throw await ResponseError.fromResponse(res);
+          }
+
+          resolve(await res.json());
+        } catch (e) {
+          if (
+            e instanceof ResponseError &&
+            e.statusCode === 429 &&
+            operationRetry.retry(e)
+          )
+            return;
 
-    return await res.json();
+          reject(e);
+        }
+      });
+    });
   }
 }

workspaces/confluence/plugins/search-confluence-backend/src/search/ConfluenceCollatorFactory/types.ts

@@ -4,6 +4,12 @@ export type ConfluenceInstanceConfig = {
   wikiUrl: string;
   auth: { username?: string; password?: string; token?: string };
   category: string[];
+  retries?: ConfluenceRetryConfig;
+};
+
+export type ConfluenceRetryConfig = {
+  attempts?: number;
+  delay?: number;
 };
 
 export type ConfluenceDocumentMetadata = {

workspaces/confluence/yarn.lock

@@ -3289,7 +3289,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@backstage/errors@npm:^1.2.4":
+"@backstage/errors@backstage:1.32.3, @backstage/errors@npm:^1.2.4":
   version: 1.2.4
   resolution: "@backstage/errors@npm:1.2.4"
   dependencies:
@@ -6791,16 +6791,19 @@ __metadata:
     "@backstage/backend-tasks": "npm:^0.6.1"
     "@backstage/cli": "backstage:^"
     "@backstage/config": "backstage:^"
+    "@backstage/errors": "backstage:^"
     "@backstage/plugin-auth-backend": "backstage:^"
     "@backstage/plugin-auth-backend-module-guest-provider": "backstage:^"
     "@backstage/plugin-search-backend-node": "backstage:^"
     "@backstage/plugin-search-common": "backstage:^"
     "@backstage/types": "backstage:^"
+    "@types/retry": "npm:^0.12.5"
     "@types/supertest": "npm:^6.0.2"
     "@types/uuid": "npm:^9.0.8"
     jest-fetch-mock: "npm:^3.0.3"
     msw: "npm:^1.0.0"
     p-limit: "npm:^3.1.0"
+    retry: "npm:^0.13.1"
     uuid: "npm:^9.0.1"
     winston: "npm:^3.2.1"
   languageName: unknown
@@ -9273,6 +9276,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/retry@npm:^0.12.5":
+  version: 0.12.5
+  resolution: "@types/retry@npm:0.12.5"
+  checksum: 10/3fb6bf91835ca0eb2987567d6977585235a7567f8aeb38b34a8bb7bbee57ac050ed6f04b9998cda29701b8c893f5dfe315869bc54ac17e536c9235637fe351a2
+  languageName: node
+  linkType: hard
+
 "@types/sarif@npm:^2.1.4":
   version: 2.1.7
   resolution: "@types/sarif@npm:2.1.7"