Implement retry for likely transient errors

Author: Ndpnt

src/archivist/errors.js

@@ -1,9 +1,11 @@
 export class InaccessibleContentError extends Error {
-  constructor(reasonOrReasons) {
-    const reasons = [].concat(reasonOrReasons);
+  constructor(errorOrErrors) {
+    const errors = [].concat(errorOrErrors);
+    const reasons = errors.map(error => (error instanceof Error ? error.message : error));
 
     super(`The documents cannot be accessed or their contents can not be selected:${`\n - ${reasons.join('\n - ')}`}`);
     this.name = 'InaccessibleContentError';
     this.reasons = reasons;
+    this.errors = errors;
   }
 }

src/archivist/index.js

@@ -29,6 +29,7 @@ export const EVENTS = [
   'trackingCompleted',
   'inaccessibleContent',
   'info',
+  'warn',
   'error',
   'pluginError',
 ];
@@ -76,15 +77,35 @@ export default class Archivist extends events.EventEmitter {
 
   initQueue() {
     this.trackingQueue = async.queue(this.trackTermsChanges.bind(this), MAX_PARALLEL_TRACKING);
-    this.trackingQueue.error((error, { terms }) => {
-      if (error instanceof InaccessibleContentError) {
-        this.emit('inaccessibleContent', error, terms);
+    this.trackingQueue.error(this.handleTrackingError.bind(this));
+  }
 
-        return;
-      }
+  handleTrackingError(error, { terms, isRetry }) {
+    if (!(error instanceof InaccessibleContentError)) {
+      this.emit('error', {
+        message: error.stack,
+        serviceId: terms.service.id,
+        termsType: terms.type,
+      });
 
-      this.emit('error', error, terms);
-    });
+      return;
+    }
+
+    const isErrorLikelyTransient = error.errors.some(err => err instanceof FetchDocumentError && err.mayBeTransient);
+
+    if (isErrorLikelyTransient && !isRetry) {
+      this.emit('warn', {
+        message: `The documents cannot be accessed due to the following likely transient errors:\n- ${error.errors.map(err => err.message).join('\n- ')}\nRetry later…`,
+        serviceId: terms.service.id,
+        termsType: terms.type,
+      });
+
+      this.trackingQueue.push({ terms, isRetry: true });
+
+      return;
+    }
+
+    this.emit('inaccessibleContent', error, terms);
   }
 
   attach(listener) {
@@ -171,7 +192,7 @@ export default class Archivist extends events.EventEmitter {
           throw error;
         }
 
-        fetchDocumentErrors.push(error.message);
+        fetchDocumentErrors.push(error);
       }
     }));
 
@@ -206,7 +227,7 @@ export default class Archivist extends events.EventEmitter {
           throw error;
         }
 
-        extractDocumentErrors.push(error.message);
+        extractDocumentErrors.push(error);
       }
     }));
 

src/archivist/index.test.js

@@ -8,6 +8,8 @@ import nock from 'nock';
 import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
 
+import { InaccessibleContentError } from './errors.js';
+import { FetchDocumentError } from './fetcher/index.js';
 import Git from './recorder/repositories/git/git.js';
 
 import Archivist, { EVENTS } from './index.js';
@@ -245,6 +247,100 @@ describe('Archivist', function () {
     });
   });
 
+  describe('#handleTrackingError', () => {
+    let errorSpy;
+    let warnSpy;
+    let inaccessibleContentSpy;
+    let pushSpy;
+    let terms;
+    let app;
+    const retryableError = new FetchDocumentError(FetchDocumentError.LIKELY_TRANSIENT_ERRORS[0]);
+
+    before(async () => {
+      app = new Archivist({
+        recorderConfig: config.get('@opentermsarchive/engine.recorder'),
+        fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
+      });
+      await app.initialize();
+    });
+
+    beforeEach(() => {
+      errorSpy = sinon.spy();
+      warnSpy = sinon.spy();
+      inaccessibleContentSpy = sinon.spy();
+      pushSpy = sinon.spy(app.trackingQueue, 'push');
+      app.on('error', errorSpy);
+      app.on('warn', warnSpy);
+      app.on('inaccessibleContent', inaccessibleContentSpy);
+
+      terms = {
+        service: { id: 'test-service' },
+        type: 'test-type',
+        sourceDocuments: [
+          { location: 'https://example.com/doc1' },
+          { location: 'https://example.com/doc2' },
+        ],
+      };
+    });
+
+    afterEach(() => {
+      errorSpy.resetHistory();
+      warnSpy.resetHistory();
+      inaccessibleContentSpy.resetHistory();
+      pushSpy.restore();
+    });
+
+    context('with an InaccessibleContentError', () => {
+      context('when error may be transient', () => {
+        beforeEach(() => {
+          const error = new InaccessibleContentError([retryableError]);
+
+          app.handleTrackingError(error, { terms });
+        });
+
+        it('does not emit an error event', () => {
+          expect(errorSpy).to.not.have.been.called;
+        });
+
+        it('does not emit an inaccessibleContent event', () => {
+          expect(inaccessibleContentSpy).to.not.have.been.called;
+        });
+
+        it('emits a warning', () => {
+          expect(warnSpy).to.have.been.called;
+        });
+
+        it('pushes terms to tracking queue for retry', () => {
+          expect(pushSpy).to.have.been.calledWith({ terms, isRetry: true });
+        });
+      });
+
+      context('when error comes from a retry', () => {
+        beforeEach(() => {
+          const error = new InaccessibleContentError([retryableError]);
+
+          app.handleTrackingError(error, { terms, isRetry: true });
+        });
+
+        it('does not emit an error event', () => {
+          expect(errorSpy).to.not.have.been.called;
+        });
+
+        it('does not emit a warning', () => {
+          expect(warnSpy).to.not.have.been.called;
+        });
+
+        it('emits an inaccessibleContent event with error and terms', () => {
+          expect(inaccessibleContentSpy).to.have.been.called;
+        });
+
+        it('does not push terms to tracking queue for retry', () => {
+          expect(pushSpy).to.not.have.been.called;
+        });
+      });
+    });
+  });
+
   describe('Plugin system', () => {
     const plugin = {};