Configure IAM token service retry behaviour.

Author: smithsz

README.md

@@ -355,7 +355,7 @@ All other configuration is plugin specific. It must be passed within an object
 to the `plugins` parameter in the client constructor. For example:
 
 ```js
-var cloudant = new Cloudant({ url: myurl, maxAttempt: 5, plugins: [ 'iamauth', { retry: { retryDelayMultiplier: 4 } } ]);
+var cloudant = new Cloudant({ url: myurl, maxAttempt: 5, plugins: [ { iamauth: { iamApiKey: 'abcxyz' } }, { retry: { retryDelayMultiplier: 4 } } ]);
 ```
 
 `maxAttempt` can _not_ be overridden by plugin specific configuration.
@@ -397,9 +397,31 @@ var cloudant = new Cloudant({ url: myurl, maxAttempt: 5, plugins: [ 'iamauth', {
    override this. To authenticate with the IAM token service set `iamClientId`
    and `iamClientSecret` in your plugin configuration.
 
+   The plugin will retry failed requests to the token service (specifically
+   `429` and `5xx` responses) until the number of retry requests reaches
+   `maxAttempt`. Be aware that retrying requests to the token service delays the
+   client request. It also increases the number of token exchange attempts and
+   therefore may result in rate limiting by the IAM token service.
+
+   The retry behavior can be configured using the following options:
+
+    - `retryDelayMultiplier`
+
+      The multiplication factor used for increasing the timeout after each
+      subsequent attempt _(default: 2)_.
+
+    - `retryInitialDelayMsecs`
+
+      The initial retry delay in milliseconds _(default: 500)_.
+
+   If the IAM token cannot be retrieved (either because the IAM token service is
+   down or the IAM API key is incorrect) then the request will continue without
+   IAM authentication allowing the database to return a `401` response to the
+   caller so that it may be handled appropriately.
+
    For example:
    ```js
-   var cloudant = new Cloudant({ url: 'https://examples.cloudant.com', plugins: { iamauth: { iamApiKey: 'xxxxxxxxxx' } } });
+   var cloudant = new Cloudant({ url: 'https://examples.cloudant.com', plugins: { iamauth: { iamApiKey: 'xxxxxxxxxx', retryDelayMultiplier: 4 } } });
    ```
 
    See [IBM Cloud Identity and Access Management](https://console.bluemix.net/docs/services/Cloudant/guides/iam.html#ibm-cloud-identity-and-access-management) for more information.

lib/client.js

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017, 2019 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -312,21 +312,19 @@ class CloudantClient {
       request.state.sending = false;
 
       debug(`Request attempt: ${request.state.attempt}`);
-
-      utils.runHooks('onRequest', request, request.options, function(err) {
-        utils.processState(request, function(stop) {
-          if (request.state.retry) {
-            debug('The onRequest hook issued retry.');
-            return done();
-          }
-          if (stop) {
-            debug(`The onRequest hook issued abort: ${stop}`);
-            return done(stop);
-          }
-
-          debug(`Delaying request for ${request.state.retryDelayMsecs} Msecs.`);
-
-          setTimeout(function() {
+      debug(`Delaying request for ${request.state.retryDelayMsecs} Msecs.`);
+
+      setTimeout(function() {
+        utils.runHooks('onRequest', request, request.options, function(err) {
+          utils.processState(request, function(stop) {
+            if (request.state.retry) {
+              debug('The onRequest hook issued retry.');
+              return done();
+            }
+            if (stop) {
+              debug(`The onRequest hook issued abort: ${stop}`);
+              return done(stop);
+            }
             if (request.abort) {
               debug('Client issued abort during plugin execution.');
               return done(new Error('Client issued abort'));
@@ -354,9 +352,9 @@ class CloudantClient {
                   .pipe(concatStream);
               }
             }
-          }, request.state.retryDelayMsecs);
+          });
         });
-      });
+      }, request.state.retryDelayMsecs);
     }, function(err) { debug(err.message); });
 
     return request.clientStream; // return stream to client

plugins/iamauth.js

@@ -25,31 +25,29 @@ const BasePlugin = require('./base.js');
  */
 class IAMPlugin extends BasePlugin {
   constructor(client, cfg) {
-    super(client, cfg);
+    if (typeof cfg.iamApiKey === 'undefined') {
+      throw new Error('Missing IAM API key from configuration');
+    }
 
-    var self = this;
-    self.iamApiKey = null;
-    self.baseUrl = cfg.baseUrl || null;
-    self.cookieJar = request.jar();
-    self.tokenUrl = cfg.iamTokenUrl || 'https://iam.cloud.ibm.com/identity/token';
+    // token service retry configuration
+    cfg = Object.assign({
+      retryDelayMultiplier: 2,
+      retryInitialDelayMsecs: 500
+    }, cfg);
 
-    // Specifies whether IAM authentication should be applied to the request being intercepted.
-    self.shouldApplyIAMAuth = true;
-    self.refreshRequired = true;
+    super(client, cfg);
 
-    if (typeof cfg.iamApiKey === 'undefined') {
-      debug('Missing IAM API key. Skipping IAM authentication.');
-      self.shouldApplyIAMAuth = false;
-    }
+    this.iamApiKey = null;
+    this.baseUrl = cfg.baseUrl || null;
+    this.cookieJar = request.jar();
+    this.tokenUrl = cfg.iamTokenUrl || 'https://iam.cloud.ibm.com/identity/token';
+    this.shouldApplyIAMAuth = true;
+    this.refreshRequired = true;
   }
 
   onRequest(state, req, callback) {
     var self = this;
 
-    if (typeof self._cfg.iamApiKey === 'undefined') {
-      throw new Error('Missing IAM API key from configuration');
-    }
-
     if (self._cfg.iamApiKey !== self.iamApiKey) {
       debug('New credentials identified. Renewing session cookie...');
       self.shouldApplyIAMAuth = self.refreshRequired = true;
@@ -85,6 +83,13 @@ class IAMPlugin extends BasePlugin {
         debug(error.message);
         if (self.shouldApplyIAMAuth) {
           state.retry = true;
+          if (state.attempt === 1) {
+            state.retryDelayMsecs = self._cfg.retryInitialDelayMsecs;
+          } else {
+            state.retryDelayMsecs *= self._cfg.retryDelayMultiplier;
+          }
+        } else {
+          console.warn(`Disabling IAM authentication: ${error.message}`);
         }
       } else {
         req.jar = self.cookieJar; // add jar

test/plugins/iamauth.js

@@ -513,10 +513,10 @@ describe('#db IAMAuth Plugin', function() {
   });
 
   it('throws error for unspecified IAM API key', function() {
-    var cloudantClient = new Client({ plugins: 'iamauth' });
     assert.throws(
       () => {
-        cloudantClient.request({ url: SERVER + DBNAME });
+        /* eslint-disable no-new */
+        new Client({ plugins: 'iamauth' });
       },
       /Missing IAM API key from configuration/,
       'did not throw with expected message'
@@ -557,4 +557,51 @@ describe('#db IAMAuth Plugin', function() {
       assert.fail(`Unexpected reject: ${err}`);
     });
   });
+
+  it('successfully retries request on 500 IAM token service response and returns 200 response', function(done) {
+    if (process.env.NOCK_OFF) {
+      this.skip();
+    }
+
+    var iamMocks = nock(TOKEN_SERVER)
+      .post('/identity/token', {
+        'grant_type': 'urn:ibm:params:oauth:grant-type:apikey',
+        'response_type': 'cloud_iam',
+        'apikey': IAM_API_KEY
+      })
+      .times(2)
+      .reply(500, {error: 'internal_server_error', reason: 'Internal Server Error'})
+      .post('/identity/token', {
+        'grant_type': 'urn:ibm:params:oauth:grant-type:apikey',
+        'response_type': 'cloud_iam',
+        'apikey': IAM_API_KEY
+      })
+      .reply(200, MOCK_IAM_TOKEN_RESPONSE);
+
+    var cloudantMocks = nock(SERVER)
+      .post('/_iam_session', {access_token: MOCK_ACCESS_TOKEN})
+      .reply(200, {ok: true}, MOCK_SET_IAM_SESSION_HEADER)
+      .get(DBNAME)
+      .reply(200, {doc_count: 0});
+
+    var cloudantClient = new Client({ maxAttempt: 3, plugins: { iamauth: { iamApiKey: IAM_API_KEY } } });
+    var req = { url: SERVER + DBNAME, method: 'GET' };
+
+    var startTs = (new Date()).getTime();
+
+    cloudantClient.request(req, function(err, resp, data) {
+      assert.equal(err, null);
+      assert.equal(resp.request.headers.cookie, MOCK_IAM_SESSION);
+      assert.equal(resp.statusCode, 200);
+      assert.ok(data.indexOf('"doc_count":0') > -1);
+
+      // validate retry delay
+      var now = (new Date()).getTime();
+      assert.ok(now - startTs > (500 + 1000));
+
+      iamMocks.done();
+      cloudantMocks.done();
+      done();
+    });
+  });
 });

test/readmeexamples.js

@@ -1,4 +1,4 @@
-// Copyright © 2018 IBM Corp. All rights reserved.
+// Copyright © 2018, 2019 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -260,10 +260,11 @@ describe('#db README Examples', function() {
       var cloudant = new Cloudant({
         url: `https://${ME}.cloudant.com`,
         maxAttempt: 5,
-        plugins: [ 'iamauth', { retry: { retryDelayMultiplier: 4 } } ]
+        plugins: [ { iamauth: { iamApiKey: 'abcxyz' } }, { retry: { retryDelayMultiplier: 4 } } ]
       });
       assert.equal(cloudant.cc._plugins.length, 2);
       assert.equal(cloudant.cc._plugins[0].constructor.name, 'IAMPlugin');
+      assert.equal(cloudant.cc._plugins[0]._cfg.iamApiKey, 'abcxyz');
       assert.equal(cloudant.cc._plugins[1].constructor.name, 'RetryPlugin');
       assert.equal(cloudant.cc._plugins[1]._cfg.retryDelayMultiplier, 4);
     });