1.5.0 (#184)

Author: glynnbird

README.md

@@ -223,6 +223,7 @@ If you are importing data into a CouchDB database that already contains data, an
 * COUCH_OVERWRITE - overwrite existing document revisions with supplied data
 * COUCH_PARALLELISM - the maximum number of HTTP requests to have in flight at any one time (default: 1)
 * COUCH_MAX_WPS - the maximum number of write API calls to make per second (rate limiting) (default: 0 - no rate limiting)
+* COUCH_RETRY - whether to retry requests which yield a 429 response (default: false)
 
 ## Command-line parameters
 
@@ -243,6 +244,7 @@ You can also configure `couchimport` and `couchexport` using command-line parame
 * `--parallelism` - the number of HTTP request to have in flight at any one time (default 1)
 * `--maxwps` - the maximum number of write API calls to make per second (default 0 - no rate limiting)
 * `--overwrite`/`-o` - overwrite existing document revisions with supplied data (default: false)
+* `--retry`/`-r` - whether to retry requests which yield a 429 response (default: false)
 
 e.g.
 

app.js

@@ -30,7 +30,7 @@ const importStream = function (rs, opts, callback) {
       headers.Authorization = 'Bearer ' + iamAccessToken
     }
 
-    const writer = require('./includes/writer.js')(opts.url, opts.database, opts.buffer, opts.parallelism, opts.ignorefields, opts.overwrite, opts.maxwps, headers)
+    const writer = require('./includes/writer.js')(opts.url, opts.database, opts.buffer, opts.parallelism, opts.ignorefields, opts.overwrite, opts.maxwps, opts.retry, headers)
     const transformer = require('./includes/transformer.js')(opts.transform, opts.meta)
     const JSONStream = require('JSONStream')
     if (opts.type === 'jsonl') {

includes/args.js

@@ -71,6 +71,12 @@ const parse = function () {
       describe: 'whether to overwrite existing revisions with supplied data',
       default: process.env.COUCH_OVERWRITE ? process.env.COUCH_OVERWRITE : false
     })
+    .option('retry', {
+      alias: 'r',
+      boolean: true,
+      describe: 'whether to retry requests that yield a HTTP 429 response',
+      default: process.env.COUCH_RETRY ? process.env.COUCH_RETRY : false
+    })
     .argv
 
   // load the transformation JavaScript

includes/writer.js

@@ -3,7 +3,7 @@ const qrate = require('qrate')
 const debug = require('debug')('couchimport')
 const axios = require('axios').default
 
-module.exports = function (couchURL, couchDatabase, bufferSize, parallelism, ignoreFields, overwrite, maxwps, headers) {
+module.exports = function (couchURL, couchDatabase, bufferSize, parallelism, ignoreFields, overwrite, maxwps, retry, headers) {
   const stream = require('stream')
 
   let buffer = []
@@ -47,7 +47,6 @@ module.exports = function (couchURL, couchDatabase, bufferSize, parallelism, ign
         headers: headers
       }
       const response = await axios(req)
-      
       const existingData = response.data
       // make lookup table between id-->rev
       const lookup = {}
@@ -111,6 +110,9 @@ module.exports = function (couchURL, couchDatabase, bufferSize, parallelism, ign
       statusCode = e.response ? e.response.status : e.code
       failed = payload.docs.length
       writer.emit('writeerror', e)
+      if (retry) {
+        q.push(payload)
+      }
     }
 
     // log response code
@@ -122,7 +124,7 @@ module.exports = function (couchURL, couchDatabase, bufferSize, parallelism, ign
 
     written += ok
     totalfailed += failed
-    const status = { documents: ok, failed: failed, total: written, totalfailed: totalfailed, statusCodes: errorCodes, latency: latency}
+    const status = { documents: ok, failed: failed, total: written, totalfailed: totalfailed, statusCodes: errorCodes, latency: latency }
     writer.emit('written', status)
     debug(JSON.stringify(status))
   }, parallelism, maxwps || undefined)

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "couchimport",
-  "version": "1.4.6",
+  "version": "1.5.0",
   "description": "CouchImport - command-line helper to bulk import/export data from CSV/TSV",
   "repository": "https://github.com/glynnbird/couchimport.git",
   "keywords": [