Fix import

Fixes #461. Fixes #496.

Author: pluma

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Changed
+
+- Reimplemented `collection.import`
+
+  The previous implementation was broken. The new implementation should be backwards-compatible
+  in cases where it previously wasn't broken but is more flexible and also handles buffers.
+
 ### Fixed
 
 - Added missing dependency on `@types/node` ([#567](https://github.com/arangodb/arangojs/issues/567))

docs/Drivers/JS/Reference/Collection/BulkImport.md

@@ -11,90 +11,140 @@ Bulk imports the given _data_ into the collection.
 
 **Arguments**
 
-* **data**: `Array<Array<any>> | Array<Object>`
+- **data**: `Array | Buffer | string`
 
-  The data to import. This can be an array of documents:
+  The data to import. Depending on the _type_ option this can be any of the
+  following:
 
-  ```js
-  [
-    {key1: value1, key2: value2}, // document 1
-    {key1: value1, key2: value2}, // document 2
-    ...
-  ]
-  ```
+  For type `"documents"` or `"auto"`:
 
-  Or it can be an array of value arrays following an array of keys.
+  - an array of documents, e.g.
 
-  ```js
-  [
-    ['key1', 'key2'], // key names
-    [value1, value2], // document 1
-    [value1, value2], // document 2
-    ...
-  ]
-  ```
-
-* **opts**: `Object` (optional) If _opts_ is set, it must be an object with any
+    ```json
+    [
+      { "_key": "banana", "color": "yellow" },
+      { "_key": "peach", "color": "pink" }
+    ]
+    ```
+
+  - a string or buffer containing one JSON document per line, e.g.
+
+    ```
+    {"_key":"banana","color":"yellow"}
+    {"_key":"peach","color":"pink"}
+    ```
+
+  For type `"array"` or `"auto"`:
+
+  - a string or buffer containing a JSON array of documents, e.g.
+
+    ```json
+    [
+      { "_key": "banana", "color": "yellow" },
+      { "_key": "peach", "color": "pink" }
+    ]
+    ```
+
+  For type `null`:
+
+  - an array containing an array of keys followed by arrays of values, e.g.
+
+    ```
+    [
+      ["_key", "color"],
+      ["banana", "yellow"],
+      ["peach", "pink"]
+    ]
+    ```
+
+  - a string or buffer containing a JSON array of keys followed by
+    one JSON array of values per line, e.g.
+
+    ```
+    ["_key", "color"]
+    ["banana", "yellow"]
+    ["peach", "pink"]
+    ```
+
+- **opts**: `Object` (optional) If _opts_ is set, it must be an object with any
   of the following properties:
 
-  * **waitForSync**: `boolean` (Default: `false`)
+  - **type**: `string | null` (Default: `"auto"`)
+
+    Indicates which format the data uses.
+    Can be `"documents"`, `"array"` or `"auto"`.
+    Use `null` to explicitly set no type.
+
+  - **fromPrefix**: `string` (optional)
+
+    Prefix to prepend to `_from` attributes.
+
+  - **toPrefix**: `string` (optional)
+
+    Prefix to prepend to `_to` attributes.
+
+  - **overwrite**: `boolean` (Default: `false`)
+
+    If set to `true`, the collection is truncated before the data is imported.
+
+  - **waitForSync**: `boolean` (Default: `false`)
 
     Wait until the documents have been synced to disk.
 
-  * **details**: `boolean` (Default: `false`)
+  - **onDuplicate**: `string` (Default: `"error"`)
 
-    Whether the response should contain additional details about documents that
-    could not be imported.false\*.
+    Controls behavior when a unique constraint is violated.
+    Can be `"error"`, `"update"`, `"replace"` or `"ignore"`.
+
+  - **complete**: `boolean` (Default: `false`)
 
-  * **type**: `string` (Default: `"auto"`)
+    If set to `true`, the import will abort if any error occurs.
 
-    Indicates which format the data uses. Can be `"documents"`, `"array"` or
-    `"auto"`.
+  - **details**: `boolean` (Default: `false`)
 
-If _data_ is a JavaScript array, it will be transmitted as a line-delimited JSON
-stream. If _opts.type_ is set to `"array"`, it will be transmitted as regular
-JSON instead. If _data_ is a string, it will be transmitted as it is without any
-processing.
+    Whether the response should contain additional details about documents that
+    could not be imported.
 
 For more information on the _opts_ object, see
-[the HTTP API documentation for bulk imports](https://docs.arangodb.com/latest/HTTP/BulkImports/ImportingSelfContained.html).
+[the HTTP API documentation for bulk imports](https://docs.arangodb.com/latest/HTTP/BulkImports/).
 
 **Examples**
 
 ```js
 const db = new Database();
-const collection = db.collection('users');
+const collection = db.collection("users");
 
-// document stream
-const result = await collection.import([
-  {username: 'admin', password: 'hunter2'},
-  {username: 'jcd', password: 'bionicman'},
-  {username: 'jreyes', password: 'amigo'},
-  {username: 'ghermann', password: 'zeitgeist'}
-]);
-assert.equal(result.created, 4);
+const result = await collection.import(
+  [
+    { username: "jcd", password: "bionicman" },
+    { username: "jreyes", password: "amigo" },
+    { username: "ghermann", password: "zeitgeist" }
+  ],
+  { type: "documents" } // optional
+);
 
 // -- or --
 
-// array stream with header
-const result = await collection.import([
-  ['username', 'password'], // keys
-  ['admin', 'hunter2'], // row 1
-  ['jcd', 'bionicman'], // row 2
-  ['jreyes', 'amigo'],
-  ['ghermann', 'zeitgeist']
-]);
-assert.equal(result.created, 4);
+const buf = fs.readFileSync("dx_users.json");
+// [
+//   {"username": "jcd", "password": "bionicman"},
+//   {"username": "jreyes", "password": "amigo"},
+//   {"username": "ghermann", "password": "zeitgeist"}
+// ]
+const result = await collection.import(
+  buf,
+  { type: "array" } // optional
+);
+```
 
 // -- or --
 
-// raw line-delimited JSON array stream with header
-const result = await collection.import([
-  '["username", "password"]',
-  '["admin", "hunter2"]',
-  '["jcd", "bionicman"]',
-  '["jreyes", "amigo"]',
-  '["ghermann", "zeitgeist"]'
-].join('\r\n') + '\r\n');
-assert.equal(result.created, 4);
-```
+const result = await collection.import(
+[
+["username", "password"],
+["jcd", "bionicman"],
+["jreyes", "amigo"],
+["ghermann", "zeitgeist"]
+],
+{ type: null } // required
+);

src/collection.ts

@@ -20,6 +20,27 @@ export type IndexHandle =
       id?: string;
     };
 
+export interface ImportOptions {
+  type?: null | "auto" | "documents" | "array";
+  fromPrefix?: string;
+  toPrefix?: string;
+  overwrite?: boolean;
+  waitForSync?: boolean;
+  onDuplicate?: "error" | "update" | "replace" | "ignore";
+  complete?: boolean;
+  details?: boolean;
+}
+
+export interface ImportResult {
+  error: false;
+  created: number;
+  errors: number;
+  empty: number;
+  updated: number;
+  ignored: number;
+  details?: string[];
+}
+
 export function isArangoCollection(
   collection: any
 ): collection is ArangoCollection {
@@ -500,14 +521,24 @@ export abstract class BaseCollection implements ArangoCollection {
     );
   }
 
-  import(data: any, opts?: any) {
+  import(
+    data: Buffer | Blob | string | any[],
+    { type = "auto", ...opts }: ImportOptions = {}
+  ): Promise<ImportResult> {
+    if (Array.isArray(data)) {
+      data = data.map(line => JSON.stringify(line)).join("\r\n") + "\r\n";
+    }
     return this._connection.request(
       {
         method: "POST",
         path: "/_api/import",
         body: data,
-        isJsonStream: Boolean(!opts || opts.type !== "array"),
-        qs: { type: "auto", ...opts, collection: this.name }
+        isBinary: true,
+        qs: {
+          type: type === null ? undefined : type,
+          ...opts,
+          collection: this.name
+        }
       },
       res => res.body
     );

src/connection.ts

@@ -42,7 +42,6 @@ export type RequestOptions = {
   body?: any;
   expectBinary?: boolean;
   isBinary?: boolean;
-  isJsonStream?: boolean;
   headers?: { [key: string]: string };
   absolutePath?: boolean;
   basePath?: string;
@@ -272,7 +271,6 @@ export class Connection {
       body,
       expectBinary = false,
       isBinary = false,
-      isJsonStream = false,
       headers,
       ...urlInfo
     }: RequestOptions,
@@ -284,14 +282,8 @@ export class Connection {
         contentType = "application/octet-stream";
       } else if (body) {
         if (typeof body === "object") {
-          if (isJsonStream) {
-            body =
-              body.map((obj: any) => JSON.stringify(obj)).join("\r\n") + "\r\n";
-            contentType = "application/x-ldjson";
-          } else {
-            body = JSON.stringify(body);
-            contentType = "application/json";
-          }
+          body = JSON.stringify(body);
+          contentType = "application/json";
         } else {
           body = String(body);
         }