Sprint 39 master (#68)

* Add the steward URL to the body of the outgoing message

* Wrapping csv inputs in json objects

Author: denyshld

CHANGELOG.md

@@ -1,4 +1,9 @@
-## 2.1.0 (May 7, 2020)
+## 2.1.1 (May 7, 2020)
+
+* Add input metadata for objects processing
+* Add the steward URL to the body of the outgoing message
+
+## 2.1.0 (April 22, 2020)
 
 * Add "Write CSV attachment from Array" action
 * Add "Write CSV attachment from JSON" action

README.md

@@ -117,7 +117,7 @@ This action will convert an incoming array into a CSV file by following approach
 
 Requirements:
 
-* The inbound message is an JSON Object;
+* The inbound message is an JSON Object, wrapped by 'inputObject' object;
 * This JSON object has plain structure without nested levels (structured types `objects` and `arrays` are not supported as values). Only primitive types are supported: `strings`, `numbers`, `booleans` and `null`. Otherwise, the error message will be thrown: `Inbound message should be a plain Object. At least one of entries is not a primitive type`.
 
 The keys of an input JSON will be published as the header in the first row. For each incoming
@@ -126,9 +126,9 @@ for that cell. All other properties will be ignored. For example, headers
 `foo,bar` along with the following JSON events:
 
 ```
-{"foo":"myfoo", "bar":"mybar"}
-{"foo":"myfoo", "bar":[1,2]}
-{"bar":"mybar", "baz":"mybaz"}
+{"inputObject": {"foo":"myfoo", "bar":"mybar"}}
+{"inputObject": {"foo":"myfoo", "bar":[1,2]}}
+{"inputObject": {"bar":"mybar", "baz":"mybaz"}}
 ```
 
 will produce the following `.csv` file:
@@ -158,7 +158,7 @@ This action will convert an incoming array into a CSV file by following approach
 
 Requirements:
 
-* The inbound message is an JSON Array of Objects with identical structure;
+* The inbound message is an JSON Array of Objects with identical structure, wrapped by 'inputArray' object;
 * Each JSON object has plain structure without nested levels (structured types `objects` and `arrays` are not supported as values). Only primitive types are supported: `strings`, `numbers`, `booleans` and `null`. Otherwise, the error message will be thrown: `Inbound message should be a plain Object. At least one of entries is not a primitive type`.
 
 The keys of an input JSON will be published as the header in the first row. For each incoming
@@ -167,11 +167,13 @@ for that cell. All other properties will be ignored. For example, headers
 `foo,bar` along with the following JSON events:
 
 ```
-[
-    {"foo":"myfoo", "bar":"mybar"}
-    {"foo":"myfoo", "bar":[1,2]}
-    {"bar":"mybar", "baz":"mybaz"}
-]
+{
+    "inputArray": [
+        {"foo":"myfoo", "bar":"mybar"}
+        {"foo":"myfoo", "bar":[1,2]}
+        {"bar":"mybar", "baz":"mybaz"}
+    ]
+}
 ```
 
 will produce the following `.csv` file:

component.json

@@ -117,7 +117,12 @@
       "metadata": {
         "in": {
           "type": "object",
-          "properties": {}
+          "properties": {
+            "inputObject": {
+              "type": "object",
+              "properties": {}
+            }
+          }
         },
         "out": {}
       }
@@ -156,11 +161,16 @@
       },
       "metadata": {
         "in": {
-          "type": "array",
-          "properties": {}
+          "type": "object",
+          "properties": {
+            "inputArray": {
+              "type": "array",
+              "items": {}
+            }
+          }
         },
         "out": {}
       }
     }
   }
-}
+}

lib/actions/write.js

@@ -77,6 +77,7 @@ async function ProcessAction(msg, cfg) {
 
     const messageToEmit = messages.newMessageWithBody({
       rowCount: finalRowCount,
+      url: signedUrl.get_url,
     });
     const fileName = `${messageToEmit.id}.csv`;
     messageToEmit.attachments[fileName] = {

lib/actions/writeFromArray.js

@@ -59,16 +59,17 @@ async function init(cfg) {
 async function ProcessAction(msg) {
   // eslint-disable-next-line consistent-this
   const self = this;
+  const { inputArray } = msg.body;
   let isError = false;
   let errorValue = '';
 
-  const columns = Object.keys(msg.body[0]);
-  rowCount = msg.body.length;
+  const columns = Object.keys(inputArray[0]);
+  rowCount = inputArray.length;
   logger.trace('Configured column names:', columns);
   let row = {};
 
-  await _.each(msg.body, async (item) => {
-    const entries = Object.values(msg.body);
+  await _.each(inputArray, async (item) => {
+    const entries = Object.values(inputArray);
     // eslint-disable-next-line no-restricted-syntax
     for (const entry of entries) {
       if (isError) {
@@ -104,6 +105,7 @@ async function ProcessAction(msg) {
 
   const messageToEmit = messages.newMessageWithBody({
     rowCount,
+    url: signedUrl.get_url,
   });
   const fileName = `${messageToEmit.id}.csv`;
   messageToEmit.attachments[fileName] = {

lib/actions/writeFromJson.js

@@ -64,11 +64,12 @@ async function init(cfg) {
 async function ProcessAction(msg) {
   // eslint-disable-next-line consistent-this
   const self = this;
+  const { inputObject } = msg.body;
 
-  const columns = Object.keys(msg.body);
+  const columns = Object.keys(inputObject);
   logger.trace('Configured column names:', columns);
 
-  const values = Object.values(msg.body);
+  const values = Object.values(inputObject);
   // eslint-disable-next-line no-restricted-syntax
   for (const value of values) {
     if (value !== null && value !== undefined && (typeof value === 'object' || Array.isArray(value))) {
@@ -102,6 +103,7 @@ async function ProcessAction(msg) {
 
     const messageToEmit = messages.newMessageWithBody({
       rowCount: finalRowCount,
+      url: signedUrl.get_url,
     });
     const fileName = `${messageToEmit.id}.csv`;
     messageToEmit.attachments[fileName] = {
@@ -114,7 +116,7 @@ async function ProcessAction(msg) {
     await self.emit('data', messageToEmit);
   }, TIMEOUT_BETWEEN_EVENTS);
 
-  let row = msg.body;
+  let row = inputObject;
   self.logger.trace(`Incoming data: ${JSON.stringify(row)}`);
   row = _.pick(row, columns);
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "csv-component",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "CSV Component for elastic.io platform",
   "main": "index.js",
   "scripts": {

package.json

@@ -51,22 +51,24 @@ describe('CSV Write From Array component', function () {
     }, cfg);
 
     const msg = {
-      body: [
-        {
-          name: 'Bob',
-          email: '[email protected]',
-          age: 30,
-          key1: true,
-          'not an age': null,
-          'not an age at all': undefined,
-        },
-        {
-          name: 'Joe',
-          email: '[email protected]',
-          age: 11,
-          'not an age at all': 322,
-        },
-      ],
+      body: {
+        inputArray: [
+          {
+            name: 'Bob',
+            email: '[email protected]',
+            age: 30,
+            key1: true,
+            'not an age': null,
+            'not an age at all': undefined,
+          },
+          {
+            name: 'Joe',
+            email: '[email protected]',
+            age: 11,
+            'not an age at all': 322,
+          },
+        ],
+      },
     };
 
     await write.process.call({

spec/writeFromArray.spec.js

@@ -52,8 +52,10 @@ describe('CSV Write From JSON component', function () {
 
     const msg1 = {
       body: {
-        ProductKey: 'text11',
-        CategoryGroup_1: 'text12',
+        inputObject: {
+          ProductKey: 'text11',
+          CategoryGroup_1: 'text12',
+        },
       },
     };
 
@@ -64,8 +66,10 @@ describe('CSV Write From JSON component', function () {
 
     const msg2 = {
       body: {
-        ProductKey: 'text21',
-        CategoryGroup_1: 'text22',
+        inputObject: {
+          ProductKey: 'text21',
+          CategoryGroup_1: 'text22',
+        },
       },
     };