streams + attachment size check + attachment url in body (#72)

* Created new `Read file` action instead of existing due to incorrect output metadata, old one set as deprecated 
* Fix memory leak for `Read file` action and `Get New and Updated S3 Objects` trigger, also added attachment url in message body to them
* Default value for environment variable `ATTACHMENT_MAX_SIZE` increased from `10000000` (almost **10** MB) to `104857600` bytes (**100** MB)
* Implemented additional check attachments size  in `Get New and Updated S3 Objects` trigger
* Get rid of vulnerabilities in dependencies

Author: if0s

.circleci/config.yml

@@ -72,23 +72,23 @@ commands:
 jobs:
   test:
     docker:
-      - image: circleci/node:14-stretch
+      - image: circleci/node:16-stretch
     steps:
       - checkout
       - node/install:
           node-version: << pipeline.parameters.node-version >>
-      # - run:
-      #     name: Audit Dependencies
-      #     command: npm audit --audit-level=high
+      - run:
+          name: Audit Dependencies
+          command: npm audit --audit-level=high
       - node/install-packages:
           cache-path: ./node_modules
           override-ci-command: npm install
       - run:
-          name: Running Mocha Tests
-          command: npm test
+          name: Running Mocha Unit&Integration Tests
+          command: npm test && npm run integration-test
   build:
     docker:
-      - image: circleci/node:14-stretch
+      - image: circleci/node:16-stretch
         user: root
     steps:
       - checkout

.gitignore

@@ -3,3 +3,4 @@ coverage
 .idea
 .env
 .vscode
+.nyc_output

CHANGELOG.md

@@ -1,5 +1,11 @@
-# 1.4.3 (April 22, 2022)
+# 1.5.0 (May 20, 2022)
+* Created new `Read file` action instead of existing due to incorrect output metadata, old one set as deprecated 
+* Fix memory leak for `Read file` action and `Get New and Updated S3 Objects` trigger, also added attachment url in message body to them
+* Default value for environment variable `ATTACHMENT_MAX_SIZE` increased from `10000000` (almost **10** MB) to `104857600` bytes (**100** MB)
+* Implemented additional check attachments size  in `Get New and Updated S3 Objects` trigger
+* Get rid of vulnerabilities in dependencies
 
+# 1.4.3 (April 22, 2022)
 * Update `component-commons-library` to 2.0.2
 * Update `oih-standard-library` to 2.0.2
 * Update `elasticio-sailor-nodejs` to 2.6.27

README.md

@@ -39,15 +39,15 @@ The component provides ability to connect to Amazon Simple Storage Service (Amaz
 [Completeness Matrix](https://docs.google.com/spreadsheets/d/1sptYGKkInnAbfRRbzLr5oOZUd3-COekQWGKpqQUYVfc/edit#gid=0)
 
 ### How works. SDK version  
-The component is based on [AWS S3 SDK](https://aws.amazon.com/sdk-for-node-js/ 'SDK for NodeJS') version 2.683.0.
+The component is based on [AWS S3 SDK](https://aws.amazon.com/sdk-for-node-js/ 'SDK for NodeJS') version 2.1132.0.
 
 ## Requirements
 
 #### Environment variables
 Name|Mandatory|Description|Values|
 |----|---------|-----------|------|
 |`LOG_LEVEL`| false | Controls logger level | `trace`, `debug`, `info`, `warning`, `error` |
-|`ATTACHMENT_MAX_SIZE`| false | For `elastic.io` attachments configuration. Maximal possible attachment size in bytes. By default set to 1000000 and according to platform limitations CAN'T be bigger than that. | Up to `1000000` bytes|
+|`ATTACHMENT_MAX_SIZE`| false | For `elastic.io` attachments configuration. Maximal possible attachment size in bytes. By default set to `104857600` and according to platform limitations **CAN'T** be bigger than that. | Up to `104857600` bytes (100MB)|
 |`ACCESS_KEY_ID`| false | For integration-tests is required to specify this variable |  |
 |`ACCESS_KEY_SECRET`| false | For integration-tests is required to specify this variable |  |
 |`REGION`  | false | For integration-tests is required to specify this variable |  |
@@ -72,18 +72,27 @@ Triggers to get all new and updated s3 objects since last polling.
 
 #### List of Expected Config fields
  - **Bucket Name and folder** - name of S3 bucket to read files from
- - **Emit Behaviour**: Options are: default is `Emit Individually` emits each object in separate message, `Fetch All` emits all objects in one message
+ - **Emit Behaviour**: Options are: default is `Emit Individually` emits each object in separate message, `Fetch All` emits all objects as array in one object with key `results`
  - **Start Time**: Start datetime of polling. Default min date:`-271821-04-20T00:00:00.000Z`
  - **End Time**: End datetime of polling. Default max date: `+275760-09-13T00:00:00.000Z`
- - **Enable File Attachments**: If selected, the contents of the file will be exported in addition to the file metadata.
+ - **Enable File Attachments**: If selected, the contents of the file will be exported in addition to the attachment.
+
 
 <details> 
 <summary>Output metadata</summary>
 
+If **Emit Behaviour** selected as `Emit Individually` - emits each object in separate message with schema below, if `Fetch All` emits all objects as array in one object with key `results`, each item regards schema below
+
+`attachmentUrl` appears only if selected **Enable File Attachments**
+
 ```json
 {
   "type": "object",
   "properties": {
+    "attachmentUrl": {
+      "type": "string",
+      "required": true
+    },
     "Key": {
       "type": "string",
       "required": true
@@ -181,6 +190,14 @@ File type resolves by it's extension. The name of attachment would be same to fi
     "filename": {
       "type": "string",
       "required": true
+    },
+    "attachmentUrl": {
+      "type": "string",
+      "required": true
+    },
+    "size": {
+      "type": "number",
+      "required": true
     }
   }
 }

component.json

@@ -1,6 +1,6 @@
 {
   "title": "AWS S3",
-  "version": "1.4.3",
+  "version": "1.5.0",
   "description": "Integration component that can read and write to AWS S3",
   "docsUrl": "https://github.com/elasticio/amazon-s3-component",
   "credentials": {
@@ -67,42 +67,7 @@
           "label": "Enable File Attachments"
         }
       },
-      "metadata": {
-        "out": {
-          "type": "object",
-          "properties": {
-            "Key": {
-              "type": "string",
-              "required": true
-            },
-            "LastModified": {
-              "type": "string",
-              "required": true
-            },
-            "ETag": {
-              "type": "string",
-              "required": true
-            },
-            "Size": {
-              "type": "number",
-              "required": true
-            },
-            "StorageClass": {
-              "type": "string",
-              "required": true
-            },
-            "Owner": {
-              "type": "object",
-              "properties": {
-                "ID": {
-                  "type": "string",
-                  "required": true
-                }
-              }
-            }
-          }
-        }
-      }
+      "dynamicMetadata": true
     }
   },
   "actions": {
@@ -162,9 +127,10 @@
       }
     },
     "readFile": {
+      "deprecated": true,
       "title": "Read file",
       "help": {
-        "description": "Read file from S3 bucket",
+        "description": "Action is deprecated do to incorrect output metadata. Use main 'Read file' action instead",
         "link": "/components/aws-s3/index.html#read-file"
       },
       "main": "./lib/actions/readFile.js",
@@ -203,6 +169,56 @@
         }
       }
     },
+    "readFile2": {
+      "title": "Read file",
+      "help": {
+        "description": "Read file from S3 bucket",
+        "link": "/components/aws-s3/index.html#read-file"
+      },
+      "main": "./lib/actions/readFile2.js",
+      "fields": {
+        "bucketName": {
+          "viewClass": "TextFieldView",
+          "label": "Default Bucket Name and folder",
+          "placeholder": "my-fancy-bucket",
+          "note": "Default Bucket Name and folder will override if 'Bucket Name and folder' field is set in metadata",
+          "required": false
+        }
+      },
+      "metadata": {
+        "in": {
+          "type": "object",
+          "properties": {
+            "filename": {
+              "type": "string",
+              "required": true
+            },
+            "bucketName": {
+              "title": "Bucket Name and folder",
+              "type": "string",
+              "required": false
+            }
+          }
+        },
+        "out": {
+          "type": "object",
+          "properties": {
+            "filename": {
+              "type": "string",
+              "required": true
+            },
+            "attachmentUrl": {
+              "type": "string",
+              "required": true
+            },
+            "size": {
+              "type": "number",
+              "required": true
+            }
+          }
+        }
+      }
+    },
     "getAllFilesInBucket": {
       "title": "Get filenames",
       "help": {

lib/actions/readFile.js

@@ -1,11 +1,12 @@
-/* eslint-disable no-use-before-define,consistent-return,func-names,no-param-reassign */
-const { AttachmentProcessor } = require('@elastic.io/component-commons-library');
+/* eslint-disable no-use-before-define,consistent-return,func-names */
 const { messages } = require('elasticio-node');
 const convert = require('xml-js');
 const mime = require('mime-types');
 const iconv = require('iconv-lite');
 const { Client } = require('../client');
-const { REQUEST_MAX_BODY_LENGTH } = require('../parameters');
+const params = require('../parameters');
+
+const attachmentProcessor = require('../utils/attachmentProcessor');
 
 exports.process = async function (msg, cfg) {
   const client = new Client(this.logger, cfg);
@@ -14,11 +15,11 @@ exports.process = async function (msg, cfg) {
 
   const result = await client.getObject(bucketName, filename);
 
-  if (result.ContentLength > REQUEST_MAX_BODY_LENGTH) {
+  if (result.ContentLength > params.ATTACHMENT_MAX_SIZE) {
     this.logger.error('File %s with size %d bytes is too big for attachment usage. '
-      + 'Current attachment max size is %d bytes', filename, result.ContentLength, REQUEST_MAX_BODY_LENGTH);
+      + 'Current attachment max size is %d bytes', filename, result.ContentLength, params.ATTACHMENT_MAX_SIZE);
     throw new Error(`File ${filename} with size ${result.ContentLength} bytes is too big for attachment usage. `
-      + `Current attachment max size is ${REQUEST_MAX_BODY_LENGTH} bytes`);
+      + `Current attachment max size is ${params.ATTACHMENT_MAX_SIZE} bytes`);
   }
 
   const fileContent = iconv.decode(result.Body, 'iso-8859-15');
@@ -31,18 +32,10 @@ exports.process = async function (msg, cfg) {
     const xmlDoc = JSON.parse(convert.xml2json(fileContent));
     await this.emit('data', messages.newMessageWithBody(xmlDoc));
   } else {
-    const attachmentProcessor = new AttachmentProcessor();
-    const response = await attachmentProcessor.uploadAttachment(result.Body, contentType);
-    const attachmentUrl = `${response.config.url}${response.data.objectId}?storage_type=maester`;
-    msg.attachments = {
-      [filename]: {
-        url: attachmentUrl,
-        size: fileContent.length,
-        'content-type': contentType,
-      },
-    };
-    const output = messages.newMessageWithBody(msg);
-    output.attachments = msg.attachments;
+    const response = await attachmentProcessor.addAttachment.call(this, msg, filename,
+      fileContent, contentType);
+    const output = messages.newMessageWithBody(response);
+    output.attachments = response.attachments;
     return output;
   }
 };

lib/actions/readFile2.js

@@ -0,0 +1,56 @@
+/* eslint-disable no-use-before-define,consistent-return,func-names,no-param-reassign */
+const { messages } = require('elasticio-node');
+const convert = require('xml-js');
+const mime = require('mime-types');
+const iconv = require('iconv-lite');
+const { AttachmentProcessor } = require('@elastic.io/component-commons-library');
+const { Client } = require('../client');
+const params = require('../parameters');
+
+
+exports.process = async function (msg, cfg) {
+  const client = new Client(this.logger, cfg);
+  const bucketName = msg.body.bucketName ? msg.body.bucketName : cfg.bucketName;
+  const { filename } = msg.body;
+
+  // const result = await client.getObjectReadStream(bucketName, filename);
+  const result = await client.getObjectMetadata(bucketName, filename);
+
+  if (result.ContentLength > params.ATTACHMENT_MAX_SIZE) {
+    const err = `File ${filename} with size ${result.ContentLength} bytes is too big for attachment usage. `
+      + `Current attachment max size is ${params.ATTACHMENT_MAX_SIZE} bytes`;
+    this.logger.error(err);
+    throw new Error(err);
+  }
+
+  const contentType = mime.lookup(filename);
+  this.logger.info(`File type - "${contentType}"`);
+
+  if (['application/json', 'application/xml'].includes(contentType)) {
+    const data = await client.getObject(bucketName, filename);
+    const fileContent = iconv.decode(data.Body, 'iso-8859-15');
+    let doc;
+    if (contentType === 'application/json') doc = JSON.parse(fileContent);
+    if (contentType === 'application/xml') doc = JSON.parse(convert.xml2json(fileContent));
+    await this.emit('data', messages.newMessageWithBody(doc));
+  } else {
+    const readStream = client.getObjectReadStream(bucketName, filename);
+    const results = await new AttachmentProcessor().uploadAttachment(readStream);
+    const attachmentUrl = `${results.config.url}${results.data.objectId}?storage_type=maester`;
+
+    const attachments = {
+      [filename]: {
+        url: attachmentUrl,
+        size: result.ContentLength,
+        'content-type': contentType,
+      },
+    };
+    const output = messages.newMessageWithBody({
+      filename,
+      attachmentUrl,
+      size: result.ContentLength,
+    });
+    output.attachments = attachments;
+    return output;
+  }
+};

lib/parameters.js

@@ -1 +1 @@
-exports.REQUEST_MAX_BODY_LENGTH = process.env.REQUEST_MAX_BODY_LENGTH ? parseInt(process.env.REQUEST_MAX_BODY_LENGTH, 10) : 104857600; // 100MB
+exports.ATTACHMENT_MAX_SIZE = process.env.ATTACHMENT_MAX_SIZE ? parseInt(process.env.ATTACHMENT_MAX_SIZE, 10) : 1024 * 1024 * 100;

lib/triggers/pollingTrigger.js

@@ -1,10 +1,34 @@
 const { Client } = require('../client');
 const { AwsS3Polling } = require('../utils/pollingUtil');
 
-// eslint-disable-next-line func-names
-exports.process = async function (msg, cfg, snapshot) {
+async function process(_msg, cfg, snapshot) {
   const client = new Client(this.logger, cfg);
 
   const pollingTrigger = new AwsS3Polling(this.logger, this, client, cfg);
   await pollingTrigger.process(cfg, snapshot);
-};
+}
+
+async function getMetaModel(cfg) {
+  const metadataBase = {
+    attachmentUrl: { type: 'string', required: true },
+    Key: { type: 'string', required: true },
+    LastModified: { type: 'string', required: true },
+    ETag: { type: 'string', required: true },
+    Size: { type: 'number', required: true },
+    StorageClass: { type: 'string', required: true },
+    Owner: {
+      type: 'object',
+      properties: { ID: { type: 'string', required: true } },
+    },
+  };
+  if (cfg.emitBehaviour === 'emitIndividually') {
+    return { out: { type: 'object', properties: metadataBase } };
+  }
+  if (cfg.emitBehaviour === 'fetchAll') {
+    return { out: { type: 'object', properties: { results: { type: 'array', items: metadataBase, required: true }, required: true } } };
+  }
+  return {};
+}
+
+module.exports.getMetaModel = getMetaModel;
+module.exports.process = process;