Sprint 33 master (#40)

* Add bucket name validation rules

* Fix `Error! Cannot convert undefined or null to object` error on no attachments object in message

* Add attachment size check on file reading

* Add empty response to Delete file action when file already not exists

* Add Get New And Updated files trigger

* Add files metadata polling trigger

* Add possibility to retrieve more than 1,000 files for 'Get filenames' action

Author: denyshld

.circleci/config.yml

@@ -17,28 +17,8 @@ jobs:
       - run:
           name: Running Mocha Unit&Integration Tests
           command: npm test && npm run integration-test
-  docs:
-    docker:
-      - image: circleci/node:12-stretch
-    steps:
-      - checkout
-      - run:
-          name: Download Script
-          command: mkdir $HOME/script && curl -o $HOME/script/create_pr.bash https://raw.githubusercontent.com/elasticio/elasticio.github.io/master/scripts/create_pr.bash
-      - run:
-          shell: /bin/bash
-          name: Create PR
-          command: bash $HOME/script/create_pr.bash -p $PWD -n sftp
-
 workflows:
   version: 2
   build_and_test:
     jobs:
       - test
-  deploy_docs:
-    jobs:
-      - docs:
-          filters:
-            branches:
-              only:
-                - master

CHANGELOG.md

@@ -1,3 +1,14 @@
+# 1.3.0 (February 13, 2020)
+
+## General Changes
+    * Add Get New and Updated S3 Objects trigger
+    * Fix `Error! Cannot convert undefined or null to object` error on no attachments object in message in Write file action
+    * Add attachment size limitation
+    * Add empty response to Delete file action when file already not exists
+    * Add possibility to retrieve more than 1,000 files for 'Get filenames' action
+    * Improved error handling for Get filenames action
+    * Removed invalid docs job from circle ci
+    
 # 1.2.1 (December 26, 2019)
 
 ## General Changes

README.md

@@ -5,13 +5,15 @@
    * [Description](#description)
    * [Purpose](#purpose)
    * [Completeness Matrix](#completeness-matrix)
-   * [How works. SDK version](#how-works.-sdk-version)
+   * [How works. SDK version](#how-works-sdk-version)
 * [Requirements](#requirements)
    * [Environment variables](#environment-variables)
 * [Credentials](#credentials)
      * [Access Key Id](#access-key-id)
      * [Secret Access Key](#secret-access-key)
      * [Region](#region)
+* [Triggers](#triggers)
+   * [Get New and Updated S3 Objects](#get-new-and-updated-s3-objects)
 * [Actions](#actions)
    * [Write file](#write-file)
    * [Read file](#read-file)
@@ -30,7 +32,7 @@ This is the component for working with AWS S3 object storage service on [elastic
 The component provides ability to connect to Amazon Simple Storage Service (Amazon S3) object storage service.
 
 ### Completeness Matrix
-![image](https://user-images.githubusercontent.com/36419533/70732156-97b31680-1d10-11ea-826f-5a2dd55b8251.png)
+![image](https://user-images.githubusercontent.com/22715422/73740795-93740a00-4751-11ea-869d-ff9433f39c4f.png)
 
 [Completeness Matrix](https://docs.google.com/spreadsheets/d/1LhKgsTvF32YAmBRh742YxnkrMEGlPEERJc9B6pj4L6E/edit#gid=0)
 
@@ -40,17 +42,13 @@ The component is based on [AWS S3 SDK](https://aws.amazon.com/sdk-for-node-js/ '
 ## Requirements
 
 #### Environment variables
-For integration-tests is required to specify following environment variables:
-
-`ACCESS_KEY_ID` - access key ID;
-
-`ACCESS_KEY_SECRET` - secret access key.
-
-`REGION` - region.
-
-For debugging purposes there is: 
-
-`LOG_LEVEL` - `trace` | `debug` | `info` | `warning` | `error` that controls logger level.
+Name|Mandatory|Description|Values|
+|----|---------|-----------|------|
+|`LOG_LEVEL`| false | Controls logger level | `trace`, `debug`, `info`, `warning`, `error` |
+|`ATTACHMENT_MAX_SIZE`| false | For `elastic.io` attachments configuration. Maximal possible attachment size in bytes. By default set to 1000000 and according to platform limitations CAN'T be bigger than that. | Up to `1000000` bytes|
+|`ACCESS_KEY_ID`| false | For integration-tests is required to specify this variable |  |
+|`ACCESS_KEY_SECRET`| false | For integration-tests is required to specify this variable |  |
+|`REGION`  | false | For integration-tests is required to specify this variable |  |
 
 ## Credentials
 Access keys consist of two parts: an access key ID and a secret access key. 
@@ -64,7 +62,59 @@ A secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`).
 
 ### Region
 Example: `ca-central-1`.
-    
+
+
+## Triggers
+### Get New and Updated S3 Objects
+Triggers to get all new and updated s3 objects since last polling.
+
+#### List of Expected Config fields
+ - **Bucket Name and folder** - name of S3 bucket to read files from
+ - **Emit Behaviour**: Options are: default is `Emit Individually` emits each object in separate message, `Fetch All` emits all objects in one message
+ - **Start Time**: Start datetime of polling. Default min date:`-271821-04-20T00:00:00.000Z`
+ - **End Time**: End datetime of polling. Default max date: `+275760-09-13T00:00:00.000Z`
+ - **Enable File Attachments**: End datetime of polling. Default max date: `+275760-09-13T00:00:00.000Z`
+
+<details> 
+<summary>Output metadata</summary>
+
+```json
+{
+  "type": "object",
+  "properties": {
+    "Key": {
+      "type": "string",
+      "required": true
+    },
+    "LastModified": {
+      "type": "string",
+      "required": true
+    },
+    "ETag": {
+      "type": "string",
+      "required": true
+    },
+    "Size": {
+      "type": "number",
+      "required": true
+    },
+    "StorageClass": {
+      "type": "string",
+      "required": true
+    },
+    "Owner": {
+      "type": "object",
+      "properties": {
+        "ID": {
+          "type": "string",
+          "required": true
+        }
+      }
+    }
+  }
+}
+```
+</details>
 
 ## Actions
 ### Write file

component.json

@@ -22,6 +22,86 @@
       }
     }
   },
+  "triggers": {
+    "pollingTrigger": {
+      "title": "Get New and Updated S3 Objects",
+      "type": "polling",
+      "description": "Get New and Updated S3 Objects by provided time limits",
+      "main": "./lib/triggers/pollingTrigger.js",
+      "fields": {
+        "bucketName": {
+          "viewClass": "TextFieldView",
+          "label": "Bucket Name and Folder",
+          "placeholder": "my-bucket/my-folder",
+          "required": true
+        },
+        "emitBehaviour": {
+          "label": "Emit Behaviour",
+          "viewClass": "SelectView",
+          "required": false,
+          "prompt": "Default Emit Individually",
+          "model": {
+            "emitIndividually": "Emit Individually",
+            "fetchAll": "Fetch All"
+          }
+        },
+        "startTime": {
+          "label": "Start Time",
+          "viewClass": "TextFieldView",
+          "required": false,
+          "note": "Input format: ISO 8601. Default: minimum time",
+          "placeholder": "2020-01-01T01:01:01Z"
+        },
+        "endTime": {
+          "label": "End Time",
+          "viewClass": "TextFieldView",
+          "required": false,
+          "note": "Input format: ISO 8601. Default: maximum time",
+          "placeholder": "2020-01-01T01:01:01Z"
+        },
+        "enableFileAttachments": {
+          "viewClass": "CheckBoxView",
+          "label": "Enable File Attachments"
+        }
+      },
+      "metadata": {
+        "out": {
+          "type": "object",
+          "properties": {
+            "Key": {
+              "type": "string",
+              "required": true
+            },
+            "LastModified": {
+              "type": "string",
+              "required": true
+            },
+            "ETag": {
+              "type": "string",
+              "required": true
+            },
+            "Size": {
+              "type": "number",
+              "required": true
+            },
+            "StorageClass": {
+              "type": "string",
+              "required": true
+            },
+            "Owner": {
+              "type": "object",
+              "properties": {
+                "ID": {
+                  "type": "string",
+                  "required": true
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
   "actions": {
     "streamToFile": {
       "title": "Write file",

lib/actions/deleteObject.js

@@ -5,6 +5,19 @@ const { Client } = require('../client');
 exports.process = async function (msg, cfg) {
   const client = new Client(this.logger, cfg);
   const bucketName = msg.body.bucketName ? msg.body.bucketName : cfg.bucketName;
+
+  try {
+    await client.getObjectMetadata(bucketName, msg.body.filename);
+  } catch (err) {
+    if (err.code === 'NotFound') {
+      this.logger.warn('Provided filename is not found: nothing to delete');
+      return messages.newEmptyMessage();
+    }
+    this.logger.error('Error occurred while getting file metadata: %j', err);
+    throw err;
+  }
+
+  this.logger.info('Provided filename is found: deleting...');
   await client.deleteObject(bucketName, msg.body.filename);
-  await this.emit('data', messages.newMessageWithBody({ filename: msg.body.filename }));
+  return messages.newMessageWithBody({ filename: msg.body.filename });
 };

lib/actions/getAllFilesInBucket.js

@@ -6,9 +6,25 @@ const { createAWSInputs } = require('../utils/utils');
 exports.process = async function (msg, cfg) {
   const client = new Client(this.logger, cfg);
   const bucketName = msg.body.bucketName ? msg.body.bucketName : cfg.bucketName;
-  const data = await client.listObjects(createAWSInputs(bucketName));
-  data.Contents.forEach((c) => {
-    this.emit('data', messages.newMessageWithBody({ filename: c.Key }));
-  });
-  this.emit('end');
+  if (!bucketName) {
+    throw new Error(`Bucket name cant be empty. Provided bucket name: ${bucketName}`);
+  }
+  const awsInput = createAWSInputs(bucketName);
+  if (awsInput.Prefix === null || awsInput.Prefix === undefined) {
+    awsInput.Prefix = '/';
+    awsInput.Delimiter = '/';
+  }
+  const exist = await client.exist(awsInput);
+  if (!exist) {
+    throw new Error(`Bucket ${bucketName} not exist`);
+  }
+  const data = await client.listObjects(awsInput);
+  if (data.length === 0) {
+    this.emit('data', messages.newEmptyMessage());
+  }
+
+  for (let i = 0; i < data.length; i += 1) {
+    /* eslint-disable-next-line no-await-in-loop */
+    await this.emit('data', messages.newMessageWithBody({ filename: data[i].Key }));
+  }
 };

lib/actions/readFile.js

@@ -4,6 +4,7 @@ const convert = require('xml-js');
 const mime = require('mime-types');
 const iconv = require('iconv-lite');
 const { Client } = require('../client');
+const params = require('../parameters');
 
 const attachmentProcessor = require('../utils/attachmentProcessor');
 
@@ -14,6 +15,13 @@ exports.process = async function (msg, cfg) {
 
   const result = await client.getObject(bucketName, filename);
 
+  if (result.ContentLength > params.ATTACHMENT_MAX_SIZE) {
+    this.logger.error('File %s with size %d bytes is too big for attachment usage. '
+      + 'Current attachment max size is %d bytes', filename, result.ContentLength, params.ATTACHMENT_MAX_SIZE);
+    throw new Error(`File ${filename} with size ${result.ContentLength} bytes is too big for attachment usage. `
+      + `Current attachment max size is ${params.ATTACHMENT_MAX_SIZE} bytes`);
+  }
+
   const fileContent = iconv.decode(result.Body, 'iso-8859-15');
   const contentType = mime.lookup(filename);
 

lib/actions/streamToFile.js

@@ -10,6 +10,13 @@ exports.process = async function (msg, cfg) {
   const client = new Client(this.logger, cfg);
   const bucketName = msg.body.bucketName ? msg.body.bucketName : cfg.bucketName;
 
+  if (!msg.attachments) {
+    throw new Error('Message has no attachments object (no attachments)');
+  } else if (Object.keys(msg.attachments).length > 1) {
+    throw new Error('More than 1 attachment');
+  } else if (Object.keys(msg.attachments).length === 0) {
+    throw new Error('Has no attachment');
+  }
   // eslint-disable-next-line arrow-body-style
   const attachments = Object.keys(msg.attachments).map((attachmentKey) => {
     return {
@@ -18,12 +25,6 @@ exports.process = async function (msg, cfg) {
     };
   });
 
-  if (attachments.length > 1) {
-    throw new Error('More than 1 attachment');
-  } else if (attachments.length === 0) {
-    throw new Error('Has no attachment');
-  }
-
   this.logger.trace('trying to get attachment from %j', attachments[0].url.url);
 
   const passthrough = new PassThrough();