Multiple improvements (#47)

* Obfuscate key

Fixes #46

* Added retry logic

Fixes #40

* Tweaking config

* Migrate to github actions

* Dropped node v10

Author: rferreira

.circleci/deploy.sh .github/workflows/deploy.sh.circleci/deploy.sh renamed to .github/workflows/deploy.sh

@@ -25,9 +25,9 @@ aws serverlessrepo create-application-version \
 echo "SAM application ${APPLICATION} version ${VERSION} published!"
 
 # tag repo
-git config --global user.email "circleci@uvasoftware.com"
-git config --global user.name "CircleCI"
-git tag -a v"${VERSION}" -m "Release by CircleCI v${VERSION}"
+git config --global user.email "ci@uvasoftware.com"
+git config --global user.name "Github Actions"
+git tag -a v"${VERSION}" -m "Release by Github Actions v${VERSION}"
 git push origin v"${VERSION}"
 
 # bumping version

.github/workflows/master.yaml

@@ -0,0 +1,17 @@
+on:
+  push:
+    branches:
+      - master
+jobs:
+  deploy:
+    if: "!contains(github.event.commits[0].message, '[ci skip]')"
+    name: Deploy
+    runs-on: ubuntu-latest
+    container:
+      image: debian:9
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: apt-get update -qq && apt-get install -qqy git unzip ssh ca-certificates tar gzip
+      - name: Deploy
+        run: bash ./.github/workflows/deploy.sh

.github/workflows/pr.yaml

@@ -0,0 +1,21 @@
+name: PR Builds
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [ 12.x, 14.x ]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: npm install
+      - run: npm run build --if-present
+      - run: npm test
+        env:
+          CI: true

README.md

@@ -2,13 +2,13 @@
 A Sam-Packaged AWS Lambda client to the [scanii.com](https://scanii.com) content processing engine. For a detailed walk-through of deploying this application see: https://support.scanii.com/article/55-how-do-i-analyze-content-stored-on-amazon-s3.
 
 ## How it works
-This is, essentially, a series of lambda functions packaged in a  one-click deployable application that configures everything needed so your S3 objects are submitted automatically to scanii’s content analysis [API](https://docs.scanii.com/v2.1/overview.html).  Once the content is processed,  you can choose from a couple of different actions:
+This is, essentially, a series of lambda functions packaged in a one-click deployable application that configures everything needed so your S3 objects are submitted automatically to scanii’s content analysis [API](https://docs.scanii.com/v2.1/overview.html).  Once the content is processed,  you can choose from a couple of different actions:
 
 1. Tag the content - this is defaulted to on and adds the following tag to objects processed: 
 	1. `ScaniiId` -> the resource id of the processed content
 	2. `ScaniiFindings` ->  list of identified findings (content engine dependent) 
 	3. `ScaniiContentType` -> the identified content type of the file processed 
-2. Delete object with findings - this is defaulted to **off** and will delete S3 objects with findings (such as malware or NSFW content) - for a full list of available content identification see https://support.scanii.com/article/20-content-detection-engines
+2. Delete the object with findings - this is defaulted to **off** and will delete S3 objects with findings (such as malware or NSFW content) - for a full list of available content identification see https://support.scanii.com/article/20-content-detection-engines
 
 ## Working with the source code
 The source code for this application is written using Javascript and requires, at least, nodejs 8 to run. Before getting started we strongly advise you to become familiar with the following technologies: 

lib/ag-handler.js

@@ -27,7 +27,8 @@ exports.handler = async (event, context, callback) => {
     console.log("metadata:", result.metadata);
 
     // now asserting bucket/keys were not tampered with:
-    assert.ok(result.metadata.signature === utils.generateSignature(result.metadata.bucket.toString(), result.metadata.key.toString()), "invalid signature");
+    assert.ok(result.metadata.signature === utils.generateSignature(result.metadata.bucket.toString(),
+      result.metadata.key.toString()), "invalid signature");
     console.log('signature check passed for signature', result.metadata.signature);
 
     if (result.error === undefined) {

lib/client.js

@@ -1,16 +1,27 @@
 const VERSION = require('../package.json').version;
-const request = require('request');
 const assert = require('assert');
+const axios = require('axios');
+const querystring = require('querystring')
 
 /**
  * Minimal Scanii API client in javascript (@see https://docs.scanii.com/v2.1/resources.html)
  */
 class ScaniiClient {
-  constructor(key, secret, endpoint = "api.scanii.com") {
+  constructor(key, secret, endpoint = "api.scanii.com", maxAttempts, maxAttemptDelay) {
     this.key = key;
     this.secret = secret;
-    this.endpoint = endpoint;
+    this.maxAttempts = maxAttempts;
+    this.maxAttemptDelay = maxAttemptDelay;
     this.userAgent = `scanii-lambda/v${VERSION}`;
+    this.client = axios.create({
+      auth: {
+        username: key, password: secret
+      },
+      headers: {
+        'User-Agent': this.userAgent
+      },
+      baseURL: `https://${endpoint}`
+    });
     console.log(`scanii client created using endpoint ${endpoint} and version ${VERSION}`)
   }
 
@@ -23,43 +34,26 @@ class ScaniiClient {
    */
 
   async fetch(location, callback, metadata) {
-    const options = {
-      url: `https://${this.endpoint}/v2.1/files/fetch`,
-      auth: {
-        'user': this.key,
-        'pass': this.secret,
-      },
-      headers: {
-        'User-Agent': this.userAgent
-      },
-      method: 'POST',
-      form: {
-        location: location,
-      }
-    };
+    let data = {
+      location: location
+    }
 
     if (callback !== null) {
-      options.form.callback = callback;
+      data.callback = callback;
     }
-
     if (metadata !== null) {
       for (const k in metadata) {
         if (metadata.hasOwnProperty(k)) {
-          options.form[`metadata[${k}]`] = metadata[k];
+          data[`metadata[${k}]`] = metadata[k];
         }
       }
     }
-    return new Promise((resolve, reject) => {
-      request(options, (error, response, body) => {
-        if (error) {
-          reject(error)
-        } else {
-          assert.ok(response.statusCode === 202, `Invalid response from server, with HTTP code: ${response.statusCode}`);
-          let file = JSON.parse(body);
-          console.log(`submit successful with id: ${file.id}`);
-          resolve({id: file.id, location: response.headers.location});
-        }
-      });
+
+    return await this._retry(async () => {
+      const response = await this.client.post('/v2.1/files/fetch', querystring.stringify(data), {headers: {'content-type': 'application/x-www-form-urlencoded'}});
+      assert.ok(response.status === 202, `Invalid response from server, with HTTP code: ${response.status}`);
+      console.log(`submit successful with id: ${response.data.id}`);
+      return ({id: response.data.id, location: response.headers.location});
     });
   }
 
@@ -70,32 +64,49 @@ class ScaniiClient {
    * @returns {Promise<*>}
    */
   async retrieve(id) {
-    const options = {
-      url: `https://${this.endpoint}/v2.1/files/${id}`,
-      auth: {
-        'user': this.key,
-        'pass': this.secret,
-      },
-      headers: {
-        'User-Agent': this.userAgent
-      },
-      method: 'GET'
-    };
+    return await this._retry(async () => {
+      const response = await this.client.get(`/v2.1/files/fetch/${id}`);
+      assert.ok(response.status === 200, `Invalid response from server, with HTTP code: ${response.status}`);
+      let result = JSON.parse(response.data);
+      console.log(`retrieve successful with id: ${result.id}`);
+      return result;
+    });
+  }
 
-    return new Promise((resolve, reject) => {
-      request(options, (error, response, body) => {
-        if (error) {
-          reject(error);
-        }
-        assert.ok(response.statusCode === 200, `Invalid response from server, with HTTP code: ${response.statusCode}`);
-        let result = JSON.parse(body);
-        console.log(`retrieve successful with id: ${result.id}`);
-        resolve(result);
+  /**
+   * Wraps an async function call around a basic retry logic
+   * @param func
+   * @returns {Promise<void>}
+   * @private
+   */
+  async _retry(func) {
+    let attempt = 1;
+    while (attempt <= this.maxAttempts) {
+      if (attempt > 1) {
+        const wait = Math.round(Math.random() * this.maxAttemptDelay);
+        console.log(`retrying is enabled, going to wait ${wait}ms and try again`);
+        await new Promise(resolve => setTimeout(resolve, wait));
+      }
 
-      });
-    });
+      try {
+        return await func()
+      } catch (e) {
+        console.error(e.message);
+      } finally {
+        attempt++
+      }
+    }
+
+    throw new ScaniiError(attempt - 1);
   }
+}
 
+class ScaniiError extends Error {
+  constructor(attempts) {
+    super(`Scanii ERROR, could not get a successful response from service after ${attempts} attempts`);
+    this.attempts = attempts;
+  }
 }
 
 exports.ScaniiClient = ScaniiClient;
+exports.ScaniiError = ScaniiError;

lib/config.js

@@ -1,36 +1,50 @@
-CONFIG = {
-  KEY: null,
-  SECRET: null,
-  API_ENDPOINT: "api.scanii.com",
-  CALLBACK_URL: null,
-  ACTION_TAG_OBJECT: false,
-  ACTION_DELETE_OBJECT: false,
-};
+const CONFIG = {}
 
 if (process.env.AWS_SAM_LOCAL !== undefined) {
   console.log("starting...");
   console.log(process.env);
 }
 
+function defaults() {
+  CONFIG.KEY = null;
+  CONFIG.SECRET = null;
+  CONFIG.API_ENDPOINT = "api.scanii.com";
+  CONFIG.CALLBACK_URL = null;
+  CONFIG.ACTION_TAG_OBJECT = false;
+  CONFIG.ACTION_DELETE_OBJECT = false;
+  CONFIG.MAX_ATTEMPTS = 10;
+  CONFIG.MAX_ATTEMPT_DELAY_MSEC = 30_000;
+
 // extracting config overwrites from the environment:
+  if (process.env.API_KEY) {
+    CONFIG.KEY = process.env.API_KEY;
+  }
+  if (process.env.API_SECRET) {
+    CONFIG.SECRET = process.env.API_SECRET;
+  }
 
-if (process.env.API_KEY) {
-  CONFIG.KEY = process.env.API_KEY;
-}
-if (process.env.API_SECRET) {
-  CONFIG.SECRET = process.env.API_SECRET;
-}
+  if (process.env.ACTION_TAG_OBJECT === "true") {
+    CONFIG.ACTION_TAG_OBJECT = true;
+  }
 
-if (process.env.ACTION_TAG_OBJECT === "true") {
-  CONFIG.ACTION_TAG_OBJECT = true;
-}
+  if (process.env.ACTION_DELETE_OBJECT === "true") {
+    CONFIG.ACTION_DELETE_OBJECT = true;
+  }
 
-if (process.env.ACTION_DELETE_OBJECT === "true") {
-  CONFIG.ACTION_DELETE_OBJECT = true;
-}
+  if (process.env.CALLBACK_URL) {
+    CONFIG.CALLBACK_URL = process.env.CALLBACK_URL;
+  }
+
+  if (process.env.MAX_ATTEMPTS) {
+    CONFIG.MAX_ATTEMPTS = process.env.MAX_ATTEMPTS;
+  }
+
+  if (process.env.MAX_ATTEMPT_DELAY_MSEC) {
+    CONFIG.MAX_ATTEMPT_DELAY_MSEC = process.env.MAX_ATTEMPT_DELAY_MSEC;
+  }
 
-if (process.env.CALLBACK_URL) {
-  CONFIG.CALLBACK_URL = process.env.CALLBACK_URL;
 }
 
+defaults();
+exports.defaults = defaults;
 exports.CONFIG = CONFIG;

lib/s3-handler.js

@@ -5,6 +5,7 @@ const CONFIG = require('./config').CONFIG;
 const utils = require('./utils');
 const scanii = require('./client');
 const pkg = require('../package.json');
+const crypto = require('crypto');
 
 
 /**
@@ -23,7 +24,9 @@ exports.handler = async (event, context, callback) => {
     assert.ok(CONFIG.KEY !== null, "api key cannot be null");
     assert.ok(CONFIG.SECRET !== null, "api secret cannot be null");
 
-    const scaniiClient = new scanii.ScaniiClient(CONFIG.KEY, CONFIG.SECRET, CONFIG.API_ENDPOINT);
+    const scaniiClient = new scanii.ScaniiClient(CONFIG.KEY, CONFIG.SECRET, CONFIG.API_ENDPOINT,
+      CONFIG.MAX_ATTEMPTS, CONFIG.MAX_ATTEMPT_DELAY_MSEC);
+
     const S3 = new AWS.S3({apiVersion: '2006-03-01'});
 
     // Get the object from the event and show its content type
@@ -51,11 +54,13 @@ exports.handler = async (event, context, callback) => {
     // signing request
     const signature = utils.generateSignature(bucket, key);
     console.log('using signature ' + signature);
+    const sha1sum = crypto.createHash('sha1');
 
+    // we obfuscate the object key since it can contain sensitive information we don't want stored:
     const metadata = {
       "signature": signature,
       "bucket": bucket,
-      "key": key
+      "key-sha1": sha1sum.update(key).digest('hex')
     };
 
     const submitResult = await scaniiClient.fetch(url, CONFIG.CALLBACK_URL, metadata);