user-agent (#13)

Author: EmptyInfinity

.circleci/config.yml

@@ -72,14 +72,14 @@ commands:
 jobs:
   test:
     docker:
-      - image: circleci/node:14-stretch
+      - image: circleci/node:16-stretch
     steps:
       - checkout
       - node/install:
           node-version: << pipeline.parameters.node-version >>
       - run:
           name: Audit Dependencies
-          command: npm audit --audit-level=high
+          command: npm audit --production --audit-level=high
       - node/install-packages:
           cache-path: ./node_modules
           override-ci-command: npm install
@@ -88,7 +88,7 @@ jobs:
           command: npm test
   build:
     docker:
-      - image: circleci/node:14-stretch
+      - image: circleci/node:16-stretch
         user: root
     steps:
       - checkout

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.1.5 (August 26, 2022)
+
+* Update Sailor version to 2.6.29
+* Get rid of vulnerabilities in dependencies
+* Update component-commons-library version to 3.0.2
+
 ## 3.1.4 (May 06, 2022)
 * Fix memory leak on `Emit Batch` behavior for `Read CSV attachment` action
 

component.json

@@ -2,7 +2,7 @@
   "title": "CSV",
   "description": "A comma-separated values (CSV) file stores tabular data (numbers and text) in plain-text form",
   "docsUrl": "https://github.com/elasticio/csv-component",
-  "version": "3.1.4",
+  "version": "3.1.5",
   "actions": {
     "read_action": {
       "main": "./lib/actions/read.js",
@@ -85,4 +85,4 @@
       "dynamicMetadata": true
     }
   }
-}
+}

lib/actions/read.js

@@ -6,9 +6,9 @@ const { messages } = require('elasticio-node')
 const stream = require('stream')
 const util = require('util')
 const papa = require('papaparse')
+const { getUserAgent } = require('../util');
 
 const pipeline = util.promisify(stream.pipeline);
-const attachmentProcessor = new AttachmentProcessor()
 
 // transform array to obj, for example:
 // ['aa', 'bb', 'cc'] => {column0: 'aa', column1: 'bb', column2: 'cc'}
@@ -72,6 +72,7 @@ async function readCSV(msg, cfg) {
   let dataStream
   const parseStream = papa.parse(papa.NODE_STREAM_INPUT, parseOptions)
 
+  const attachmentProcessor = new AttachmentProcessor(getUserAgent(), msg.id);
   try {
     dataStream = await attachmentProcessor.getAttachment(body.url, 'stream')
     this.logger.info('File received, trying to parse CSV')

lib/actions/write.js

@@ -4,14 +4,23 @@ no-param-reassign */
 const { AttachmentProcessor } = require('@elastic.io/component-commons-library')
 const { messages } = require('elasticio-node')
 const papa = require('papaparse')
+const { Readable } = require('stream');
+const { getUserAgent } = require('../util');
+
+const formStream = (data) => {
+  const stream = new Readable();
+  stream.push(data);
+  stream.push(null);
+  return stream;
+}
 
 const TIMEOUT_BETWEEN_EVENTS = process.env.TIMEOUT_BETWEEN_EVENTS || 10000; // 10s;
 
 const rawData = []
 let timeout
 
-async function proceedData(data, cfg) {
-  let csvString
+async function proceedData(data, msg, cfg) {
+  let csvString;
   const delimiter = cfg.separator ? cfg.separator : ','
 
   const unparseOptions = {
@@ -29,30 +38,31 @@ async function proceedData(data, cfg) {
         return filtered
       })
       return result
-    })
+    });
     csvString = papa.unparse({
       fields,
       data: orderedData
-    }, unparseOptions)
+    }, unparseOptions);
   } else {
-    csvString = papa.unparse(data, unparseOptions)
+    csvString = papa.unparse(data, unparseOptions);
   }
+  const getAttachment = () => formStream(csvString);
 
   if (!cfg.uploadToAttachment) {
     await this.emit('data', messages.newMessageWithBody({ csvString }))
     this.logger.info(`Complete, memory used: ${process.memoryUsage().heapUsed / 1024 / 1024} Mb`)
     return
   }
 
-  const attachmentProcessor = new AttachmentProcessor()
-  let uploadResult
+  const attachmentProcessor = new AttachmentProcessor(getUserAgent(), msg.id)
+  let attachmentId;
   try {
-    uploadResult = await attachmentProcessor.uploadAttachment(csvString, 'stream')
+    attachmentId = await attachmentProcessor.uploadAttachment(getAttachment)
   } catch (err) {
     this.logger.error(`Upload attachment failed: ${err}`)
     this.emit('error', `Upload attachment failed: ${err}`)
   }
-  const attachmentUrl = `${uploadResult.config.url}${uploadResult.data.objectId}?storage_type=maester`;
+  const attachmentUrl = attachmentProcessor.getMaesterAttachmentUrlById(attachmentId)
   const body = {
     attachmentUrl,
     type: '.csv',
@@ -91,13 +101,13 @@ async function writeCSV(msg, cfg) {
   // if not array - create array from all fn calls and send data to proceedData
   if (Array.isArray(body.items)) {
     this.logger.info('input metadata is array. Proceed with data ')
-    await proceedData.call(this, body.items, cfg)
+    await proceedData.call(this, body.items, msg, cfg)
   } else {
     rawData.push(body.items)
     if (timeout) clearTimeout(timeout)
     timeout = setTimeout(() => {
       this.logger.info(`input metadata is object. Array creation (wait up to ${TIMEOUT_BETWEEN_EVENTS}ms for more records)`)
-      proceedData.call(this, rawData, cfg)
+      proceedData.call(this, rawData, msg, cfg)
     }, TIMEOUT_BETWEEN_EVENTS)
   }
 }

lib/util.js

@@ -1,4 +1,13 @@
 const papa = require('papaparse');
+const packageJson = require('../package.json');
+const compJson = require('../component.json');
+
+function getUserAgent() {
+  const { name: compName } = packageJson;
+  const { version: compVersion } = compJson;
+  const libVersion = packageJson.dependencies['@elastic.io/component-commons-library'];
+  return `${compName}/${compVersion} component-commons-library/${libVersion}`;
+}
 
 function getItemsProperties(cfg) {
   const properties = {};
@@ -65,5 +74,6 @@ function getOut(cfg) {
   return out;
 }
 
+module.exports.getUserAgent = getUserAgent;
 module.exports.getItemsProperties = getItemsProperties;
 module.exports.getOut = getOut;