From 5b1fca82bd378dabdd2cabe0efa8be4a1bf94388 Mon Sep 17 00:00:00 2001 From: makeros Date: Sun, 20 Jun 2021 07:57:55 +0200 Subject: [PATCH 1/4] feat(perf): remove lodash.pick from dependencies --- bench/bench.sh | 2 +- lib/stream-logs.js | 1 - lib/transformer/index.js | 2 +- lib/transport.js | 7 +++++-- package-lock.json | 5 ----- package.json | 1 - 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/bench/bench.sh b/bench/bench.sh index ffebcc6..0e2c765 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -1,2 +1,2 @@ #!/bin/bash -time node producer.js 1000000 | node --trace-uncaught ../index.js log -c ../__mocks__/custom-schema.json -v \ No newline at end of file +time node producer.js 1000000 | node --trace-uncaught ../index.js log -c ../__mocks__/custom-schema.json -v > /dev/null \ No newline at end of file diff --git a/lib/stream-logs.js b/lib/stream-logs.js index b5bcfab..1f03379 100644 --- a/lib/stream-logs.js +++ b/lib/stream-logs.js @@ -24,7 +24,6 @@ function getVerbose () { } function getTransport (opts) { - console.log('transport is prepared') const transport = new Transport(opts) return new Transform({ writableObjectMode: true, diff --git a/lib/transformer/index.js b/lib/transformer/index.js index 18a7e9a..78becd8 100644 --- a/lib/transformer/index.js +++ b/lib/transformer/index.js @@ -13,7 +13,7 @@ module.exports = function (opts) { function setMessageField (opts) { const _field = opts.messageField return function (data) { - data.msg = data[_field] !== undefined ? data[_field] : 'No msg property found or msg is empty' + data.msg = data[_field] || 'No msg property found or msg is empty' return data } } diff --git a/lib/transport.js b/lib/transport.js index fceb3df..c367dea 100644 --- a/lib/transport.js +++ b/lib/transport.js @@ -4,13 +4,16 @@ const crypto = require('crypto') const deflate = require('zlib').deflate const dgram = require('dgram') const EventEmitter = require('events').EventEmitter -const pick = require('lodash.pick') const utils = require('./utils') const Transport = function (opts) { const self = this - self.config = pick(opts, ['host', 'port', 'maxChunkSize']) + self.config = { + host: opts.host, + post: opts.port, + maxChunkSize: opts.maxChunkSize + } self.stringify = utils.stringify(opts) self.on('log', function (msg) { diff --git a/package-lock.json b/package-lock.json index 0b8f654..ec4a24f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5591,11 +5591,6 @@ "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" }, - "lodash.pick": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", - "integrity": "sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM=" - }, "lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", diff --git a/package.json b/package.json index b5c09b4..febb54b 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "fast-json-parse": "^1.0.3", "fast-json-stringify": "^2.4.2", "lodash.get": "^4.4.2", - "lodash.pick": "^4.4.0", "readable-stream": "^3.6.0", "split2": "^3.2.2" }, From 3b52d9f333ead165ce9c0cf9e103cdc5de5eae8e Mon Sep 17 00:00:00 2001 From: makeros Date: Sun, 20 Jun 2021 19:59:29 +0200 Subject: [PATCH 2/4] ci: use github actions for PR testing --- .circleci/config.yml | 74 ++++++++++++++++++++++++++++++++ .github/workflows/test_on_pr.yml | 1 + package.json | 5 ++- 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..3e46dd7 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,74 @@ +version: 2 +jobs: + # build: + # docker: + # - image: circleci/node:10.15.2 + # working_directory: ~/repo + + # steps: + # - checkout + + # # Download and cache dependencies + # - restore_cache: + # keys: + # - v1-dependencies-{{ checksum "package.json" }} + # # fallback to using the latest cache if no exact match is found + # - v1-dependencies- + + # - run: npm install + + # - save_cache: + # paths: + # - node_modules + # key: v1-dependencies-{{ checksum "package.json" }} + + # test: + # docker: + # - image: circleci/node:10.15.2 + + # working_directory: ~/repo + + # steps: + # - checkout + # - restore_cache: + # keys: + # - v1-dependencies-{{ checksum "package.json" }} + # - v1-dependencies- + # - run: npm test + + release: + docker: + - image: circleci/node:14.17.1 + + working_directory: ~/repo + + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies-{{ checksum "package.json" }} + - v1-dependencies- + - run: npx semantic-release + +workflows: + version: 2 + build_test__release: + jobs: + # - build + # - test: + # requires: + # - build + - release_approval: + type: approval + filters: + branches: + only: + - master + - release: + requires: + # - test + - release_approval + filters: + branches: + only: + - master diff --git a/.github/workflows/test_on_pr.yml b/.github/workflows/test_on_pr.yml index 1078d85..2d28386 100644 --- a/.github/workflows/test_on_pr.yml +++ b/.github/workflows/test_on_pr.yml @@ -27,6 +27,7 @@ jobs: restore-keys: | ${{ runner.OS }}-${{ matrix.node-version }}-node- ${{ runner.OS }}-${{ matrix.node-version }}- + - run: npm ci env: diff --git a/package.json b/package.json index febb54b..abb2dca 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "scripts": { "pretest": "standard", "test": "jest", - "semantic-release": "semantic-release" + "semantic-release": "semantic-release", + "lint": "standard" }, "repository": { "type": "git", @@ -45,6 +46,6 @@ "standard": "^16.0.3" }, "engines": { - "node": ">=10" + "node": ">=12" } } From e5f6282c1c139132aaf16bd25ecac615442f144c Mon Sep 17 00:00:00 2001 From: makeros Date: Wed, 23 Jun 2021 08:51:39 +0200 Subject: [PATCH 3/4] fix: improve transtort test --- __mocks__/udp-server.js | 8 ++-- lib/transport.js | 2 +- package.json | 3 +- test/gelf-transformer-transport.spec.js | 62 ++++++++----------------- 4 files changed, 28 insertions(+), 47 deletions(-) diff --git a/__mocks__/udp-server.js b/__mocks__/udp-server.js index a345c3c..d4327ae 100644 --- a/__mocks__/udp-server.js +++ b/__mocks__/udp-server.js @@ -14,9 +14,11 @@ function start () { return server } -function stop () { - server.close() - onMessage.mockReset() +function stop (cb) { + server.close(function () { + onMessage.mockReset() + cb && cb() + }) } module.exports = { diff --git a/lib/transport.js b/lib/transport.js index c367dea..54849b0 100644 --- a/lib/transport.js +++ b/lib/transport.js @@ -11,7 +11,7 @@ const Transport = function (opts) { self.config = { host: opts.host, - post: opts.port, + port: opts.port, maxChunkSize: opts.maxChunkSize } self.stringify = utils.stringify(opts) diff --git a/package.json b/package.json index abb2dca..30ee92b 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "pretest": "standard", "test": "jest", "semantic-release": "semantic-release", - "lint": "standard" + "lint": "standard", + "jest": "jest" }, "repository": { "type": "git", diff --git a/test/gelf-transformer-transport.spec.js b/test/gelf-transformer-transport.spec.js index e763dfe..ba22615 100644 --- a/test/gelf-transformer-transport.spec.js +++ b/test/gelf-transformer-transport.spec.js @@ -1,4 +1,4 @@ -/* globals describe, test, expect, afterEach, beforeEach */ +/* globals describe, it, expect, afterEach, beforeEach */ const cp = require('child_process') const path = require('path') const { unzip } = require('zlib') @@ -6,64 +6,42 @@ const mockServer = require(path.resolve('__mocks__/udp-server.js')) const gtPath = path.join(__dirname, '..', 'index.js') -function logsOutput (msg, level) { - return `{"level":${level},"time":1531171074631,"msg":"${msg}","pid":657,"hostname":"box","name":"app","v":1}` -} - describe('gelf-transform in transport mode', function () { let server + let gelfTransformer beforeEach(() => { + gelfTransformer = cp.spawn('node', [gtPath, 'log', '-t']) server = mockServer.start() }) - afterEach(() => { - mockServer.stop() + afterEach((done) => { + mockServer.stop(function () { + gelfTransformer.kill() + done() + }) }) - test('should send logs to the graylog server when transport is enabled', done => { - const gt = cp.spawn('node', [gtPath, 'log', '-t']) - + it('should send logs to the graylog server when transport is enabled', done => { + const result = [] + let callsCounter = 0 server.on('message', (msg, remote) => { unzip(msg, function (err, buffer) { if (err) { - gt.kill() return done(err) } - const result = buffer.toString() - expect(mockServer.onMessage).toBeCalledTimes(1) - expect(result).toEqual('{"version":"1.1","host":"box","short_message":"log","full_message":"log","timestamp":1531171074.631,"level":6,"facility":"app"}') - - gt.kill() - done() - }) - }) - - gt.stdin.write(logsOutput('log', '30') + '\n') - }) + result.push(buffer.toString()) + callsCounter++ - test('should not send logs to graylog', done => { - const gt = cp.spawn('node', [gtPath, 'log']) - const gtLog = cp.spawn('node', [gtPath, 'log', '-t']) - - server.on('message', (msg, remote) => { - unzip(msg, function (err, buffer) { - if (err) { - gt.kill() - gtLog.kill() - return done(err) + if (callsCounter === 2) { + expect(result[0]).toEqual('{"version":"1.1","host":"box","short_message":"log1","full_message":"log1","timestamp":1531171074.631,"level":6,"facility":"app"}') + expect(result[1]).toEqual('{"version":"1.1","host":"box","short_message":"log2","full_message":"log2","timestamp":1531171074.631,"level":6,"facility":"app"}') + done() } - - const result = buffer.toString() - expect(mockServer.onMessage).toBeCalledTimes(1) - expect(result).toEqual('{"version":"1.1","host":"box","short_message":"available","full_message":"available","timestamp":1531171074.631,"level":6,"facility":"app"}') - gt.kill() - gtLog.kill() - done() }) }) - gt.stdin.write(logsOutput('not_available', '30') + '\n') - gtLog.stdin.write(logsOutput('available', '30') + '\n') - }) + gelfTransformer.stdin.write('{"level":30,"time":1531171074631,"msg":"log1","pid":657,"hostname":"box","name":"app","v":1}' + '\n') + gelfTransformer.stdin.write('{"level":30,"time":1531171074631,"msg":"log2","pid":657,"hostname":"box","name":"app","v":1}' + '\n') + }, 10000) }) From 2438053a36fd53b2773f78f864f9a827918a915a Mon Sep 17 00:00:00 2001 From: arek czogala Date: Tue, 5 Oct 2021 16:49:35 +0200 Subject: [PATCH 4/4] wip --- .circleci/config.yml | 74 ------------------------- .github/workflows/test_on_pr.yml | 2 +- bench/producer.js | 2 +- lib/schema/gelf.json | 2 +- lib/transformer/standard-gelf.js | 4 +- package-lock.json | 12 +++- package.json | 2 +- test/gel-transformer-verbose.spec.js | 12 ++-- test/gelf-transformer-transport.spec.js | 8 +-- 9 files changed, 25 insertions(+), 93 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 3e46dd7..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,74 +0,0 @@ -version: 2 -jobs: - # build: - # docker: - # - image: circleci/node:10.15.2 - # working_directory: ~/repo - - # steps: - # - checkout - - # # Download and cache dependencies - # - restore_cache: - # keys: - # - v1-dependencies-{{ checksum "package.json" }} - # # fallback to using the latest cache if no exact match is found - # - v1-dependencies- - - # - run: npm install - - # - save_cache: - # paths: - # - node_modules - # key: v1-dependencies-{{ checksum "package.json" }} - - # test: - # docker: - # - image: circleci/node:10.15.2 - - # working_directory: ~/repo - - # steps: - # - checkout - # - restore_cache: - # keys: - # - v1-dependencies-{{ checksum "package.json" }} - # - v1-dependencies- - # - run: npm test - - release: - docker: - - image: circleci/node:14.17.1 - - working_directory: ~/repo - - steps: - - checkout - - restore_cache: - keys: - - v1-dependencies-{{ checksum "package.json" }} - - v1-dependencies- - - run: npx semantic-release - -workflows: - version: 2 - build_test__release: - jobs: - # - build - # - test: - # requires: - # - build - - release_approval: - type: approval - filters: - branches: - only: - - master - - release: - requires: - # - test - - release_approval - filters: - branches: - only: - - master diff --git a/.github/workflows/test_on_pr.yml b/.github/workflows/test_on_pr.yml index 2d28386..2b518bc 100644 --- a/.github/workflows/test_on_pr.yml +++ b/.github/workflows/test_on_pr.yml @@ -3,7 +3,7 @@ name: Test Pull Request into master branch on: pull_request: branches: - - master + - master jobs: test: diff --git a/bench/producer.js b/bench/producer.js index 21f8ad2..fe56ae6 100644 --- a/bench/producer.js +++ b/bench/producer.js @@ -1,4 +1,4 @@ -'strict' +'use strict' for (let i = 0; i < process.argv.slice(2)[0]; i++) { process.stdout.write('{"level":30,"time":1531171074631,"msg":"hello world","nested":{"mock":666},"test2":"red","pid":657,"hostname":"box","name":"app","v":1}' + '\n') diff --git a/lib/schema/gelf.json b/lib/schema/gelf.json index 2bfafc7..a6e003b 100644 --- a/lib/schema/gelf.json +++ b/lib/schema/gelf.json @@ -6,7 +6,7 @@ "host": { "type": "string" }, "short_message": { "type": "string" }, "full_message": { "type": "string" }, - "timestamp": { "type": "integer" }, + "timestamp": { "type": "number" }, "level": { "type": "integer" }, "facility": { "type": "string" } } diff --git a/lib/transformer/standard-gelf.js b/lib/transformer/standard-gelf.js index 889b6a8..2c020ea 100644 --- a/lib/transformer/standard-gelf.js +++ b/lib/transformer/standard-gelf.js @@ -3,10 +3,10 @@ const utils = require('./../utils') module.exports = function (data) { return { version: '1.1', - host: data.hostname, + host: data.host, short_message: data.msg.substring(0, 65), full_message: data.msg, - timestamp: data.time / 1000, + timestamp: data.timestamp, level: utils.pinoLevelToSyslogLevel(data.level), facility: data.name // deprecated } diff --git a/package-lock.json b/package-lock.json index ec4a24f..6d0e1a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3217,12 +3217,13 @@ "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" }, "fast-json-stringify": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-2.4.2.tgz", - "integrity": "sha512-hXNC8Hj5ZYf0PRt67JyLwrw72XOIUSOj8IYXk8w8kotBox02L08Dvz6c8IMpCUXof1H+dLlBz1aszGqP9xmrBw==", + "version": "2.7.6", + "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-2.7.6.tgz", + "integrity": "sha512-ezem8qpAgpad6tXeUhK0aSCS8Fi2vjxTorI9i5M+xrq6UUbTl7/bBTxL1SjRI2zy+qpPkdD4+UblUCQdxRpvIg==", "requires": { "ajv": "^6.11.0", "deepmerge": "^4.2.2", + "rfdc": "^1.2.0", "string-similarity": "^4.0.1" }, "dependencies": { @@ -6815,6 +6816,11 @@ "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, + "rfdc": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" + }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", diff --git a/package.json b/package.json index 30ee92b..a719d16 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "dependencies": { "commander": "^3.0.1", "fast-json-parse": "^1.0.3", - "fast-json-stringify": "^2.4.2", + "fast-json-stringify": "2.7.6", "lodash.get": "^4.4.2", "readable-stream": "^3.6.0", "split2": "^3.2.2" diff --git a/test/gel-transformer-verbose.spec.js b/test/gel-transformer-verbose.spec.js index a0828b1..6761c03 100644 --- a/test/gel-transformer-verbose.spec.js +++ b/test/gel-transformer-verbose.spec.js @@ -7,7 +7,7 @@ jest.mock('dgram') const pgPath = path.join(__dirname, '..', 'index.js') const consoleOutput = function (msg, level) { - return `{"level":${level},"time":1531171074631,"msg":"${msg}","pid":657,"hostname":"box","name":"app","v":1}` + return `{"level":${level},"timestamp":1531171074,"msg":"${msg}","pid":657,"hostname":"box","name":"app","v":1}` } describe('gelf-transform in verbose mode', function () { @@ -37,7 +37,7 @@ describe('gelf-transform in verbose mode', function () { pg.stdin.end('this is not json\n') }) - test('logger output output is transformed to gelf output', done => { + test.only('logger output is transformed to gelf output', done => { const pg = cp.spawn('node', [pgPath, 'log', '-v']) pg.stdout.on('data', dataSpy) @@ -49,7 +49,7 @@ describe('gelf-transform in verbose mode', function () { done() }) - pg.stdin.end(consoleOutput('hello world', 30) + '\n') + pg.stdin.end(consoleOutput('hello world 666', 30) + '\n') }) test('short message is trimmed down', done => { @@ -71,7 +71,7 @@ describe('gelf-transform in verbose mode', function () { test('logger output with custom fields (also nested) is transformed to gelf output', done => { const pg = cp.spawn('node', [pgPath, 'log', '-c', '__mocks__/custom-schema.json', '-v']) - const consoleCustomOutput = '{"level":30,"time":1531171074631,"msg":"hello world","nested":{"mock":666},"test2":"red","pid":657,"hostname":"box","name":"app","v":1}' + const consoleCustomOutput = '{"level":30,"time":1531171074.631,"msg":"hello world","nested":{"mock":666},"test2":"red","pid":657,"hostname":"box","name":"app","v":1}' pg.stdout.on('data', dataSpy) @@ -87,7 +87,7 @@ describe('gelf-transform in verbose mode', function () { test('logger output with custom fields should ignore mismatched type fields', done => { const pg = cp.spawn('node', [pgPath, 'log', '-c', '__mocks__/custom-schema.json', '-v']) - const consoleCustomOutput = '{"level":30,"time":1531171074631,"msg":"hello world","nested":{"mock":"this_should_be_a_number"},"test2":222,"pid":657,"hostname":"box","name":"app","v":1}' + const consoleCustomOutput = '{"level":30,"time":1531171074.631,"msg":"hello world","nested":{"mock":"this_should_be_a_number"},"test2":222,"pid":657,"hostname":"box","name":"app","v":1}' pg.stdout.on('data', dataSpy) @@ -148,7 +148,7 @@ describe('gelf-transform in verbose mode', function () { done() }) - pg.stdin.end('{"level":30,"time":1531171074631,"pid":657,"hostname":"box","name":"app","v":1}\n') + pg.stdin.end('{"level":30,"time":1531171074.631,"pid":657,"hostname":"box","name":"app","v":1}\n') }) describe('log levels transformations', () => { diff --git a/test/gelf-transformer-transport.spec.js b/test/gelf-transformer-transport.spec.js index ba22615..ab4bba8 100644 --- a/test/gelf-transformer-transport.spec.js +++ b/test/gelf-transformer-transport.spec.js @@ -34,14 +34,14 @@ describe('gelf-transform in transport mode', function () { callsCounter++ if (callsCounter === 2) { - expect(result[0]).toEqual('{"version":"1.1","host":"box","short_message":"log1","full_message":"log1","timestamp":1531171074.631,"level":6,"facility":"app"}') - expect(result[1]).toEqual('{"version":"1.1","host":"box","short_message":"log2","full_message":"log2","timestamp":1531171074.631,"level":6,"facility":"app"}') + expect(result[0]).toEqual('{"version":"1.1","host":"box","short_message":"log1","full_message":"log1","timestamp":1531171074,"level":6,"facility":"app"}') + expect(result[1]).toEqual('{"version":"1.1","host":"box","short_message":"log2","full_message":"log2","timestamp":1531171074,"level":6,"facility":"app"}') done() } }) }) - gelfTransformer.stdin.write('{"level":30,"time":1531171074631,"msg":"log1","pid":657,"hostname":"box","name":"app","v":1}' + '\n') - gelfTransformer.stdin.write('{"level":30,"time":1531171074631,"msg":"log2","pid":657,"hostname":"box","name":"app","v":1}' + '\n') + gelfTransformer.stdin.write('{"level":30,"time":1531171074,"msg":"log1","pid":657,"hostname":"box","name":"app","v":1}' + '\n') + gelfTransformer.stdin.write('{"level":30,"time":1531171074,"msg":"log2","pid":657,"hostname":"box","name":"app","v":1}' + '\n') }, 10000) })