test: add tests for truncated utf8 sequences

If a source gives us an invalid utf8 sequence, ensure that it is
replaced by a Unicode replacement character as expected.

One of the semver-major changes in node 8 was to change string_decoder
behaviour to better align with the implementation in v8 itself. Now that
we have tests for the handling of incomplete utf8 sequences, this
actually affects us. To account for this, use a simple version check to
determine the expected string output of our bad-utf8 tests.

For background see nodejs/node@24ef1e6775

Signed-off-by: Ryan Graham <[email protected]>

Author: rmg

test/fixtures/basic.in

@@ -1,4 +1,5 @@
 one
 two
 three
+
 four

test/test-bad-utf8.js

@@ -0,0 +1,62 @@
+var tap = require('tap');
+var Log = require('../');
+
+tap.test('truncated utf8', function(t) {
+  var slt = Log();
+  var input = [
+    Buffer.from('good line\n'),
+    Buffer.from('good line\n'),
+    Buffer.from('good line\n'),
+    Buffer.from('good line\n'),
+    Buffer.from([
+      // an incomplete utf8 sequence (3/4 bytes)
+      0xf0, // byte 1 of 4 marker
+      0xbf, // byte 2 of 4 marker
+      0xbf, // byte 3 of 4 marker
+    ]),
+  ];
+  var expected = Buffer.concat([
+    Buffer.from('good line\n'),
+    Buffer.from('good line\n'),
+    Buffer.from('good line\n'),
+    Buffer.from('good line\n'),
+    Buffer.from([
+      0xef, 0xbf, 0xbd, // single replacement character
+      0x0a,             // trailing newline adde by strong-log-transformer
+    ]),
+  ]);
+  var received = '';
+
+  if (/^v(4|6)\./.test(process.version)) {
+    expected = Buffer.concat([
+      Buffer.from('good line\n'),
+      Buffer.from('good line\n'),
+      Buffer.from('good line\n'),
+      Buffer.from('good line\n'),
+      Buffer.from([
+        // prior to node 8 each byte of an invalid utf8 sequence would be
+        // replaced by a UTF replacement character. For more details, see
+        // https://github.com/nodejs/node/commit/24ef1e6775
+        0xef, 0xbf, 0xbd, // replacement character
+        0xef, 0xbf, 0xbd, // replacement character
+        0xef, 0xbf, 0xbd, // replacement character
+        0x0a,             // trailing newline adde by strong-log-transformer
+      ]),
+    ]);
+  }
+  slt.on('data', function(buf) {
+    t.comment(buf);
+    if (Buffer.isBuffer(buf)) {
+      received += buf.toString('utf8');
+    } else if (buf !== null) {
+      received += buf;
+    }
+  });
+  slt.on('end', function() {
+    var expectedStr = expected.toString('utf8');
+    t.same(received, expectedStr, 'output is input + trailing newline');
+    t.end();
+  });
+  slt.write(Buffer.concat(input));
+  slt.end();
+});