Merge pull request #1 from thomasgauvin/deoptimise-large-rows

Deoptimise large rows

Author: thomasgauvin

lib/commands/query.js

@@ -8,6 +8,7 @@ const Readable = require('stream').Readable;
 const Command = require('./command.js');
 const Packets = require('../packets/index.js');
 const getTextParser = require('../parsers/text_parser.js');
+const staticParser = require('../parsers/static_text_parser.js');
 const ServerStatus = require('../constants/server_status.js');
 
 const EmptyPacket = new Packets.Packet(0, Buffer.allocUnsafe(4), 0, 4);
@@ -212,7 +213,11 @@ class Query extends Command {
     if (this._receivedFieldsCount === this._fieldCount) {
       const fields = this._fields[this._resultIndex];
       this.emit('fields', fields);
-      this._rowParser = new (getTextParser(fields, this.options, connection.config))(fields);
+      if (this.options.useStaticParser) {
+        this._rowParser = staticParser(fields, this.options, connection.config);
+      } else {
+        this._rowParser = new (getTextParser(fields, this.options, connection.config))(fields);
+      }
       return Query.prototype.fieldsEOF;
     }
     return Query.prototype.readField;

lib/connection_config.js

@@ -59,6 +59,7 @@ const validOptions = {
   typeCast: 1,
   uri: 1,
   user: 1,
+  useStaticParser: 1,
   // These options are used for Pool
   connectionLimit: 1,
   maxIdle: 1,
@@ -147,6 +148,7 @@ class ConnectionConfig {
     this.nestTables =
       options.nestTables === undefined ? undefined : options.nestTables;
     this.typeCast = options.typeCast === undefined ? true : options.typeCast;
+    this.useStaticParser = options.useStaticParser || false;
     if (this.timezone[0] === ' ') {
       // "+" is a url encoded char for space so it
       // gets translated to space when giving a

lib/parsers/static_text_parser.js

@@ -0,0 +1,142 @@
+'use strict';
+
+const Types = require('../constants/types.js');
+const Charsets = require('../constants/charsets.js');
+const helpers = require('../helpers');
+
+const typeNames = [];
+for (const t in Types) {
+  typeNames[Types[t]] = t;
+}
+
+function readField({ packet, type, charset, encoding, config, options }) {
+  const supportBigNumbers =
+    options.supportBigNumbers || config.supportBigNumbers;
+  const bigNumberStrings = options.bigNumberStrings || config.bigNumberStrings;
+  const timezone = options.timezone || config.timezone;
+  const dateStrings = options.dateStrings || config.dateStrings;
+
+  switch (type) {
+    case Types.TINY:
+    case Types.SHORT:
+    case Types.LONG:
+    case Types.INT24:
+    case Types.YEAR:
+      return packet.parseLengthCodedIntNoBigCheck();
+    case Types.LONGLONG:
+      if (supportBigNumbers && bigNumberStrings) {
+        return packet.parseLengthCodedIntString();
+      }
+      return packet.parseLengthCodedInt(supportBigNumbers);
+    case Types.FLOAT:
+    case Types.DOUBLE:
+      return packet.parseLengthCodedFloat();
+    case Types.NULL:
+    case Types.DECIMAL:
+    case Types.NEWDECIMAL:
+      if (config.decimalNumbers) {
+        return packet.parseLengthCodedFloat();
+      }
+      return packet.readLengthCodedString("ascii");
+    case Types.DATE:
+      if (helpers.typeMatch(type, dateStrings, Types)) {
+        return packet.readLengthCodedString("ascii");
+      }
+      return packet.parseDate(timezone);
+    case Types.DATETIME:
+    case Types.TIMESTAMP:
+      if (helpers.typeMatch(type, dateStrings, Types)) {
+        return packet.readLengthCodedString('ascii');
+      }
+      return packet.parseDateTime(timezone);
+    case Types.TIME:
+      return packet.readLengthCodedString('ascii');
+    case Types.GEOMETRY:
+      return packet.parseGeometryValue();
+    case Types.JSON:
+      // Since for JSON columns mysql always returns charset 63 (BINARY),
+      // we have to handle it according to JSON specs and use "utf8",
+      // see https://github.com/sidorares/node-mysql2/issues/409
+      return JSON.parse(packet.readLengthCodedString('utf8'));
+    default:
+      if (charset === Charsets.BINARY) {
+        return packet.readLengthCodedBuffer();
+      }
+      return packet.readLengthCodedString(encoding);
+  }
+}
+
+function compile(fields, options, config) {
+  if (
+    typeof config.typeCast === 'function' &&
+    typeof options.typeCast !== 'function'
+  ) {
+    options.typeCast = config.typeCast;
+  }
+}
+
+function createTypecastField(field, packet) {
+  return {
+    type: typeNames[field.columnType],
+    length: field.columnLength,
+    db: field.schema,
+    table: field.table,
+    name: field.name,
+    string: function(encoding = field.encoding) {
+      if (field.columnType === Types.JSON && encoding === field.encoding) {
+        // Since for JSON columns mysql always returns charset 63 (BINARY),
+        // we have to handle it according to JSON specs and use "utf8",
+        // see https://github.com/sidorares/node-mysql2/issues/1661
+        console.warn(`typeCast: JSON column "${field.name}" is interpreted as BINARY by default, recommended to manually set utf8 encoding: \`field.string("utf8")\``);
+      }
+      return packet.readLengthCodedString(encoding);
+    },
+    buffer: function() {
+      return packet.readLengthCodedBuffer();
+    },
+    geometry: function() {
+      return packet.parseGeometryValue();
+    }
+  };
+}
+
+function getTextParser(fields, options, config) {
+  return {
+    next(packet, fields, options) {
+      const result = options.rowsAsArray ? [] : {};
+      for (let i = 0; i < fields.length; i++) {
+        const field = fields[i];
+        const typeCast = options.typeCast ? options.typeCast : config.typeCast;
+        const next = () => {
+          return readField({
+            packet,
+            type: field.columnType,
+            encoding: field.encoding,
+            charset: field.characterSet,
+            config,
+            options
+          })
+        }
+        let value;
+        if (typeof typeCast === 'function') {
+          value = typeCast(createTypecastField(field, packet), next);
+        } else {
+          value = next();
+        }
+        if (options.rowsAsArray) {
+          result.push(value);
+        } else if (options.nestTables) {
+          if (!result[field.table]) {
+            result[field.table] = {};
+          }
+          result[field.table][field.name] = value;
+        } else {
+          result[field.name] = value;
+        }
+      }
+      return result;
+    }
+  }
+}
+
+module.exports = getTextParser;

test/integration/regressions/test-#433.test.cjs

@@ -64,9 +64,12 @@ connection.query(
 );
 
 /* eslint quotes: 0 */
-const expectedError =
+const expectedErrorMysql =
   "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '`МояТаблица' at line 1";
 
+const expectedErrorMariaDB =
+  "You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '`МояТаблица' at line 1";
+
 process.on('exit', () => {
   testRows.map((tRow, index) => {
     const cols = testFields;
@@ -76,6 +79,10 @@ process.on('exit', () => {
     assert.equal(aRow[cols[2]], tRow[2]);
     assert.equal(aRow[cols[3]], tRow[3]);
   });
-
-  assert.equal(actualError, expectedError);
+  
+  if (connection._handshakePacket.serverVersion.match(/MariaDB/)) {
+    assert.equal(actualError, expectedErrorMariaDB);
+  } else {
+    assert.equal(actualError, expectedErrorMysql);
+  }
 });

typings/mysql/lib/Connection.d.ts

@@ -323,6 +323,8 @@ export interface ConnectionOptions {
 
   waitForConnections?: boolean;
 
+  useStaticParser?: boolean;
+
   authPlugins?: {
     [key: string]: AuthPlugin;
   };