Merge pull request #12 from revolut-engineering/dpl-267-fix-broken-fetches

Fix large fetches, switch away from batch execution

Author: bruwozniak

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Change Log
 
+
+## 0.0.7
+
+Fix websocket max message size leading to broken fetches
+Switch away from batch mode
+Always show full schema
+Fix issue with global connection state
+
 ## 0.0.6
 
 Fix pagination, fetch up to 1000 records from queries

package.json

@@ -2,7 +2,7 @@
   "name": "exasol-driver",
   "displayName": "Exasol Driver",
   "description": "Exasol Driver for SQLTools",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "engines": {
     "vscode": "^1.42.0"
   },

src/ls/driver.ts

@@ -9,6 +9,7 @@ import { zipObject, range, Dictionary } from 'lodash';
 import Exasol from './wsjsapi';
 import keywordsCompletion from './keywords';
 import LRUCache from 'lru-cache';
+import { IClientConfig } from 'websocket';
 
 // DriverLib type is any since the connection object obtained from Exasol is a plain JS object
 type DriverLib = any;
@@ -43,7 +44,7 @@ type QueryFetchResult = {
 }
 
 const MAX_RESULTS = 1000 // rows
-const FETCH_SIZE = 1000000 // 1MiB (bytes)
+const FETCH_SIZE = 4 * 1024 * 1024 // 4MB (bytes)
 const QUERY_CACHE_SIZE = 100 // queries count
 const QUERY_CACHE_AGE = 1000 * 60 * 10 // 10 minutes (ms)
 
@@ -73,7 +74,8 @@ export default class ExasolDriver extends AbstractDriver<DriverLib, DriverOption
       Exasol.call({}, // we must pass a new thisArg object each time as connection state is kept there and we might spawn multiple connections
         `ws://${this.credentials.server}:${this.credentials.port}`, this.credentials.username, this.credentials.password,
         resolve,
-        this.rejectErr(reject))
+        this.rejectErr(reject),
+        <IClientConfig>{ maxReceivedFrameSize: 2 * FETCH_SIZE })
     ).then(db =>
       new Promise((resolve, reject) =>
         db.com({
@@ -111,20 +113,20 @@ export default class ExasolDriver extends AbstractDriver<DriverLib, DriverOption
     const db = await this.open();
     const splitQueries = parse(queries);
 
-    const responseData: QueryResponse = await this.queue.add(
-      () => new Promise(
-        (resolve, reject) => db.com({ 'command': 'executeBatch', 'sqlTexts': splitQueries },
-          resolve,
-          this.rejectErr(reject))
-      )
-    );
-    const res = [];
-    for (let index = 0; index < responseData.results.length; index++) {
-      const result = responseData.results[index];
+    const responses: QueryResponse[] = await Promise.all<QueryResponse>(
+      splitQueries.map((query) => this.queue.add(() =>
+        new Promise((resolve, reject) =>
+          db.com({ 'command': 'execute', 'sqlText': query }, resolve, this.rejectErr(reject))
+        )
+      )));
+
+    const res: NSDatabase.IResult[] = [];
+    for (let index = 0; index < responses.length; index++) {
+      const result = responses[index].results[0];
       if (result.resultType === 'rowCount') {
         const message = `Query ok with ${result.rowCount} rows affected`
         this.log.info(message)
-        res.push(<NSDatabase.IResult>{
+        res.push({
           cols: [],
           connId: this.getId(),
           messages: [{ date: new Date(), message: message }],

src/ls/wsjsapi.js

@@ -302,7 +302,7 @@ var json_parse = (function () {
     };
 }());
 
-var Exasol = function(url, user, pass, onconnect, onerror) {
+var Exasol = function(url, user, pass, onconnect, onerror, websocketConfig) {
     var context = this;
     context.onerror = onerror;
     context.sessionId = "-1";
@@ -489,7 +489,7 @@ var Exasol = function(url, user, pass, onconnect, onerror) {
     };
 
     context.inwork = false;
-    context.connection = new WebSocket(url);
+    context.connection = new WebSocket(url, null, null, null, null, websocketConfig);
     context.connection.onerror = function(err) {
         onerror('Error connecting to "' + url + '"');
     };