add tabular.data.pageSize default config option support (#67)

and reorganize table view and tabulator table config vars in tableView.js

Author: RandomFractals

package.json

@@ -163,7 +163,19 @@
 					}
 				]
 			}
-		]
+		],
+		"configuration": {
+			"title": "Tabular Data Viewer",
+			"type": "object",
+			"properties": {
+				"tabular.data.pageSize": {
+					"type": "integer",
+					"enum": [1000, 10000, 100000],
+					"default": "100000",
+					"description": "Default data page size to use for loading and displaying large datasets."
+				}
+			}
+		}
 	},
 	"scripts": {
 		"vscode:prepublish": "npm run package",

src/configuration/settings.ts

@@ -0,0 +1,6 @@
+/**
+ * Tabular data view configuraiton settings.
+ */
+export enum Settings {
+	dataPageSize = 'pageSize'
+}

src/views/tableView.ts

@@ -15,15 +15,18 @@ import {
 
 import * as fs from 'fs';
 import * as path from 'path';
+import { Stream } from 'stream';
+
+import * as config from '../configuration/configuration';
 import * as fileUtils from '../utils/fileUtils';
 
 import { FileInfo } from './fileInfo';
 import { FileTypes } from './fileTypes';
 import { ViewTypes } from './viewTypes';
 import { statusBar } from './statusBar';
 
-import { Stream } from 'stream';
 import { ViewCommands } from '../commands/viewCommands';
+import { Settings } from '../configuration/Settings';
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 const {Table} = require('tableschema');
@@ -55,9 +58,6 @@ export class TableView {
 
 
   // TODO: move the settings below to tabular data viewer config options later
-  // default page data size for incremental data loading into table view
-  private readonly _pageDataSize: number = 100000;
-
   // infer table schema rows sample size limit
   private readonly _inferDataSize = 100;
 
@@ -279,14 +279,14 @@ export class TableView {
     dataStream.on('data', (row: any) => {
       tableRows.push(row);
       rowCount++;
-      if ((rowCount % (this._pageDataSize * 10)) === 0) {
+      if ((rowCount % (this.dataPageSize * 10)) === 0) {
         console.log(`tabular.data.view:refresh(): parsing rows ${rowCount.toLocaleString()}+ ...`);
         if (this.visible) {
           statusBar.showMessage(`Parsing rows ${rowCount.toLocaleString()}+`);
         }
       }
 
-      if (rowCount === this._pageDataSize) {
+      if (rowCount === this.dataPageSize) {
         // send initial set of data rows to table view for display
         this.loadData(tableRows);
         // TODO: add pause/resume stream later
@@ -309,13 +309,13 @@ export class TableView {
         statusBar.totalRows = tableRows.length;
       }
 
-      if (tableRows.length < this._pageDataSize) {
+      if (tableRows.length < this.dataPageSize) {
         // load first page of data
         this.loadData(tableRows);
         // clear data loading status bar message
         statusBar.showMessage('');
       }
-      else if (tableRows.length > this._pageDataSize) {
+      else if (tableRows.length > this.dataPageSize) {
         // load remaining table rows
         const dataPageIndex: number = 1;
         this.addData(dataPageIndex);
@@ -344,7 +344,7 @@ export class TableView {
     }
 
     // send initial set of data rows to table view for display
-    const nextRows: number = Math.min(this._pageDataSize, this._totalRows);
+    const nextRows: number = Math.min(this.dataPageSize, this._totalRows);
     const initialDataRows: Array<any> = tableRows.slice(0, nextRows);
     this.webviewPanel.webview.postMessage({
       command: 'refresh',
@@ -353,7 +353,8 @@ export class TableView {
       tableConfig: this._tableConfig,
       tableSchema: this._tableSchema,
       totalRows: this._totalRows,
-      tableData: initialDataRows
+      tableData: initialDataRows,
+      dataPageSize: this.dataPageSize
     });
   }
 
@@ -363,7 +364,7 @@ export class TableView {
    * @param dataPage Requested data page index for loading the next set of data rows.
    */
   public async addData(dataPage: number): Promise<void> {
-    let nextRows: number = dataPage * this._pageDataSize;
+    let nextRows: number = dataPage * this.dataPageSize;
     if (this._loadedDataPage <= dataPage && nextRows < this._totalRows) {
       // console.log(`tabular.data.view:addData(): loading rows ${nextRows.toLocaleString()}+ ...`);
       if (this.visible) {
@@ -372,7 +373,7 @@ export class TableView {
 
       // get the next set of data rows to load in table view
       const dataRows: Array<any> =
-        this._tableData.slice(nextRows, Math.min(nextRows + this._pageDataSize, this._totalRows));
+        this._tableData.slice(nextRows, Math.min(nextRows + this.dataPageSize, this._totalRows));
 
       // increment next rows for data loading status update
       nextRows += dataRows.length;
@@ -563,6 +564,13 @@ export class TableView {
     return delimiter;
   }
 
+  /**
+   * Gets data page size configuration setting.
+   */
+  get dataPageSize(): number {
+    return <number>config.get(Settings.dataPageSize);
+  }
+
   /**
    * Creates webview html content for the webview panel display.
    * 

web/scripts/tableView.js

@@ -10,6 +10,7 @@ let saveDataFileName = '';
 
 // table view elements
 let tableContainer, table, progressRing, saveFileTypeSelector, tablePageSelector;
+const toolbarHeight = 40; // table view toolbar height offset
 
 // table view vars
 let tableConfig = {};
@@ -19,27 +20,24 @@ let tableData = [];
 let loadedRows = 0;
 let totalRows = 0;
 let loadedDataPage = 0;
+let dataPageSize = 100000;
 let viewState = {
   tableConfig: tableConfig
 };
 
-// table view settings
-const toolbarHeight = 40; // table view toolbar height offset
-const autoResize = true;
+// tabulator table settings
 let autoColumns = true;
+const autoResize = true;
 const enableClipboard = true; // enable clipboard copy and paste
 const clipboardPasteAction = 'replace';
 const movableColumns = true;
 const movableRows = true;
 const selectableRows =  true;
-
 const pagination = false;
 const paginationSize = 1000;
-const pageDataSize = 100000;
 const pageSizes = [100, 1000, 10000, 100000];
-
-const reactiveData = false;
 const renderVerticalBuffer = 300; // virtual view buffer height in px for redraw on scroll
+const reactiveData = false;
 
 // tabulator debug options
 const debugInvalidOptions = true; // tabulator warnings
@@ -176,6 +174,7 @@ function updateViewState(tableInfo) {
   tableSchema = tableInfo.tableSchema;
   tableData = tableInfo.tableData;
   totalRows = tableInfo.totalRows;
+  dataPageSize = tableInfo.dataPageSize;
 
   if (tableInfo.tableConfig.columns) {
     // update table config
@@ -410,7 +409,7 @@ function onTableBuilt () {
  */
 function getNextDataPage() {
   const nextDataPage = loadedDataPage + 1;
-  if (loadedRows < totalRows && (nextDataPage * pageDataSize) < totalRows) {
+  if (loadedRows < totalRows && (nextDataPage * dataPageSize) < totalRows) {
     progressRing.style.visibility = 'visible';
     vscode.postMessage({
       command: 'addData',
@@ -460,8 +459,8 @@ function addData(table, dataRows, dataPageIndex) {
 function showDataPage() {
   let dataPageIndex = Number(tablePageSelector.value);
   // console.log('tableView.showDataPage(): loading data page:', dataPageIndex);
-  const pageStart = (dataPageIndex * pageDataSize);
-  const pageData = tableData.slice(pageStart, Math.min(pageStart + pageDataSize, totalRows));
+  const pageStart = (dataPageIndex * dataPageSize);
+  const pageData = tableData.slice(pageStart, Math.min(pageStart + dataPageSize, totalRows));
   table.clearData();
   table.replaceData(pageData);
 }