Skip to content

Commit 15829d6

Browse files
RandomFractalsRandomFractals
committed
#142 switched to async data loading in file utils & data.manger & providers impl. apis
1 parent 849ee20 commit 15829d6

File tree

9 files changed

+105
-103
lines changed

9 files changed

+105
-103
lines changed

‎src/data.providers/arrow.data.provider.ts‎

Lines changed: 45 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -32,57 +32,57 @@ export class ArrowDataProvider implements IDataProvider {
3232
* @param parseOptions Data parse options.
3333
* @param loadData Load data callback.
3434
*/
35-
public getData(dataUrl: string, parseOptions: any, loadData: Function): void {
35+
public async getData(dataUrl: string, parseOptions: any, loadData: Function): Promise<void> {
3636
const dataFileType: string = dataUrl.substr(dataUrl.lastIndexOf('.')); // file extension
3737

38-
// get binary arrow data
39-
const dataBuffer: Buffer = fileUtils.readDataFile(dataUrl);
40-
41-
// create typed data array
42-
const dataArray: Uint8Array = new Uint8Array(dataBuffer);
43-
this.logger.debug('getData(): data size in bytes:', dataArray.byteLength.toLocaleString());
44-
45-
// create arrow table
46-
const dataTable: Table = Table.from(dataArray);
47-
48-
// remap arrow data schema to columns for data viewer
49-
const dataSchema = {};
50-
dataTable.schema.fields.map(field => {
51-
let fieldType: string = field.type.toString();
52-
const typesIndex: number = fieldType.indexOf('<');
53-
if (typesIndex > 0) {
54-
fieldType = fieldType.substring(0, typesIndex);
38+
// read binary Arrow data
39+
await fileUtils.readDataFile(dataUrl, null).then((dataBuffer: Buffer) => {
40+
// create typed data array
41+
const dataArray: Uint8Array = new Uint8Array(dataBuffer);
42+
this.logger.debug('getData(): data size in bytes:', dataArray.byteLength.toLocaleString());
43+
44+
// create arrow table
45+
const dataTable: Table = Table.from(dataArray);
46+
47+
// remap arrow data schema to columns for data viewer
48+
const dataSchema = {};
49+
dataTable.schema.fields.map(field => {
50+
let fieldType: string = field.type.toString();
51+
const typesIndex: number = fieldType.indexOf('<');
52+
if (typesIndex > 0) {
53+
fieldType = fieldType.substring(0, typesIndex);
54+
}
55+
dataSchema[field.name] = config.dataTypes[fieldType];
56+
});
57+
// cache arrow data view schema
58+
this.dataSchemaMap[dataUrl] = dataSchema;
59+
60+
// create Arrow schema.json file for data schema text data preview
61+
const dataSchemaFilePath: string = dataUrl.replace(dataFileType, '.schema.json');
62+
if (parseOptions.createJsonSchema && !fs.existsSync(dataSchemaFilePath)) {
63+
fileUtils.createJsonFile(dataSchemaFilePath, dataTable.schema);
5564
}
56-
dataSchema[field.name] = config.dataTypes[fieldType];
57-
});
58-
// cache arrow data view schema
59-
this.dataSchemaMap[dataUrl] = dataSchema;
6065

61-
// create Arrow schema.json file for data schema text data preview
62-
const dataSchemaFilePath: string = dataUrl.replace(dataFileType, '.schema.json');
63-
if (parseOptions.createJsonSchema && !fs.existsSync(dataSchemaFilePath)) {
64-
fileUtils.createJsonFile(dataSchemaFilePath, dataTable.schema);
65-
}
66-
67-
// create arrow data.json for text arrow data preview
68-
let dataRows: Array<any> = [];
69-
const jsonFilePath: string = dataUrl.replace(dataFileType, '.json');
70-
if (parseOptions.createJsonFiles && !fs.existsSync(jsonFilePath)) {
71-
// convert arrow table data to array of objects (happens only on the 1st run :)
72-
dataRows = Array(dataTable.length);
73-
const fields = dataTable.schema.fields.map(field => field.name);
74-
for (let i=0, n=dataRows.length; i<n; ++i) {
75-
const proto = {};
76-
fields.forEach((fieldName, index) => {
77-
const column = dataTable.getColumnAt(index);
78-
proto[fieldName] = column.get(i);
79-
});
80-
dataRows[i] = proto;
66+
// create arrow data.json for text arrow data preview
67+
let dataRows: Array<any> = [];
68+
const jsonFilePath: string = dataUrl.replace(dataFileType, '.json');
69+
if (parseOptions.createJsonFiles && !fs.existsSync(jsonFilePath)) {
70+
// convert arrow table data to array of objects (happens only on the 1st run :)
71+
dataRows = Array(dataTable.length);
72+
const fields = dataTable.schema.fields.map(field => field.name);
73+
for (let i=0, n=dataRows.length; i<n; ++i) {
74+
const proto = {};
75+
fields.forEach((fieldName, index) => {
76+
const column = dataTable.getColumnAt(index);
77+
proto[fieldName] = column.get(i);
78+
});
79+
dataRows[i] = proto;
80+
}
81+
fileUtils.createJsonFile(jsonFilePath, dataRows);
8182
}
82-
fileUtils.createJsonFile(jsonFilePath, dataRows);
83-
}
8483

85-
loadData(dataArray);
84+
loadData(dataArray);
85+
});
8686
} // end of getData()
8787

8888
/**

‎src/data.providers/excel.data.provider.ts‎

Lines changed: 42 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -34,51 +34,53 @@ export class ExcelDataProvider implements IDataProvider {
3434
* @param parseOptions Data parse options.
3535
* @param loadData Load data callback.
3636
*/
37-
public getData(dataUrl: string, parseOptions: any, loadData: Function): void {
38-
// load Excel workbook
37+
public async getData(dataUrl: string, parseOptions: any, loadData: Function): Promise<void> {
3938
const dataFileName: string = path.basename(dataUrl);
4039
const dataFileType: string = dataUrl.substr(dataUrl.lastIndexOf('.')); // file extension
41-
const dataBuffer: Buffer = fileUtils.readDataFile(dataUrl);
42-
const workbook: xlsx.WorkBook = xlsx.read(dataBuffer, {
43-
cellDates: true,
44-
});
40+
// read Excel file data
41+
await fileUtils.readDataFile(dataUrl, null).then((dataBuffer: Buffer) => {
42+
// create Excel 'workbook'
43+
const workbook: xlsx.WorkBook = xlsx.read(dataBuffer, {
44+
cellDates: true,
45+
});
46+
47+
// load data sheets
48+
let dataRows: Array<any> = [];
49+
this.dataTableNamesMap[dataUrl] = [];
50+
if (workbook.SheetNames.length > 0) {
51+
if (workbook.SheetNames.length > 1) {
52+
// cache sheet names
53+
this.dataTableNamesMap[dataUrl] = workbook.SheetNames;
54+
this.logger.debug(`getData(): file: ${dataFileName} sheetNames:`, workbook.SheetNames);
55+
}
4556

46-
// load data sheets
47-
let dataRows: Array<any> = [];
48-
this.dataTableNamesMap[dataUrl] = [];
49-
if (workbook.SheetNames.length > 0) {
50-
if (workbook.SheetNames.length > 1) {
51-
// cache sheet names
52-
this.dataTableNamesMap[dataUrl] = workbook.SheetNames;
53-
this.logger.debug(`getData(): file: ${dataFileName} sheetNames:`, workbook.SheetNames);
54-
}
55-
56-
// determine spreadsheet to load
57-
let sheetName:string = workbook.SheetNames[0];
58-
if (parseOptions.dataTable.length > 0 &&
59-
workbook.SheetNames.indexOf(parseOptions.dataTable) >= 0) {
60-
// reset to requested table name
61-
sheetName = parseOptions.dataTable;
62-
}
63-
64-
// get worksheet data row objects array
65-
const worksheet: xlsx.Sheet = workbook.Sheets[sheetName];
66-
dataRows = xlsx.utils.sheet_to_json(worksheet);
67-
68-
// create json data file for binary Excel file text data preview
69-
if (parseOptions.createJsonFiles && config.supportedBinaryDataFiles.test(dataFileName)) {
70-
// create json data file path
71-
let jsonFilePath: string = dataUrl.replace(dataFileType, '.json');
72-
if (parseOptions.dataTable.length > 0 && workbook.SheetNames.length > 1) {
73-
// append table name to the generated json data file name
74-
jsonFilePath = jsonFilePath.replace('.json', `-${parseOptions.dataTable}.json`);
57+
// determine spreadsheet to load
58+
let sheetName:string = workbook.SheetNames[0];
59+
if (parseOptions.dataTable.length > 0 &&
60+
workbook.SheetNames.indexOf(parseOptions.dataTable) >= 0) {
61+
// reset to requested table name
62+
sheetName = parseOptions.dataTable;
7563
}
76-
if (!fs.existsSync(jsonFilePath)) {
77-
fileUtils.createJsonFile(jsonFilePath, dataRows);
64+
65+
// get worksheet data row objects array
66+
const worksheet: xlsx.Sheet = workbook.Sheets[sheetName];
67+
dataRows = xlsx.utils.sheet_to_json(worksheet);
68+
69+
// create json data file for binary Excel file text data preview
70+
if (parseOptions.createJsonFiles && config.supportedBinaryDataFiles.test(dataFileName)) {
71+
// create json data file path
72+
let jsonFilePath: string = dataUrl.replace(dataFileType, '.json');
73+
if (parseOptions.dataTable.length > 0 && workbook.SheetNames.length > 1) {
74+
// append table name to the generated json data file name
75+
jsonFilePath = jsonFilePath.replace('.json', `-${parseOptions.dataTable}.json`);
76+
}
77+
if (!fs.existsSync(jsonFilePath)) {
78+
fileUtils.createJsonFile(jsonFilePath, dataRows);
79+
}
7880
}
79-
}
80-
}
81-
loadData(dataRows);
81+
}
82+
loadData(dataRows);
83+
});
8284
} // end of getData()
8385

8486
/**

‎src/data.providers/hjson.data.provider.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,10 @@ export class HjsonDataProvider implements IDataProvider {
3030
* @param parseOptions Data parse options.
3131
* @param loadData Load data callback.
3232
*/
33-
public getData(dataUrl: string, parseOptions: any, loadData: Function): void {
33+
public async getData(dataUrl: string, parseOptions: any, loadData: Function): Promise<void> {
3434
let data: any = [];
3535
try {
36-
let content: string = fileUtils.readDataFile(dataUrl, 'utf8');
36+
let content: string = String(await fileUtils.readDataFile(dataUrl, 'utf8'));
3737
data = hjson.parse(content);
3838
}
3939
catch (error) {

‎src/data.providers/json.data.provider.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,10 @@ export class JsonDataProvider implements IDataProvider {
2929
* @param parseOptions Data parse options.
3030
* @param loadData Load data callback.
3131
*/
32-
public getData(dataUrl: string, parseOptions: any, loadData: Function): void {
32+
public async getData(dataUrl: string, parseOptions: any, loadData: Function): Promise<void> {
3333
let data: any = [];
3434
try {
35-
let content: string = fileUtils.readDataFile(dataUrl, 'utf8');
35+
let content: string = String(await fileUtils.readDataFile(dataUrl, 'utf8'));
3636
if (dataUrl.endsWith('.json')) {
3737
// strip out comments from vscode settings .json :)
3838
const comments: RegExp = new RegExp(/\/\*[\s\S]*?\*\/|\/\/.*/g);

‎src/data.providers/json5.data.provider.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,10 @@ export class Json5DataProvider implements IDataProvider {
3030
* @param parseOptions Data parse options.
3131
* @param loadData Load data callback.
3232
*/
33-
public getData(dataUrl: string, parseOptions: any, loadData: Function): void {
33+
public async getData(dataUrl: string, parseOptions: any, loadData: Function): Promise<void> {
3434
let data: any = [];
3535
try {
36-
let content: string = fileUtils.readDataFile(dataUrl, 'utf8');
36+
let content: string = String(await fileUtils.readDataFile(dataUrl, 'utf8'));
3737
data = json5.parse(content);
3838
}
3939
catch (error) {

‎src/data.providers/markdown.data.provider.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,11 @@ export class MarkdownDataProvider implements IDataProvider {
3232
* @param parseOptions Data parse options.
3333
* @param loadData Load data callback.
3434
*/
35-
public getData(dataUrl: string, parseOptions: any, loadData: Function): void {
35+
public async getData(dataUrl: string, parseOptions: any, loadData: Function): Promise<void> {
3636
let content: string = '';
3737
try {
3838
// read markdown file content
39-
content = fileUtils.readDataFile(dataUrl, 'utf8');
39+
content = String(await fileUtils.readDataFile(dataUrl, 'utf8'));
4040

4141
// convert it to to CSV for loading into data view
4242
content = this.markdownToCsv(dataUrl, content, parseOptions.dataTable);

‎src/data.providers/properties.data.provider.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ export class PropertiesDataProvider implements IDataProvider {
3030
* @param parseOptions Data parse options.
3131
* @param loadData Load data callback.
3232
*/
33-
public getData(dataUrl: string, parseOptions: any, loadData: Function): void {
33+
public async getData(dataUrl: string, parseOptions: any, loadData: Function): Promise<void> {
3434
let data: any = [];
3535
// TODO: add mime types later for remote http data loading
3636
const dataFileType: string = dataUrl.substr(dataUrl.lastIndexOf('.')); // file extension
3737
try {
3838
// read and parse properties file type data
39-
let content: string = fileUtils.readDataFile(dataUrl, 'utf8');
39+
let content: string = String(await fileUtils.readDataFile(dataUrl, 'utf8'));
4040
data = props.parse(content, this.getDataParseOptions(dataFileType));
4141
}
4242
catch (error) {

‎src/data.providers/yaml.data.provider.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,10 @@ export class YamlDataProvider implements IDataProvider {
2929
* @param parseOptions Data parse options.
3030
* @param loadData Load data callback.
3131
*/
32-
public getData(dataUrl: string, parseOptions: any, loadData: Function): void {
32+
public async getData(dataUrl: string, parseOptions: any, loadData: Function): Promise<void> {
3333
let data: any = [];
3434
try {
35-
let content: string = fileUtils.readDataFile(dataUrl, 'utf8');
35+
let content: string = String(await fileUtils.readDataFile(dataUrl, 'utf8'));
3636
data = yaml.load(content);
3737
}
3838
catch (error) {

‎src/utils/file.utils.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import * as fs from 'fs';
2+
import * as util from 'util';
23
import * as request from 'superagent';
34
import * as path from 'path';
45
import * as config from '../config';
56
import {Logger, LogLevel} from '../logger';
67
import {window, workspace, Uri} from 'vscode';
78

9+
const readFileAsync = util.promisify(fs.readFile);
810
const fileSizeLabels: string[] = ['bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
911
const logger: Logger = new Logger(`file.utils:`, config.logLevel);
1012

@@ -13,7 +15,7 @@ const logger: Logger = new Logger(`file.utils:`, config.logLevel);
1315
* @param dataFilePath Data file path or public data source url.
1416
* @param encoding Data file encoding: 'utf8' for text data files, null for binary data reads.
1517
*/
16-
export function readDataFile(dataFilePath: string, encoding:string = null) {
18+
export async function readDataFile(dataFilePath: string, encoding:string = null): Promise<string | Buffer> {
1719
let data: any = '';
1820
const fileName: string = path.basename(dataFilePath);
1921
const fileUri: Uri = Uri.file(dataFilePath); // .parse(dataFilePath);
@@ -29,7 +31,7 @@ export function readDataFile(dataFilePath: string, encoding:string = null) {
2931
}
3032
else if (fs.existsSync(fileUri.fsPath)) {
3133
// read local data file via fs.readFile() api
32-
data = readLocalData(fileUri.fsPath, encoding);
34+
data = await readLocalData(fileUri.fsPath, encoding);
3335
}
3436
else {
3537
// try to find requested data file(s) in open workspace
@@ -102,12 +104,10 @@ export function createJsonFile(jsonFilePath: string, jsonData: any): void {
102104
* TODO: change this to read data async later
103105
* TODO: rework this to using fs.ReadStream for large data files support later
104106
*/
105-
function readLocalData(dataFilePath: string, encoding: string = null): any {
106-
let data: any = '';
107+
async function readLocalData(dataFilePath: string, encoding: string = null): Promise<string | Buffer> {
107108
logger.debug('readLocalData():', dataFilePath);
108109
// read local data file via fs read file api
109-
data = fs.readFileSync(dataFilePath, encoding);
110-
return data;
110+
return await readFileAsync(dataFilePath, encoding);
111111
}
112112

113113
/**

0 commit comments

Comments
 (0)